diff --git a/.github/workflows/verify-pr.yml b/.github/workflows/verify-pr.yml index 015bc07cd..ef9420ab8 100644 --- a/.github/workflows/verify-pr.yml +++ b/.github/workflows/verify-pr.yml @@ -106,26 +106,26 @@ jobs: run: | source venv/bin/activate convert_if_missing() { - local model_id=$1 output_dir=$2 precision=$3 + local model_id=$1 output_dir=$2 bits=$3 if [ ! -f "weights/$output_dir/config.txt" ]; then echo "Converting $model_id -> weights/$output_dir" - python3 -m src.cli convert "$model_id" "weights/$output_dir" --precision "$precision" ${HF_TOKEN:+--token "$HF_TOKEN"} || echo "WARNING: conversion of $model_id failed (exit $?), checking fallback directory" + python3 -m cactus convert "$model_id" "weights/$output_dir" --bits "$bits" ${HF_TOKEN:+--token "$HF_TOKEN"} || echo "WARNING: conversion of $model_id failed (exit $?), checking fallback directory" local default_dir="weights/$(echo "$model_id" | sed 's|.*/||' | tr '[:upper:]' '[:lower:]')" if [ ! -f "weights/$output_dir/config.txt" ] && [ -f "$default_dir/config.txt" ]; then mv "$default_dir" "weights/$output_dir" fi fi } - convert_if_missing "Qwen/Qwen3-0.6B" "qwen3-0.6b" "INT8" - convert_if_missing "nomic-ai/nomic-embed-text-v2-moe" "nomic-embed-text-v2-moe" "INT8" - convert_if_missing "LiquidAI/LFM2.5-350M" "lfm2-350m" "INT8" - convert_if_missing "LiquidAI/LFM2-8B-A1B" "lfm2-8b-a1b" "INT8" - convert_if_missing "google/gemma-3-270m-it" "gemma-3-270m-it" "INT8" - convert_if_missing "openai/whisper-small" "whisper-small" "INT8" - convert_if_missing "UsefulSensors/moonshine-base" "moonshine-base" "INT8" - convert_if_missing "LiquidAI/LFM2-VL-450M" "lfm2-vl-450m-int8" "INT8" - convert_if_missing "LiquidAI/LFM2-VL-450M" "lfm2-vl-450m-int4" "INT4" - convert_if_missing "LiquidAI/LFM2-VL-450M" "lfm2-vl-450m-fp16" "FP16" + convert_if_missing "Qwen/Qwen3-0.6B" "qwen3-0.6b" "4" + convert_if_missing "nomic-ai/nomic-embed-text-v2-moe" "nomic-embed-text-v2-moe" "4" + convert_if_missing "LiquidAI/LFM2.5-350M" "lfm2-350m" "4" + convert_if_missing "LiquidAI/LFM2-8B-A1B" "lfm2-8b-a1b" "4" + convert_if_missing "google/gemma-3-270m-it" "gemma-3-270m-it" "4" + convert_if_missing "openai/whisper-small" "whisper-small" "4" + convert_if_missing "UsefulSensors/moonshine-base" "moonshine-base" "4" + convert_if_missing "LiquidAI/LFM2-VL-450M" "lfm2-vl-450m-int8" "4" + convert_if_missing "LiquidAI/LFM2-VL-450M" "lfm2-vl-450m-int4" "4" + convert_if_missing "LiquidAI/LFM2-VL-450M" "lfm2-vl-450m-fp16" "4" - name: Build library run: bash cactus/build.sh diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index a6ff0470f..13a5c9023 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -74,29 +74,17 @@ set(ENGINE_SOURCES src/npu.cpp ) -set(MODEL_SOURCES - models/model_lfm2.cpp - models/model_lfm2vl.cpp - models/model_qwen.cpp - models/model_siglip2.cpp - models/gemma4/model_gemma4.cpp - models/gemma4/model_gemma4_mm.cpp - models/gemma4/model_gemma4_audio.cpp - models/gemma4/model_gemma4_vision.cpp -) - if(APPLE) enable_language(OBJCXX) list(APPEND ENGINE_SOURCES src/npu_ane.mm) set_source_files_properties(src/npu_ane.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc") endif() -add_library(cactus_engine STATIC ${ENGINE_SOURCES} ${MODEL_SOURCES}) +add_library(cactus_engine STATIC ${ENGINE_SOURCES}) set(CACTUS_KERNELS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../cactus-kernels") target_include_directories(cactus_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src - ${CMAKE_CURRENT_SOURCE_DIR}/models ${CACTUS_KERNELS_DIR}/src ${CACTUS_KERNELS_DIR}/libs ) diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index b025cd0fe..768b18822 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -292,6 +292,7 @@ CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, i CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_concat( cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, @@ -312,6 +313,8 @@ CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( cactus_graph_t graph, const char* filename, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_bind_mmap_weights( + cactus_graph_t graph, cactus_node_t node, const char* filename); CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); diff --git a/cactus-engine/models/gemma4/model_gemma4.cpp b/cactus-engine/models/gemma4/model_gemma4.cpp deleted file mode 100644 index 6a67d77ce..000000000 --- a/cactus-engine/models/gemma4/model_gemma4.cpp +++ /dev/null @@ -1,610 +0,0 @@ -#include "model_gemma4.h" -#include "cactus_graph.h" -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Gemma4Model::Gemma4Model() : Model() {} - -Gemma4Model::Gemma4Model(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -bool Gemma4Model::is_global_layer(uint32_t idx) const { - return idx < config_.layer_types.size() && - (config_.layer_types[idx] == "global" || config_.layer_types[idx] == "full_attention"); -} - -std::vector Gemma4Model::get_kv_layer_dims() const { - uint32_t n = config_.num_layers; - - std::vector dims(n); - for (uint32_t i = 0; i < n; i++) { - if (i >= first_shared_layer_) { - dims[i] = 0; - } else if (is_global_layer(i)) { - dims[i] = config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2; - } else { - dims[i] = config_.attention_head_dim; - } - } - return dims; -} - -std::vector Gemma4Model::get_kv_layer_heads() const { - uint32_t n = config_.num_layers; - - std::vector heads(n); - for (uint32_t i = 0; i < n; i++) { - if (i >= first_shared_layer_) { - heads[i] = 0; - } else if (is_global_layer(i) && config_.num_global_kv_heads > 0) { - heads[i] = config_.num_global_kv_heads; - } else { - heads[i] = config_.attention_kv_heads; - } - } - return heads; -} - -std::vector Gemma4Model::get_kv_layer_windows() const { - uint32_t n = config_.num_layers; - std::vector windows(n, 0); - for (uint32_t i = 0; i < n; i++) { - if (i >= first_shared_layer_) { - windows[i] = 0; // shared layers don't have own cache - } else if (!is_global_layer(i)) { - windows[i] = config_.sliding_window; // local layers use sliding window - } - } - return windows; -} - -void Gemma4Model::compact_kv_cache() { -} - -void Gemma4Model::post_init() { - uint32_t n = config_.num_layers; - - cache_window_size_ = 0; // disable uniform window; per-layer windows via get_kv_layer_windows() - - auto* gb = static_cast(graph_handle_); - init_graph_cache(gb); - - kv_share_map_.resize(n, -1); - shared_k_nodes_.resize(n, 0); - shared_v_nodes_.resize(n, 0); - - for (uint32_t i = first_shared_layer_; i < n; i++) { - bool is_global = is_global_layer(i); - for (int j = static_cast(first_shared_layer_) - 1; j >= 0; j--) { - if (is_global_layer(j) == is_global) { - kv_share_map_[i] = j; - break; - } - } - } -} - -void Gemma4Model::load_weights_to_graph(CactusGraph* gb) { - uint32_t n = config_.num_layers; - uint32_t num_shared = config_.num_kv_shared_layers; - first_shared_layer_ = (n > num_shared) ? n - num_shared : n; - - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - bool has_pli = config_.hidden_size_per_layer_input > 0; - if (has_pli) { - weight_nodes_.embed_tokens_per_layer = gb->mmap_embeddings(model_folder_path_ + "/embed_tokens_per_layer.weights"); - weight_nodes_.per_layer_model_proj = gb->mmap_weights(model_folder_path_ + "/per_layer_model_proj.weights"); - weight_nodes_.per_layer_proj_norm = gb->mmap_weights(model_folder_path_ + "/per_layer_proj_norm.weights"); - } - - bool has_moe = config_.enable_moe_block; - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - bool is_shared = (i >= first_shared_layer_); - - layer.attn_q_weight = gb->mmap_weights(prefix + "attn_q.weights"); - layer.attn_k_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k.weights"); - bool k_eq_v = config_.attention_k_eq_v && is_global_layer(i); - layer.attn_v_weight = is_shared ? 0 : (k_eq_v ? layer.attn_k_weight : gb->mmap_weights(prefix + "attn_v.weights")); - layer.attn_output_weight = gb->mmap_weights(prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(prefix + "post_attn_norm.weights"); - layer.pre_feedforward_layernorm_weight = gb->mmap_weights(prefix + "pre_ffn_norm.weights"); - layer.post_feedforward_layernorm_weight = gb->mmap_weights(prefix + "post_ffn_norm.weights"); - if (has_pli) { - layer.per_layer_gate = gb->mmap_weights(prefix + "per_layer_gate.weights"); - layer.per_layer_proj = gb->mmap_weights(prefix + "per_layer_proj.weights"); - layer.post_per_layer_norm = gb->mmap_weights(prefix + "post_per_layer_norm.weights"); - } - layer.layer_scalar = std::filesystem::exists(prefix + "layer_scalar.weights") ? gb->mmap_weights(prefix + "layer_scalar.weights") : 0; - - if (has_moe) { - uint32_t num_experts = config_.num_experts; - uint32_t hidden = config_.hidden_dim; - uint32_t expert_dim = config_.expert_intermediate_size > 0 ? config_.expert_intermediate_size : config_.ffn_intermediate_dim; - layer.moe_w1_experts.resize(num_experts); - layer.moe_w3_experts.resize(num_experts); - layer.moe_w2_experts.resize(num_experts); - - auto w1_packed = gb->mmap_weights(prefix + "moe_gate_proj.weights"); - auto w3_packed = gb->mmap_weights(prefix + "moe_up_proj.weights"); - auto w2_packed = gb->mmap_weights(prefix + "moe_down_proj.weights"); - - const auto& w1_buf = gb->get_output_buffer(w1_packed); - const auto& w3_buf = gb->get_output_buffer(w3_packed); - const auto& w2_buf = gb->get_output_buffer(w2_packed); - - auto setup_experts = [&](const BufferDesc& buf, - std::vector& expert_nodes, - size_t out_dim, size_t in_dim) { - auto* base = static_cast(const_cast(buf.get_data())); - Precision prec = buf.precision; - size_t K = in_dim; - size_t expert_data_bytes = PrecisionTraits::packed_size_of(prec, out_dim * K); - - for (uint32_t e = 0; e < num_experts; e++) { - expert_nodes[e] = gb->input({out_dim, K}, prec); - gb->set_external_input(expert_nodes[e], base + e * expert_data_bytes, prec); - } - }; - - setup_experts(w1_buf, layer.moe_w1_experts, expert_dim, hidden); - setup_experts(w3_buf, layer.moe_w3_experts, expert_dim, hidden); - setup_experts(w2_buf, layer.moe_w2_experts, hidden, expert_dim); - - layer.router_proj = gb->mmap_weights(prefix + "router_proj.weights"); - layer.router_scale = gb->mmap_weights(prefix + "router_scale.weights"); - layer.moe_per_expert_scale = gb->mmap_weights(prefix + "moe_per_expert_scale.weights"); - layer.post_ffn_norm_1 = gb->mmap_weights(prefix + "post_ffn_norm_1.weights"); - layer.pre_ffn_norm_2 = gb->mmap_weights(prefix + "pre_ffn_norm_2.weights"); - layer.post_ffn_norm_2 = gb->mmap_weights(prefix + "post_ffn_norm_2.weights"); - } - } - - size_t sliding_head_dim = config_.attention_head_dim; - size_t global_head_dim = config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2; - v_norm_ones_weight_.assign(std::max(sliding_head_dim, global_head_dim), static_cast<__fp16>(1.0f)); - v_norm_ones_node_ = gb->input({sliding_head_dim}, Precision::FP16); - gb->set_external_input(v_norm_ones_node_, v_norm_ones_weight_.data(), Precision::FP16); - v_norm_ones_global_node_ = gb->input({global_head_dim}, Precision::FP16); - gb->set_external_input(v_norm_ones_global_node_, v_norm_ones_weight_.data(), Precision::FP16); - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - if (std::filesystem::exists(npu_prefill_path)) { - if (!load_npu_prefill(npu_prefill_path) || !has_npu_prefill()) { - CACTUS_LOG_DEBUG("npu", "[gemma4] found model.mlpackage but failed to enable NPU prefill; using CPU prefill"); - } - } - } -} - -size_t Gemma4Model::build_per_layer_input(CactusGraph* gb, size_t hidden, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto gate = gb->gelu(gb->matmul(hidden, layer.per_layer_gate, true, backend)); - auto pli_slice = gb->slice(pli_combined, 1, layer_idx * pli_dim, pli_dim); - auto gated = gb->multiply(gate, pli_slice); - auto pli_proj = gb->matmul(gated, layer.per_layer_proj, true, backend); - auto pli_normed = gb->rms_norm(pli_proj, layer.post_per_layer_norm, config_.layer_norm_eps); - - return gb->add(hidden, pli_normed); -} - -size_t Gemma4Model::apply_partial_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, - float rope_freq, size_t position_offset) { - if (rot_dim < head_dim) { - size_t half_dim = head_dim / 2; - size_t half_rot = rot_dim / 2; - size_t pass_len = half_dim - half_rot; - float adjusted_theta = std::pow(rope_freq, static_cast(rot_dim) / static_cast(head_dim)); - - auto left_rot = gb->slice(tensor, 3, 0, half_rot); - auto left_pass = gb->slice(tensor, 3, half_rot, pass_len); - auto right_rot = gb->slice(tensor, 3, half_dim, half_rot); - auto right_pass = gb->slice(tensor, 3, half_dim + half_rot, pass_len); - - auto rotated = gb->rope(gb->concat(left_rot, right_rot, 3), adjusted_theta, position_offset); - auto rotated_left = gb->slice(rotated, 3, 0, half_rot); - auto rotated_right = gb->slice(rotated, 3, half_rot, half_rot); - - auto new_left = gb->concat(rotated_left, left_pass, 3); - auto new_right = gb->concat(rotated_right, right_pass, 3); - return gb->concat(new_left, new_right, 3); - } - return gb->rope(tensor, rope_freq, position_offset); -} - -size_t Gemma4Model::build_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t seq_len = gb->get_output_buffer(input).shape[0]; - int share_src = (layer_idx < kv_share_map_.size()) ? kv_share_map_[layer_idx] : -1; - - bool is_global = is_global_layer(layer_idx); - - size_t head_dim = is_global ? (config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2) : config_.attention_head_dim; - size_t num_heads = config_.attention_heads; - size_t kv_heads = is_global && config_.num_global_kv_heads > 0 ? config_.num_global_kv_heads : config_.attention_kv_heads; - float rope_freq = is_global ? config_.rope_theta : config_.rope_local_base_freq; - size_t window = is_global ? 0 : config_.sliding_window; - size_t rot_dim = static_cast(head_dim * (is_global ? config_.global_partial_rotary_factor : 1.0f)); - - auto q = gb->matmul(input, layer.attn_q_weight, true, backend); - q = gb->reshape(q, {seq_len * num_heads, head_dim}); - q = gb->rms_norm(q, layer.attn_q_norm_weight, config_.layer_norm_eps); - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - - size_t q4 = apply_partial_rope(gb, q, head_dim, rot_dim, rope_freq, position_offset); - - size_t k4, v4; - if (share_src >= 0 && shared_k_nodes_[share_src] != 0) { - k4 = shared_k_nodes_[share_src]; - v4 = shared_v_nodes_[share_src]; - } else { - auto k = gb->matmul(input, layer.attn_k_weight, true, backend); - k = gb->reshape(k, {seq_len * kv_heads, head_dim}); - k = gb->rms_norm(k, layer.attn_k_norm_weight, config_.layer_norm_eps); - k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); - - k4 = apply_partial_rope(gb, k, head_dim, rot_dim, rope_freq, position_offset); - - auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); - size_t v_ones = is_global ? v_norm_ones_global_node_ : v_norm_ones_node_; - auto v = gb->rms_norm(gb->reshape(v_proj, {seq_len * kv_heads, head_dim}), v_ones, config_.layer_norm_eps); - v4 = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); - - shared_k_nodes_[layer_idx] = k4; - shared_v_nodes_[layer_idx] = v4; - } - - size_t cache_src = (share_src >= 0) ? static_cast(share_src) : layer_idx; - size_t attn; - if (use_cache && graph_cache_k_nodes_[cache_src] != 0) { - if (share_src < 0) { - gb->kv_cache_append(k4, graph_cache_k_nodes_[cache_src], window, cache_sink_size_); - gb->kv_cache_append(v4, graph_cache_v_nodes_[cache_src], window, cache_sink_size_); - } - attn = gb->attention_cached(q4, k4, v4, - graph_cache_k_nodes_[cache_src], graph_cache_v_nodes_[cache_src], - attention_scale_, position_offset, window); - } else { - attn = gb->attention(q4, k4, v4, attention_scale_, position_offset, window); - } - - return gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), layer.attn_output_weight, true, backend); -} - -size_t Gemma4Model::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - static const bool dense_mlp_fused_disabled = []() { - const char* env = std::getenv("CACTUS_DISABLE_DENSE_MLP_FUSED"); - return env && env[0] && env[0] != '0'; - }(); - if (!dense_mlp_fused_disabled) { - const auto& gate_buf = gb->get_output_buffer(layer.ffn_gate_weight); - const auto& up_buf = gb->get_output_buffer(layer.ffn_up_weight); - const auto& down_buf = gb->get_output_buffer(layer.ffn_down_weight); - if (gate_buf.precision == Precision::CQ4 && up_buf.precision == Precision::CQ4 && - down_buf.precision == Precision::CQ4 && - gate_buf.group_size > 0 && up_buf.group_size > 0 && down_buf.group_size > 0) { - return gb->dense_mlp_tq_fused(input, layer.ffn_gate_weight, layer.ffn_up_weight, layer.ffn_down_weight); - } - } - auto gate = gb->gelu(gb->matmul(input, layer.ffn_gate_weight, true, backend)); - auto up = gb->matmul(input, layer.ffn_up_weight, true, backend); - return gb->matmul(gb->multiply(gate, up), layer.ffn_down_weight, true, backend); -} - -size_t Gemma4Model::build_moe(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto router_logits = gb->matmul(input, layer.router_proj, true, backend); - auto topk_result = gb->topk(router_logits, config_.num_experts_per_tok); - auto topk_indices = gb->index(topk_result, 0, 0); - auto routing_probs = gb->softmax(router_logits); - - return gb->moe_layer(input, routing_probs, topk_indices, - layer.moe_w1_experts, layer.moe_w3_experts, layer.moe_w2_experts, - config_.num_experts, config_.num_experts_per_tok, - true, 1e-6f, 1.0f, Activation::GELU, - layer.moe_per_expert_scale); -} - -size_t Gemma4Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto normed = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_raw = build_attention(gb, normed, layer_idx, backend, use_cache, position_offset); - auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto residual = gb->add(hidden, attn); - - if (config_.enable_moe_block) { - auto h1 = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - h1 = build_mlp(gb, h1, layer_idx, backend); - h1 = gb->rms_norm(h1, layer.post_ffn_norm_1, config_.layer_norm_eps); - - auto h2 = gb->multiply(residual, layer.router_scale); - h2 = gb->rms_norm(h2, layer.pre_ffn_norm_2, config_.layer_norm_eps); - h2 = build_moe(gb, h2, layer_idx, backend); - h2 = gb->rms_norm(h2, layer.post_ffn_norm_2, config_.layer_norm_eps); - - auto combined = gb->rms_norm(gb->add(h1, h2), layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - return gb->add(residual, combined); - } - - auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - auto mlp_raw = build_mlp(gb, pre_mlp, layer_idx, backend); - auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - - return gb->add(residual, mlp); -} - -size_t Gemma4Model::apply_transformer_layer(CactusGraph* gb, size_t hidden, size_t pli, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t pos_offset) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, pos_offset); - if (config_.hidden_size_per_layer_input > 0) - hidden = build_per_layer_input(gb, hidden, pli, layer_idx, backend); - if (weight_nodes_.layers[layer_idx].layer_scalar != 0) - hidden = gb->multiply(hidden, weight_nodes_.layers[layer_idx].layer_scalar); - return hidden; -} - -size_t Gemma4Model::build_pli_combined(CactusGraph* gb, size_t hidden, size_t pli_embed, - size_t seq_len, ComputeBackend backend) { - uint32_t num_layers = config_.num_layers; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto pli_proj = gb->scalar_multiply(gb->matmul(hidden, weight_nodes_.per_layer_model_proj, true, backend), - 1.0f / std::sqrt(static_cast(config_.hidden_dim))); - pli_proj = gb->reshape(pli_proj, {seq_len * num_layers, pli_dim}); - pli_proj = gb->rms_norm(pli_proj, weight_nodes_.per_layer_proj_norm, config_.layer_norm_eps); - pli_proj = gb->reshape(pli_proj, {seq_len, num_layers * pli_dim}); - return gb->scalar_multiply(gb->add(pli_proj, pli_embed), 1.0f / std::sqrt(2.0f)); -} - -std::pair Gemma4Model::build_preamble_and_embed(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input) { - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - token_input = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - pli_input = gb->input({seq_len}, Precision::FP32); - auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), - std::sqrt(static_cast(pli_dim))); - auto pli_combined = build_pli_combined(gb, hidden, pli_embed, seq_len, backend); - - return {hidden, pli_combined}; -} - -void Gemma4Model::set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens) { - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - gb->set_input(pli_input, input_data.data(), Precision::FP32); -} - -size_t Gemma4Model::forward_from_embeddings(CactusGraph* gb, size_t hidden, const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend, bool use_cache) { - return forward_from_embeddings(gb, hidden, hidden, pli_tokens, seq_len, backend, use_cache); -} - -size_t Gemma4Model::build_pli_combined_from_tokens(CactusGraph* gb, size_t hidden, - const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend) { - if (config_.hidden_size_per_layer_input == 0) - return 0; - - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto pli_input = gb->input({seq_len}, Precision::FP32); - auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), - std::sqrt(static_cast(pli_dim))); - auto pli_combined = build_pli_combined(gb, hidden, pli_embed, seq_len, backend); - - std::vector pli_data(pli_tokens.size()); - for (size_t i = 0; i < pli_tokens.size(); i++) - pli_data[i] = static_cast(pli_tokens[i]); - gb->set_input(pli_input, pli_data.data(), Precision::FP32); - - return pli_combined; -} - -size_t Gemma4Model::forward_from_embeddings(CactusGraph* gb, size_t hidden, size_t pli_hidden_source, - const std::vector& pli_tokens, size_t seq_len, - ComputeBackend backend, bool use_cache) { - size_t pos_offset = use_cache ? cache_total_seq_len_ : 0; - - std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); - std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); - - if (config_.hidden_size_per_layer_input == 0) { - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - } - - auto pli_combined = build_pli_combined_from_tokens(gb, pli_hidden_source, pli_tokens, seq_len, backend); - - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, pli_combined, i, backend, use_cache, pos_offset); - - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - -size_t Gemma4Model::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - if (tokens.empty()) - throw std::runtime_error("Token sequence cannot be empty"); - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); - std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); - - size_t pos_offset = use_cache ? cache_total_seq_len_ : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - if (config_.hidden_size_per_layer_input == 0) { - auto token_input = gb->input({tokens.size()}, Precision::FP32); - auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); - - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - } - - size_t token_input, pli_input; - auto hidden_pli = build_preamble_and_embed(gb, tokens.size(), backend, token_input, pli_input); - size_t hidden = hidden_pli.first, pli = hidden_pli.second; - - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, pli, i, backend, use_cache, pos_offset); - - set_token_inputs(gb, token_input, pli_input, tokens); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - -size_t Gemma4Model::forward_split(const std::vector& tokens, bool use_cache) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); - std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); - - size_t seq_len = tokens.size(); - size_t pos_offset = use_cache ? cache_total_seq_len_ : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - if (config_.hidden_size_per_layer_input == 0) { - auto token_input = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - for (uint32_t i = 0; i < first_shared_layer_; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); - - hidden = gb->index(hidden, seq_len - 1, 0); - hidden = gb->reshape(hidden, {1, config_.hidden_dim}); - - size_t shared_pos_offset = pos_offset + seq_len - 1; - for (uint32_t i = first_shared_layer_; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, shared_pos_offset); - - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - } - - size_t token_input, pli_input; - auto hidden_pli = build_preamble_and_embed(gb, seq_len, backend, token_input, pli_input); - size_t hidden = hidden_pli.first, pli = hidden_pli.second; - - for (uint32_t i = 0; i < first_shared_layer_; i++) - hidden = apply_transformer_layer(gb, hidden, pli, i, backend, use_cache, pos_offset); - - hidden = gb->index(hidden, seq_len - 1, 0); - hidden = gb->reshape(hidden, {1, config_.hidden_dim}); - auto pli_last = gb->index(pli, seq_len - 1, 0); - pli_last = gb->reshape(pli_last, {1, config_.num_layers * config_.hidden_size_per_layer_input}); - - size_t shared_pos_offset = pos_offset + seq_len - 1; - for (uint32_t i = first_shared_layer_; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, pli_last, i, backend, use_cache, shared_pos_offset); - - set_token_inputs(gb, token_input, pli_input, tokens); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - - -void Gemma4Model::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (tokens.empty()) - return; - - if (has_npu_prefill()) { - size_t npu_chunk_size = static_cast(npu_prefill_->get_chunk_size()); - if (tokens.size() > npu_chunk_size) { - Model::prefill(tokens, chunk_size, profile_file); - return; - } - } - - static constexpr size_t SPLIT_PREFILL_MIN_TOKENS = 32; - bool use_split = config_.num_kv_shared_layers > 0 - && tokens.size() >= SPLIT_PREFILL_MIN_TOKENS - && !std::getenv("CACTUS_DISABLE_SPLIT_PREFILL"); - - if (!use_split) { - Model::prefill(tokens, chunk_size, profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - auto process_chunk = [&](const std::vector& chunk) { - forward_split(chunk, true); - gb->execute(profile_file); - cache_total_seq_len_ += chunk.size(); - }; - - if (tokens.size() <= chunk_size) { - process_chunk(tokens); - return; - } - - size_t num_full_chunks = (tokens.size() - 1) / chunk_size; - for (size_t i = 0; i < num_full_chunks; ++i) { - size_t start = i * chunk_size; - std::vector chunk(tokens.begin() + start, tokens.begin() + start + chunk_size); - if (i == 1) - gb->set_prefill_mode(true); - process_chunk(chunk); - } - - gb->set_prefill_mode(false); - size_t final_start = num_full_chunks * chunk_size; - process_chunk(std::vector(tokens.begin() + final_start, tokens.end())); -} - -} -} diff --git a/cactus-engine/models/gemma4/model_gemma4.h b/cactus-engine/models/gemma4/model_gemma4.h deleted file mode 100644 index de4c6c60f..000000000 --- a/cactus-engine/models/gemma4/model_gemma4.h +++ /dev/null @@ -1,415 +0,0 @@ -#pragma once - -#include "../model.h" - -bool test_gemma4_vision(bool expect_npu); -bool test_gemma4_audio(bool expect_npu); - -namespace cactus { -namespace engine { - -class Gemma4Model : public Model { - friend class Gemma4MmModel; -public: - Gemma4Model(); - explicit Gemma4Model(const Config& config); - ~Gemma4Model() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_moe(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; - void load_weights_to_graph(CactusGraph* gb) override; - void post_init() override; - std::vector get_kv_layer_dims() const override; - std::vector get_kv_layer_heads() const override; - std::vector get_kv_layer_windows() const override; - void compact_kv_cache() override; - - size_t forward_from_embeddings(CactusGraph* gb, size_t hidden, const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend, bool use_cache); - size_t forward_from_embeddings(CactusGraph* gb, size_t hidden, size_t pli_hidden_source, - const std::vector& pli_tokens, size_t seq_len, - ComputeBackend backend, bool use_cache); - size_t build_pli_combined_from_tokens(CactusGraph* gb, size_t hidden, - const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend); - -private: - size_t forward_split(const std::vector& tokens, bool use_cache); - - std::pair build_preamble_and_embed(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input); - - void set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens); - - size_t build_pli_combined(CactusGraph* gb, size_t hidden, size_t pli_embed, - size_t seq_len, ComputeBackend backend); - - size_t build_per_layer_input(CactusGraph* gb, size_t hidden, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend) const; - - bool is_global_layer(uint32_t idx) const; - size_t apply_partial_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, - float rope_freq, size_t position_offset); - size_t apply_transformer_layer(CactusGraph* gb, size_t hidden, size_t pli, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t pos_offset); - - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - size_t embed_tokens_per_layer; - size_t per_layer_model_proj; - size_t per_layer_proj_norm; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - size_t per_layer_gate; - size_t per_layer_proj; - size_t post_per_layer_norm; - size_t layer_scalar; - - std::vector moe_w1_experts; - std::vector moe_w3_experts; - std::vector moe_w2_experts; - size_t moe_per_expert_scale = 0; - size_t router_proj = 0; - size_t router_scale = 0; - size_t post_ffn_norm_1 = 0; - size_t pre_ffn_norm_2 = 0; - size_t post_ffn_norm_2 = 0; - }; - - std::vector layers; - } weight_nodes_; - - uint32_t first_shared_layer_ = 0; - std::vector kv_share_map_; - std::vector shared_k_nodes_; - std::vector shared_v_nodes_; - - std::vector<__fp16> v_norm_ones_weight_; - size_t v_norm_ones_node_ = 0; - size_t v_norm_ones_global_node_ = 0; -}; - -class Gemma4VisionModel : public Model { - friend class Gemma4MmModel; -public: - struct PreprocessedImage { - std::vector pixel_values; - size_t height; - size_t width; - size_t patch_height; - size_t patch_width; - size_t num_patches; - }; - - Gemma4VisionModel(); - explicit Gemma4VisionModel(const Config& config); - ~Gemma4VisionModel() override = default; - - PreprocessedImage preprocess_image(const std::string& image_path); - size_t forward_vision(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend); - size_t build_vision_projector(CactusGraph* gb, size_t vision_features, ComputeBackend backend); - -protected: - size_t forward(const std::vector&, bool) override { - throw std::runtime_error("Gemma4VisionModel: use forward_vision() instead"); - } - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4VisionModel: build_attention unused"); - } - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Gemma4VisionModel: build_mlp unused"); - } - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4VisionModel: build_transformer_block unused"); - } - void load_weights_to_graph(CactusGraph* gb) override; - -private: - size_t build_vision_patch_embedding(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend); - size_t build_vision_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend); - size_t build_vision_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend); - size_t build_vision_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend); - size_t build_vision_pooler(CactusGraph* gb, size_t hidden, const PreprocessedImage& img, ComputeBackend backend); - std::pair build_2d_rope_nodes(CactusGraph* gb, const PreprocessedImage& img, size_t max_patches); - size_t build_padding_mask(CactusGraph* gb, size_t num_real, size_t max_patches); - - struct VisionWeightNodes { - size_t patch_input_proj = 0; - size_t position_table = 0; - - struct LayerWeights { - size_t attn_q_weight = 0; - size_t attn_k_weight = 0; - size_t attn_v_weight = 0; - size_t attn_output_weight = 0; - size_t attn_q_norm = 0; - size_t attn_k_norm = 0; - size_t input_layernorm = 0; - size_t post_attention_layernorm = 0; - size_t pre_feedforward_layernorm = 0; - size_t post_feedforward_layernorm = 0; - size_t mlp_gate_proj = 0; - size_t mlp_up_proj = 0; - size_t mlp_down_proj = 0; - size_t layer_scalar = 0; - }; - - std::vector layers; - size_t embed_vision_proj = 0; - size_t post_proj_norm = 0; - } vision_weights_; - - std::vector<__fp16> vision_v_norm_ones_; - size_t vision_v_norm_ones_node_ = 0; - std::vector<__fp16> post_proj_norm_ones_; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool disable_npu_ = false; - - friend bool ::test_gemma4_vision(bool); -}; - -class Gemma4AudioModel : public Model { - friend class Gemma4MmModel; -public: - Gemma4AudioModel(); - explicit Gemma4AudioModel(const Config& config); - ~Gemma4AudioModel() override = default; - - struct ConformerContext { - size_t timing_fp16 = 0; - size_t front_pad = 0; - size_t back_pad = 0; - size_t seq_len = 0; - }; - - size_t forward_audio(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend); - - size_t build_audio_projector(CactusGraph* gb, size_t audio_features, ComputeBackend backend); - - size_t build_sscp(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend); - - ConformerContext build_conformer_context(CactusGraph* gb, size_t sscp_output); - - size_t build_conformer_ffw(CactusGraph* gb, size_t input, uint32_t layer_idx, - bool is_end, ComputeBackend backend); - size_t build_conformer_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend); - size_t build_conformer_lconv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend); - size_t build_conformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend); - - struct AudioWeightNodes { - size_t sscp_conv0_weight = 0; - size_t sscp_conv0_norm = 0; - size_t sscp_conv1_weight = 0; - size_t sscp_conv1_norm = 0; - size_t sscp_input_proj = 0; - - struct ClipBounds { - float in_min = -1e10f, in_max = 1e10f; - float out_min = -1e10f, out_max = 1e10f; - }; - - struct ConformerLayerWeights { - size_t ffw_start_1 = 0, ffw_start_2 = 0; - ClipBounds ffw_start_1_clip, ffw_start_2_clip; - size_t ffw_start_pre_norm = 0, ffw_start_post_norm = 0; - size_t attn_q = 0, attn_k = 0, attn_v = 0; - ClipBounds attn_q_clip, attn_k_clip, attn_v_clip; - size_t attn_per_dim_scale = 0; - size_t attn_rel_pos_proj = 0; - size_t attn_post = 0; - ClipBounds attn_post_clip; - size_t attn_pre_norm = 0, attn_post_norm = 0; - size_t lconv_start = 0, lconv_depthwise = 0, lconv_end = 0; - ClipBounds lconv_start_clip, lconv_end_clip; - size_t lconv_pre_norm = 0, lconv_conv_norm = 0; - size_t ffw_end_1 = 0, ffw_end_2 = 0; - ClipBounds ffw_end_1_clip, ffw_end_2_clip; - size_t ffw_end_pre_norm = 0, ffw_end_post_norm = 0; - size_t block_norm = 0; - }; - std::vector layers; - - size_t output_proj = 0; - size_t output_proj_bias = 0; - - size_t embed_audio_proj = 0; - } audio_weights_; - - std::vector<__fp16> audio_proj_norm_ones_; - size_t audio_proj_norm_ones_node_ = 0; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool disable_npu_ = false; - std::vector<__fp16> npu_audio_input_scratch_; - std::vector<__fp16> npu_audio_output_scratch_; - std::vector<__fp16> npu_audio_reorder_scratch_; - - friend bool ::test_gemma4_audio(bool); - -protected: - size_t forward(const std::vector&, bool) override { - throw std::runtime_error("Gemma4AudioModel: use forward_audio() instead"); - } - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4AudioModel: build_attention unused"); - } - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Gemma4AudioModel: build_mlp unused"); - } - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4AudioModel: build_transformer_block unused"); - } - void load_weights_to_graph(CactusGraph* gb) override; -}; - -class Gemma4MmModel : public Model { -public: - Gemma4MmModel(); - explicit Gemma4MmModel(const Config& config); - ~Gemma4MmModel() override = default; - - bool init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - - uint32_t decode(const std::vector& tokens, - float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f) override; - - void prefill(const std::vector& tokens, size_t chunk_size = 256, - const std::string& profile_file = "") override; - - void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file = "") override; - - uint32_t decode_with_images( - const std::vector& tokens, - const std::vector& image_paths, - float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f) override; - - uint32_t decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - - uint32_t decode_with_media( - const std::vector& tokens, - const std::vector& image_paths, - const std::vector& audio_features, - float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f); - - void reset_cache() override; - std::vector get_image_embeddings(const std::string& image_path) override; - std::vector get_audio_embeddings(const std::vector& audio_features) override; - void compact_kv_cache() override; - void remove_thinking_tokens(const std::vector>& ranges) override; - - void set_tool_constraints(const std::vector& tools) override; - void clear_tool_constraints() override; - void update_tool_constraints(uint32_t token_id) override; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override; - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct ForwardResult { - size_t final_hidden_node; - size_t seq_len; - }; - - ForwardResult forward_multimodal(CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend, bool use_cache); - - uint32_t decode_multimodal(const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty); - -public: - struct MultimodalInputs { - size_t hidden_node = 0; - size_t pli_hidden_source_node = 0; - std::vector pli_tokens; - size_t seq_len = 0; - }; - - const Gemma4VisionModel& vision_encoder() const { return vision_encoder_; } - Gemma4VisionModel& vision_encoder() { return vision_encoder_; } - const Gemma4AudioModel& audio_encoder() const { return audio_encoder_; } - Gemma4AudioModel& audio_encoder() { return audio_encoder_; } - const Gemma4Model& language_model() const { return language_model_; } - Gemma4Model& language_model() { return language_model_; } - MultimodalInputs build_multimodal_inputs( - CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend); - -private: - Gemma4VisionModel vision_encoder_; - Gemma4AudioModel audio_encoder_; - Gemma4Model language_model_; -}; - -} -} diff --git a/cactus-engine/models/gemma4/model_gemma4_audio.cpp b/cactus-engine/models/gemma4/model_gemma4_audio.cpp deleted file mode 100644 index 2c1b347f1..000000000 --- a/cactus-engine/models/gemma4/model_gemma4_audio.cpp +++ /dev/null @@ -1,768 +0,0 @@ -#include "model_gemma4.h" -#include "cactus_graph.h" -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -static const float INV_LN2 = 1.0f / std::log(2.0f); -static const float GEMMA4_AUDIO_K_SCALE = std::log(1.0f + std::exp(1.0f)) / std::log(2.0f); -static const char* GEMMA4_AUDIO_NPU_INPUT_NAME = "x"; - -static size_t shape_elements(const std::vector& shape) { - if (shape.empty()) return 0; - size_t total = 1; - for (int d : shape) { - if (d <= 0) return 0; - total *= static_cast(d); - } - return total; -} - -static size_t infer_npu_audio_max_frames(const std::vector& shape, size_t mel_bins) { - if (shape.size() == 2) { - size_t s0 = static_cast(shape[0]); - size_t s1 = static_cast(shape[1]); - if (s1 == mel_bins) return s0; - if (s0 == mel_bins) return s1; - } else if (shape.size() == 3) { - size_t s0 = static_cast(shape[0]); - size_t s1 = static_cast(shape[1]); - size_t s2 = static_cast(shape[2]); - if (s0 == 1 && s2 == mel_bins) return s1; // [1, T, F] - if (s0 == 1 && s1 == mel_bins) return s2; // [1, F, T] - if (s1 == mel_bins && s2 == 1) return s0; // [T, F, 1] - if (s0 == mel_bins && s2 == 1) return s1; // [F, T, 1] - } else if (shape.size() == 4) { - size_t s0 = static_cast(shape[0]); - size_t s1 = static_cast(shape[1]); - size_t s2 = static_cast(shape[2]); - size_t s3 = static_cast(shape[3]); - if (s0 == 1 && s1 == 1 && s3 == mel_bins) return s2; // [1, 1, T, F] - if (s0 == 1 && s1 == 1 && s2 == mel_bins) return s3; // [1, 1, F, T] - if (s0 == 1 && s1 == mel_bins && s3 == 1) return s2; // [1, F, T, 1] - if (s0 == 1 && s2 == mel_bins && s3 == 1) return s1; // [1, T, F, 1] - } - return 0; -} - -static bool pack_gemma4_audio_for_npu(const std::vector& mel_features, - size_t frames, - size_t mel_bins, - const std::vector& input_shape, - std::vector<__fp16>& packed) { - if (input_shape.empty()) return false; - const size_t total = shape_elements(input_shape); - if (total == 0) return false; - packed.resize(total); - std::fill(packed.begin(), packed.end(), static_cast<__fp16>(0.0f)); - - auto tm = [&](size_t t, size_t m) -> __fp16 { - return static_cast<__fp16>(mel_features[t * mel_bins + m]); - }; - - if (input_shape.size() == 4) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - const size_t s3 = static_cast(input_shape[3]); - if (s0 != 1) return false; - - if (s1 == 1 && s2 >= frames && s3 == mel_bins) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(t * s3) + m] = tm(t, m); - return true; - } - if (s1 == 1 && s2 == mel_bins && s3 >= frames) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(m * s3) + t] = tm(t, m); - return true; - } - if (s1 >= frames && s2 == mel_bins && s3 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[((t * s2 + m) * s3)] = tm(t, m); - return true; - } - if (s1 == mel_bins && s2 >= frames && s3 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[((m * s2 + t) * s3)] = tm(t, m); - return true; - } - return false; - } - - if (input_shape.size() == 3) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - - if (s0 == 1 && s1 >= frames && s2 == mel_bins) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[t * s2 + m] = tm(t, m); - return true; - } - if (s0 == 1 && s1 == mel_bins && s2 >= frames) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[m * s2 + t] = tm(t, m); - return true; - } - if (s0 >= frames && s1 == mel_bins && s2 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(t * s1 + m) * s2] = tm(t, m); - return true; - } - if (s0 == mel_bins && s1 >= frames && s2 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(m * s1 + t) * s2] = tm(t, m); - return true; - } - return false; - } - - if (input_shape.size() == 2) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - - if (s0 >= frames && s1 == mel_bins) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[t * s1 + m] = tm(t, m); - return true; - } - if (s0 == mel_bins && s1 >= frames) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[m * s1 + t] = tm(t, m); - return true; - } - return false; - } - - return false; -} - -struct NPUAudioOutputLayout { - std::vector dims; - size_t hidden_axis = SIZE_MAX; - size_t hidden_dim = 0; - size_t time_steps = 0; -}; - -static bool infer_npu_audio_output_layout(const std::vector& output_shape, - size_t elements_written, - size_t expected_hidden_dim, - NPUAudioOutputLayout& layout) { - if (elements_written == 0) return false; - - layout = NPUAudioOutputLayout{}; - for (int d : output_shape) { - if (d > 0) layout.dims.push_back(static_cast(d)); - } - - if (layout.dims.empty()) { - if (expected_hidden_dim == 0 || (elements_written % expected_hidden_dim) != 0) { - return false; - } - layout.dims = {elements_written / expected_hidden_dim, expected_hidden_dim}; - } - - const size_t rank = layout.dims.size(); - size_t hidden_axis = SIZE_MAX; - if (expected_hidden_dim > 0) { - if (layout.dims[rank - 1] == expected_hidden_dim) { - hidden_axis = rank - 1; - } else if (rank >= 2 && layout.dims[rank - 2] == expected_hidden_dim) { - hidden_axis = rank - 2; - } else { - for (size_t i = 0; i < rank; ++i) { - if (layout.dims[i] == expected_hidden_dim) { - hidden_axis = i; - break; - } - } - } - } - - if (hidden_axis == SIZE_MAX) { - hidden_axis = rank - 1; - } - - const size_t hidden_dim = layout.dims[hidden_axis]; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) { - return false; - } - - layout.hidden_axis = hidden_axis; - layout.hidden_dim = hidden_dim; - layout.time_steps = elements_written / hidden_dim; - return layout.time_steps > 0; -} - -static bool materialize_npu_audio_time_major(const __fp16* src, - const NPUAudioOutputLayout& layout, - size_t time_steps, - std::vector<__fp16>& dst) { - if (!src || layout.dims.empty() || layout.hidden_dim == 0 || layout.time_steps == 0) { - return false; - } - if (layout.hidden_axis >= layout.dims.size() || time_steps == 0 || time_steps > layout.time_steps) { - return false; - } - - const size_t rank = layout.dims.size(); - if (layout.hidden_axis == rank - 1) { - // Already [time, hidden] in row-major flattening. - return false; - } - - std::vector strides(rank, 1); - for (size_t i = rank; i-- > 1;) { - strides[i - 1] = strides[i] * layout.dims[i]; - } - - std::vector non_hidden_axes; - non_hidden_axes.reserve(rank - 1); - for (size_t i = 0; i < rank; ++i) { - if (i != layout.hidden_axis) non_hidden_axes.push_back(i); - } - - dst.assign(time_steps * layout.hidden_dim, static_cast<__fp16>(0.0f)); - - for (size_t t = 0; t < time_steps; ++t) { - size_t rem = t; - size_t base_offset = 0; - for (size_t i = non_hidden_axes.size(); i-- > 0;) { - size_t axis = non_hidden_axes[i]; - size_t dim = layout.dims[axis]; - size_t coord = rem % dim; - rem /= dim; - base_offset += coord * strides[axis]; - } - - for (size_t h = 0; h < layout.hidden_dim; ++h) { - size_t src_index = base_offset + h * strides[layout.hidden_axis]; - dst[t * layout.hidden_dim + h] = src[src_index]; - } - } - - return true; -} - -static size_t clipped_matmul(CactusGraph* gb, size_t input, size_t weight, - float in_min, float in_max, float out_min, float out_max, - ComputeBackend backend) { - size_t clamped_in = (in_min < in_max && !std::isinf(in_min) && !std::isinf(in_max)) - ? gb->clamp(input, in_min, in_max) : input; - size_t result = gb->matmul(clamped_in, weight, true, backend); - return (out_min < out_max && !std::isinf(out_min) && !std::isinf(out_max)) - ? gb->clamp(result, out_min, out_max) : result; -} - -static float read_scalar_weight(CactusGraph* gb, const std::string& path) { - size_t node = gb->mmap_weights(path); - const auto& buf = gb->get_output_buffer(node); - if (buf.precision == Precision::FP16) - return static_cast(*buf.data_as<__fp16>()); - if (buf.precision == Precision::FP32) - return *buf.data_as(); - return 0.0f; -} - -static Gemma4AudioModel::AudioWeightNodes::ClipBounds load_clip_bounds(CactusGraph* gb, const std::string& prefix) { - Gemma4AudioModel::AudioWeightNodes::ClipBounds cb{0, 0, 0, 0}; - std::string in_min_path = prefix + "_input_min.weights"; - if (!std::filesystem::exists(in_min_path)) - return cb; - cb.in_min = read_scalar_weight(gb, in_min_path); - cb.in_max = read_scalar_weight(gb, prefix + "_input_max.weights"); - cb.out_min = read_scalar_weight(gb, prefix + "_output_min.weights"); - cb.out_max = read_scalar_weight(gb, prefix + "_output_max.weights"); - return cb; -} - -static void get_conv2d_output_hw(const CactusGraph* gb, size_t node, size_t& h, size_t& w) { - const auto& buf = gb->get_output_buffer(node); - h = buf.shape[2]; - w = buf.shape[3]; -} - -Gemma4AudioModel::Gemma4AudioModel() : Model() {} -Gemma4AudioModel::Gemma4AudioModel(const Config& config) : Model(config) {} - -void Gemma4AudioModel::load_weights_to_graph(CactusGraph* gb) { - auto resolve = [&](const std::string& name) -> std::string { - return model_folder_path_ + "/" + name; - }; - auto resolve_existing = [&](const std::string& preferred, - const std::string& legacy = std::string()) -> std::string { - std::string preferred_path = resolve(preferred); - if (std::filesystem::exists(preferred_path)) { - return preferred_path; - } - if (!legacy.empty()) { - std::string legacy_path = resolve(legacy); - if (std::filesystem::exists(legacy_path)) { - return legacy_path; - } - } - return preferred_path; - }; - auto load_weight = [&](const std::string& preferred, - const std::string& legacy = std::string()) -> size_t { - return gb->mmap_weights(resolve_existing(preferred, legacy)); - }; - auto load_clip_from_weight = [&](const std::string& preferred, - const std::string& legacy = std::string()) - -> Gemma4AudioModel::AudioWeightNodes::ClipBounds { - std::string weight_path = resolve_existing(preferred, legacy); - static const std::string suffix = ".weights"; - if (weight_path.size() > suffix.size() && - weight_path.compare(weight_path.size() - suffix.size(), suffix.size(), suffix) == 0) { - return load_clip_bounds(gb, weight_path.substr(0, weight_path.size() - suffix.size())); - } - return Gemma4AudioModel::AudioWeightNodes::ClipBounds{0, 0, 0, 0}; - }; - - if (!disable_npu_ && npu::is_npu_available()) { - std::string npu_path = model_folder_path_ + "/audio_encoder.mlpackage"; - if (std::filesystem::exists(npu_path)) { - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - npu_encoder_->preallocate(input_shape, GEMMA4_AUDIO_NPU_INPUT_NAME, ""); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - CACTUS_LOG_DEBUG("npu", "[gemma4-audio] found audio_encoder.mlpackage but failed to enable NPU audio encoder; using CPU"); - } - } - } - - // Keep CPU audio weights available even when the NPU encoder is enabled. - // The runtime may still fall back to the graph path for unsupported inputs. - { - audio_weights_.sscp_conv0_weight = load_weight( - "audio_subsample_conv_projection_conv_0_conv.weights", - "audio_subsample_conv_projection_layer0_conv.weights"); - audio_weights_.sscp_conv0_norm = load_weight( - "audio_subsample_conv_projection_conv_0_norm.weights", - "audio_subsample_conv_projection_layer0_norm.weights"); - audio_weights_.sscp_conv1_weight = load_weight( - "audio_subsample_conv_projection_conv_1_conv.weights", - "audio_subsample_conv_projection_layer1_conv.weights"); - audio_weights_.sscp_conv1_norm = load_weight( - "audio_subsample_conv_projection_conv_1_norm.weights", - "audio_subsample_conv_projection_layer1_norm.weights"); - audio_weights_.sscp_input_proj = load_weight( - "audio_subsample_conv_projection_input_proj.weights", - "audio_subsample_conv_projection_input_proj_linear.weights"); - - audio_weights_.layers.resize(config_.audio_num_layers); - for (uint32_t i = 0; i < config_.audio_num_layers; i++) { - auto& layer = audio_weights_.layers[i]; - - auto layer_weight = [&](const std::string& preferred_suffix, - const std::string& legacy_suffix = std::string()) -> size_t { - std::string legacy = legacy_suffix.empty() ? preferred_suffix : legacy_suffix; - return load_weight( - "audio_conformer_" + std::to_string(i) + "_" + preferred_suffix, - "audio_layers_" + std::to_string(i) + "_" + legacy); - }; - auto layer_clip = [&](const std::string& preferred_suffix, - const std::string& legacy_suffix = std::string()) - -> Gemma4AudioModel::AudioWeightNodes::ClipBounds { - std::string legacy = legacy_suffix.empty() ? preferred_suffix : legacy_suffix; - return load_clip_from_weight( - "audio_conformer_" + std::to_string(i) + "_" + preferred_suffix, - "audio_layers_" + std::to_string(i) + "_" + legacy); - }; - - layer.ffw_start_1 = layer_weight("ffw_layer_start_ffw_layer_1.weights", "feed_forward1_ffw_layer_1.weights"); - layer.ffw_start_2 = layer_weight("ffw_layer_start_ffw_layer_2.weights", "feed_forward1_ffw_layer_2.weights"); - layer.ffw_start_1_clip = layer_clip("ffw_layer_start_ffw_layer_1.weights", "feed_forward1_ffw_layer_1.weights"); - layer.ffw_start_2_clip = layer_clip("ffw_layer_start_ffw_layer_2.weights", "feed_forward1_ffw_layer_2.weights"); - layer.ffw_start_pre_norm = layer_weight("ffw_layer_start_pre_layer_norm.weights", "feed_forward1_pre_layer_norm.weights"); - layer.ffw_start_post_norm = layer_weight("ffw_layer_start_post_layer_norm.weights", "feed_forward1_post_layer_norm.weights"); - - layer.attn_q = layer_weight("attention_attn_q_proj.weights", "self_attn_q_proj.weights"); - layer.attn_k = layer_weight("attention_attn_k_proj.weights", "self_attn_k_proj.weights"); - layer.attn_v = layer_weight("attention_attn_v_proj.weights", "self_attn_v_proj.weights"); - layer.attn_q_clip = layer_clip("attention_attn_q_proj.weights", "self_attn_q_proj.weights"); - layer.attn_k_clip = layer_clip("attention_attn_k_proj.weights", "self_attn_k_proj.weights"); - layer.attn_v_clip = layer_clip("attention_attn_v_proj.weights", "self_attn_v_proj.weights"); - layer.attn_per_dim_scale = layer_weight("attention_attn_per_dim_scale.weights", "self_attn_per_dim_scale.weights"); - layer.attn_rel_pos_proj = layer_weight( - "attention_attn_relative_position_embedding_pos_proj.weights", - "self_attn_relative_k_proj.weights"); - layer.attn_post = layer_weight("attention_post.weights", "self_attn_post.weights"); - layer.attn_post_clip = layer_clip("attention_post.weights", "self_attn_post.weights"); - layer.attn_pre_norm = layer_weight("attention_pre_attn_norm.weights", "norm_pre_attn.weights"); - layer.attn_post_norm = layer_weight("attention_post_norm.weights", "norm_post_attn.weights"); - - layer.lconv_start = layer_weight("lconv1d_linear_start.weights"); - layer.lconv_depthwise = layer_weight("lconv1d_depthwise_conv1d.weights"); - layer.lconv_end = layer_weight("lconv1d_linear_end.weights"); - layer.lconv_start_clip = layer_clip("lconv1d_linear_start.weights"); - layer.lconv_end_clip = layer_clip("lconv1d_linear_end.weights"); - layer.lconv_pre_norm = layer_weight("lconv1d_pre_layer_norm.weights"); - layer.lconv_conv_norm = layer_weight("lconv1d_conv_norm.weights"); - - layer.ffw_end_1 = layer_weight("ffw_layer_end_ffw_layer_1.weights", "feed_forward2_ffw_layer_1.weights"); - layer.ffw_end_2 = layer_weight("ffw_layer_end_ffw_layer_2.weights", "feed_forward2_ffw_layer_2.weights"); - layer.ffw_end_1_clip = layer_clip("ffw_layer_end_ffw_layer_1.weights", "feed_forward2_ffw_layer_1.weights"); - layer.ffw_end_2_clip = layer_clip("ffw_layer_end_ffw_layer_2.weights", "feed_forward2_ffw_layer_2.weights"); - layer.ffw_end_pre_norm = layer_weight("ffw_layer_end_pre_layer_norm.weights", "feed_forward2_pre_layer_norm.weights"); - layer.ffw_end_post_norm = layer_weight("ffw_layer_end_post_layer_norm.weights", "feed_forward2_post_layer_norm.weights"); - - layer.block_norm = layer_weight("norm.weights", "norm_out.weights"); - } - - audio_weights_.output_proj = load_weight("audio_output_proj.weights"); - audio_weights_.output_proj_bias = load_weight("audio_output_proj.bias"); - } - audio_weights_.embed_audio_proj = load_weight("embed_audio_proj.weights"); - - size_t proj_out_dim = config_.audio_output_proj_dims > 0 - ? static_cast(config_.audio_output_proj_dims) - : static_cast(config_.audio_hidden_dim); - audio_proj_norm_ones_.assign(proj_out_dim, static_cast<__fp16>(1.0f)); - audio_proj_norm_ones_node_ = gb->input({proj_out_dim}, Precision::FP16); - gb->set_external_input(audio_proj_norm_ones_node_, audio_proj_norm_ones_.data(), Precision::FP16); - - output_weight_node_id_ = 0; -} - -size_t Gemma4AudioModel::build_sscp(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend) { - size_t mel_bins = config_.audio_input_feat_size; - size_t conv0_ch = config_.audio_sscp_conv0_channels; - size_t conv1_ch = config_.audio_sscp_conv1_channels; - - size_t mel_input = gb->input({num_frames, mel_bins}, Precision::FP32); - gb->set_input(mel_input, mel_features.data(), Precision::FP32); - size_t mel_fp16 = gb->precision_cast(mel_input, Precision::FP16); - - size_t x = gb->reshape(mel_fp16, {1, 1, num_frames, mel_bins}); - - x = gb->conv2d_k3s2p1(x, audio_weights_.sscp_conv0_weight); - size_t h1, w1; - get_conv2d_output_hw(gb, x, h1, w1); - - x = gb->reshape(x, {conv0_ch, h1, w1}); - x = gb->transposeN(x, {1, 2, 0}); - x = gb->reshape(x, {h1 * w1, conv0_ch}); - x = gb->layernorm(x, audio_weights_.sscp_conv0_norm, config_.audio_sscp_conv_eps); - x = gb->reshape(x, {h1, w1, conv0_ch}); - x = gb->transposeN(x, {2, 0, 1}); - x = gb->relu(x); - x = gb->reshape(x, {1, conv0_ch, h1, w1}); - - x = gb->conv2d_k3s2p1(x, audio_weights_.sscp_conv1_weight); - size_t h2, w2; - get_conv2d_output_hw(gb, x, h2, w2); - - x = gb->reshape(x, {conv1_ch, h2, w2}); - x = gb->transposeN(x, {1, 2, 0}); - x = gb->reshape(x, {h2 * w2, conv1_ch}); - x = gb->layernorm(x, audio_weights_.sscp_conv1_norm, config_.audio_sscp_conv_eps); - x = gb->reshape(x, {h2, w2, conv1_ch}); - x = gb->transposeN(x, {2, 0, 1}); - x = gb->relu(x); - - x = gb->transposeN(x, {1, 2, 0}); - x = gb->reshape(x, {h2, w2 * conv1_ch}); - - return gb->matmul(x, audio_weights_.sscp_input_proj, true, backend); -} - -static std::vector compute_timing_signal(size_t num_positions, size_t hidden_dim, size_t max_past) { - size_t num_timescales = hidden_dim / 2; - float log_ts_inc = std::log(1.0e4f) / std::max(static_cast(num_timescales) - 1, 1); - - std::vector signal(num_positions * hidden_dim); - for (size_t p = 0; p < num_positions; p++) { - float pos = static_cast(max_past) - static_cast(p); - for (size_t i = 0; i < num_timescales; i++) { - float scaled = pos * std::exp(-static_cast(i) * log_ts_inc); - signal[p * hidden_dim + i] = std::sin(scaled); - signal[p * hidden_dim + num_timescales + i] = std::cos(scaled); - } - } - return signal; -} - -static size_t create_zero_pad_node(CactusGraph* gb, size_t time_len, size_t num_heads, size_t head_dim) { - if (time_len == 0) return 0; - std::vector<__fp16> zeros(time_len * num_heads * head_dim, static_cast<__fp16>(0.0f)); - size_t node = gb->input({time_len, num_heads, head_dim}, Precision::FP16); - gb->set_input(node, zeros.data(), Precision::FP16); - return node; -} - -Gemma4AudioModel::ConformerContext Gemma4AudioModel::build_conformer_context( - CactusGraph* gb, size_t sscp_output) { - - size_t num_heads = config_.audio_num_heads; - size_t head_dim = config_.audio_head_dim; - size_t hidden_dim = num_heads * head_dim; - size_t max_past = config_.audio_context_left > 0 ? config_.audio_context_left - 1 : 0; - size_t num_positions = max_past + config_.audio_context_right + 1; - size_t seq_len = gb->get_output_buffer(sscp_output).shape[0]; - - auto timing_signal = compute_timing_signal(num_positions, hidden_dim, max_past); - size_t timing_input = gb->input({num_positions, hidden_dim}, Precision::FP32); - gb->set_input(timing_input, timing_signal.data(), Precision::FP32); - size_t timing_fp16 = gb->precision_cast(timing_input, Precision::FP16); - - size_t front_pad_len = (seq_len >= num_positions) ? seq_len - num_positions : 0; - size_t back_pad_len = seq_len > 0 ? seq_len - 1 : 0; - - return { - timing_fp16, - create_zero_pad_node(gb, front_pad_len, num_heads, head_dim), - create_zero_pad_node(gb, back_pad_len, num_heads, head_dim), - seq_len - }; -} - -size_t Gemma4AudioModel::build_conformer_ffw(CactusGraph* gb, size_t input, uint32_t layer_idx, - bool is_end, ComputeBackend backend) { - const auto& layer = audio_weights_.layers[layer_idx]; - float residual_weight = config_.audio_residual_weight; - float eps = config_.audio_rms_norm_eps; - - size_t residual = input; - - size_t pre_norm = is_end ? layer.ffw_end_pre_norm : layer.ffw_start_pre_norm; - size_t post_norm = is_end ? layer.ffw_end_post_norm : layer.ffw_start_post_norm; - size_t w1 = is_end ? layer.ffw_end_1 : layer.ffw_start_1; - size_t w2 = is_end ? layer.ffw_end_2 : layer.ffw_start_2; - - auto& c1 = is_end ? layer.ffw_end_1_clip : layer.ffw_start_1_clip; - auto& c2 = is_end ? layer.ffw_end_2_clip : layer.ffw_start_2_clip; - - size_t x = gb->rms_norm(input, pre_norm, eps); - - x = clipped_matmul(gb, x, w1, c1.in_min, c1.in_max, c1.out_min, c1.out_max, backend); - x = gb->silu(x); - x = clipped_matmul(gb, x, w2, c2.in_min, c2.in_max, c2.out_min, c2.out_max, backend); - - x = gb->rms_norm(x, post_norm, eps); - - x = gb->scalar_multiply(x, residual_weight); - return gb->add(residual, x); -} - -size_t Gemma4AudioModel::build_conformer_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend) { - const auto& layer = audio_weights_.layers[layer_idx]; - float eps = config_.audio_rms_norm_eps; - size_t num_heads = config_.audio_num_heads; - size_t head_dim = config_.audio_head_dim; - size_t hidden_dim = num_heads * head_dim; - float logit_cap = config_.audio_logit_cap; - size_t max_past = config_.audio_context_left > 0 ? config_.audio_context_left - 1 : 0; - size_t num_positions = max_past + config_.audio_context_right + 1; - size_t seq_len = ctx.seq_len; - - size_t residual = input; - - size_t x = gb->rms_norm(input, layer.attn_pre_norm, eps); - - size_t q = clipped_matmul(gb, x, layer.attn_q, layer.attn_q_clip.in_min, layer.attn_q_clip.in_max, layer.attn_q_clip.out_min, layer.attn_q_clip.out_max, backend); - size_t k = clipped_matmul(gb, x, layer.attn_k, layer.attn_k_clip.in_min, layer.attn_k_clip.in_max, layer.attn_k_clip.out_min, layer.attn_k_clip.out_max, backend); - size_t v = clipped_matmul(gb, x, layer.attn_v, layer.attn_v_clip.in_min, layer.attn_v_clip.in_max, layer.attn_v_clip.out_min, layer.attn_v_clip.out_max, backend); - - float q_scale = (1.0f / std::sqrt(static_cast(head_dim))) * INV_LN2; - size_t q_dim_scale_fp16 = gb->precision_cast(layer.attn_per_dim_scale, Precision::FP16); - size_t q_dim_scale = gb->scalar_exp(q_dim_scale_fp16); - q_dim_scale = gb->scalar_add(q_dim_scale, 1.0f); - q_dim_scale = gb->scalar_log(q_dim_scale); - q_dim_scale = gb->scalar_multiply(q_dim_scale, q_scale); - - size_t q_flat = gb->reshape(q, {seq_len * num_heads, head_dim}); - q_flat = gb->multiply(q_flat, q_dim_scale); - q = gb->reshape(q_flat, {seq_len, hidden_dim}); - - k = gb->scalar_multiply(k, GEMMA4_AUDIO_K_SCALE); - - size_t sin_emb = gb->matmul(ctx.timing_fp16, layer.attn_rel_pos_proj, true, backend); - size_t sin_emb_3d = gb->reshape(sin_emb, {num_positions, num_heads, head_dim}); - size_t R = 2 * seq_len - 1; - - size_t rel_key; - if (seq_len >= num_positions) { - rel_key = (ctx.front_pad != 0) ? gb->concat(ctx.front_pad, sin_emb_3d, 0) : sin_emb_3d; - if (ctx.back_pad != 0) - rel_key = gb->concat(rel_key, ctx.back_pad, 0); - } else { - size_t start_idx = num_positions - seq_len; - size_t sin_emb_sliced = gb->slice(sin_emb_3d, 0, start_idx, seq_len); - rel_key = (ctx.back_pad != 0) ? gb->concat(sin_emb_sliced, ctx.back_pad, 0) : sin_emb_sliced; - } - - size_t rel_key_4d = gb->reshape(rel_key, {1, R, num_heads, head_dim}); - size_t q4 = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - size_t rel_bias = gb->rel_pos_bias(q4, rel_key_4d, 1.0f); - - size_t k4 = gb->reshape(k, {1, seq_len, num_heads, head_dim}); - size_t v4 = gb->reshape(v, {1, seq_len, num_heads, head_dim}); - - size_t attn = gb->attention_masked(q4, k4, v4, rel_bias, 1.0f, - true, backend, true, 0, max_past, logit_cap); - - size_t attn_out = gb->reshape(attn, {seq_len, hidden_dim}); - - size_t out = clipped_matmul(gb, attn_out, layer.attn_post, layer.attn_post_clip.in_min, layer.attn_post_clip.in_max, layer.attn_post_clip.out_min, layer.attn_post_clip.out_max, backend); - out = gb->rms_norm(out, layer.attn_post_norm, eps); - - return gb->add(residual, out); -} - -size_t Gemma4AudioModel::build_conformer_lconv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) { - const auto& layer = audio_weights_.layers[layer_idx]; - float eps = config_.audio_rms_norm_eps; - size_t kernel_size = config_.audio_conf_conv_kernel_size; - size_t hidden_dim = config_.audio_hidden_dim; - - size_t residual = input; - - size_t x = gb->rms_norm(input, layer.lconv_pre_norm, eps); - - x = clipped_matmul(gb, x, layer.lconv_start, layer.lconv_start_clip.in_min, layer.lconv_start_clip.in_max, layer.lconv_start_clip.out_min, layer.lconv_start_clip.out_max, backend); - - x = gb->glu(x, -1); - - const auto& x_buf = gb->get_output_buffer(x); - size_t seq_len = x_buf.shape[0]; - x = gb->reshape(x, {1, seq_len, hidden_dim}); - - x = gb->conv1d_causal(x, layer.lconv_depthwise, kernel_size); - - x = gb->reshape(x, {seq_len, hidden_dim}); - - x = gb->rms_norm(x, layer.lconv_conv_norm, eps); - x = gb->silu(x); - - x = clipped_matmul(gb, x, layer.lconv_end, layer.lconv_end_clip.in_min, layer.lconv_end_clip.in_max, layer.lconv_end_clip.out_min, layer.lconv_end_clip.out_max, backend); - - return gb->add(x, residual); -} - -size_t Gemma4AudioModel::build_conformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend) { - hidden = build_conformer_ffw(gb, hidden, layer_idx, false, backend); - hidden = build_conformer_attention(gb, hidden, layer_idx, ctx, backend); - hidden = build_conformer_lconv1d(gb, hidden, layer_idx, backend); - hidden = build_conformer_ffw(gb, hidden, layer_idx, true, backend); - return gb->rms_norm(hidden, audio_weights_.layers[layer_idx].block_norm, config_.audio_rms_norm_eps); -} - -size_t Gemma4AudioModel::forward_audio(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend) { - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector npu_input_shape = npu_encoder_->get_input_shape(); - size_t mel_bins = static_cast(config_.audio_input_feat_size); - size_t max_npu_frames = infer_npu_audio_max_frames(npu_input_shape, mel_bins); - size_t copy_frames = max_npu_frames > 0 ? std::min(num_frames, max_npu_frames) : num_frames; - size_t input_values = copy_frames * mel_bins; - - if (mel_features.size() < input_values) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] insufficient mel input values; falling back to CPU audio encoder"); - } else if (!pack_gemma4_audio_for_npu(mel_features, copy_frames, mel_bins, npu_input_shape, - npu_audio_input_scratch_)) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] unsupported NPU input shape; falling back to CPU audio encoder"); - } else { - size_t t1 = (copy_frames + 1) / 2; - size_t t2 = (t1 + 1) / 2; - size_t expected_out_dim = config_.audio_output_proj_dims > 0 - ? static_cast(config_.audio_output_proj_dims) - : static_cast(config_.audio_hidden_dim); - - std::vector out_shape = npu_encoder_->get_output_shape(); - size_t out_elements = npu_encoder_->get_output_buffer_size(); - if (out_elements == 0) out_elements = shape_elements(out_shape); - if (out_elements == 0) out_elements = std::max(1, t2 * expected_out_dim); - - if (npu_audio_output_scratch_.size() < out_elements) - npu_audio_output_scratch_.resize(out_elements); - - size_t written = npu_encoder_->encode( - npu_audio_input_scratch_.data(), npu_audio_output_scratch_.data(), npu_input_shape, - GEMMA4_AUDIO_NPU_INPUT_NAME, ""); - - NPUAudioOutputLayout out_layout; - if (written > 0 && - infer_npu_audio_output_layout(out_shape, written, expected_out_dim, out_layout) && - out_layout.hidden_dim == expected_out_dim) { - const __fp16* src = npu_audio_output_scratch_.data(); - __fp16* cached_output = npu_encoder_->get_output_buffer(); - size_t cached_count = npu_encoder_->get_output_buffer_size(); - size_t required = out_layout.time_steps * out_layout.hidden_dim; - if (cached_output != nullptr && cached_count >= required) { - src = cached_output; - } - - size_t actual_time = std::min(t2, out_layout.time_steps); - if (actual_time > 0) { - const __fp16* final_src = src; - if (out_layout.hidden_axis != (out_layout.dims.size() - 1)) { - if (!materialize_npu_audio_time_major(src, out_layout, actual_time, - npu_audio_reorder_scratch_)) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] failed to reorder NPU output layout; falling back to CPU audio encoder"); - } else { - final_src = npu_audio_reorder_scratch_.data(); - } - } - - if (final_src != nullptr) { - size_t hidden = gb->input({actual_time, out_layout.hidden_dim}, Precision::FP16); - gb->set_input(hidden, final_src, Precision::FP16); - return hidden; - } - } - } - - if (written > 0 && out_layout.hidden_dim != expected_out_dim) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] NPU output dim mismatch; falling back to CPU audio encoder"); - } else { - CACTUS_LOG_WARN("npu", "audio encode failed, falling back to CPU"); - } - } - } - - size_t hidden = build_sscp(gb, mel_features, num_frames, backend); - ConformerContext ctx = build_conformer_context(gb, hidden); - for (uint32_t i = 0; i < config_.audio_num_layers; i++) - hidden = build_conformer_block(gb, hidden, i, ctx, backend); - - if (config_.audio_output_proj_dims > 0) { - hidden = gb->matmul(hidden, audio_weights_.output_proj, true, backend); - hidden = gb->add(hidden, audio_weights_.output_proj_bias); - } - - return hidden; -} - -size_t Gemma4AudioModel::build_audio_projector(CactusGraph* gb, size_t audio_features, ComputeBackend backend) { - size_t normed = gb->rms_norm(audio_features, audio_proj_norm_ones_node_, config_.audio_rms_norm_eps); - size_t projected = gb->matmul(normed, audio_weights_.embed_audio_proj, true, backend); - return gb->scalar_multiply(projected, 1.0f / 16.0f); -} - -} -} diff --git a/cactus-engine/models/gemma4/model_gemma4_mm.cpp b/cactus-engine/models/gemma4/model_gemma4_mm.cpp deleted file mode 100644 index d8632ae3a..000000000 --- a/cactus-engine/models/gemma4/model_gemma4_mm.cpp +++ /dev/null @@ -1,473 +0,0 @@ -#include "model_gemma4.h" -#include "cactus_graph.h" -#include -#include -#include - -namespace cactus { -namespace engine { - -Gemma4MmModel::Gemma4MmModel() : Model() { - config_.model_type = Config::ModelType::GEMMA4; -} - -Gemma4MmModel::Gemma4MmModel(const Config& config) - : Model(config), vision_encoder_(config), audio_encoder_(config), language_model_(config) {} - -bool Gemma4MmModel::init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!Model::init(model_folder, context_size, system_prompt, false)) - return false; - - auto* shared_graph = static_cast(graph_handle_); - if (!shared_graph) - throw std::runtime_error("Shared graph was not initialized for Gemma4MmModel"); - - bool has_vision = config_.vision_num_layers > 0 || config_.vision_embed_dim > 0; - bool has_audio = config_.audio_num_layers > 0 || config_.audio_hidden_dim > 0; - - if (has_vision) { - if (!vision_encoder_.init(shared_graph, model_folder, context_size, "", false)) - throw std::runtime_error("Failed to initialize vision encoder"); - } - - if (has_audio) { - if (!audio_encoder_.init(shared_graph, model_folder, context_size, "", false)) - throw std::runtime_error("Failed to initialize audio encoder"); - } - - if (!language_model_.init(shared_graph, model_folder, context_size, system_prompt, false)) - throw std::runtime_error("Failed to initialize language model"); - - output_weight_node_id_ = language_model_.output_weight_node_id_; - - if (do_warmup) { - std::vector warmup_tokens = {2}; - language_model_.forward(warmup_tokens); - auto* gb2 = static_cast(language_model_.graph_handle_); - gb2->execute(); - language_model_.reset_cache(); - } - - return true; -} - -void Gemma4MmModel::reset_cache() { - Model::reset_cache(); - language_model_.reset_cache(); -} - -void Gemma4MmModel::compact_kv_cache() { - language_model_.compact_kv_cache(); -} - -void Gemma4MmModel::remove_thinking_tokens(const std::vector>& ranges) { - language_model_.remove_thinking_tokens(ranges); -} - -void Gemma4MmModel::set_tool_constraints(const std::vector& tools) { - language_model_.set_tool_constraints(tools); -} - -void Gemma4MmModel::clear_tool_constraints() { - language_model_.clear_tool_constraints(); -} - -void Gemma4MmModel::update_tool_constraints(uint32_t token_id) { - language_model_.update_tool_constraints(token_id); -} - -void Gemma4MmModel::load_weights_to_graph(CactusGraph*) { - output_weight_node_id_ = 0; -} - -Gemma4MmModel::ForwardResult Gemma4MmModel::forward_multimodal( - CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend, bool use_cache) { - - auto inputs = build_multimodal_inputs( - gb, tokens, image_paths, audio_features, audio_num_frames, backend); - - size_t final_hidden = language_model_.forward_from_embeddings( - gb, - inputs.hidden_node, - inputs.pli_hidden_source_node, - inputs.pli_tokens, - inputs.seq_len, - backend, - use_cache); - - return ForwardResult{final_hidden, inputs.seq_len}; -} - -Gemma4MmModel::MultimodalInputs Gemma4MmModel::build_multimodal_inputs( - CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend) { - - size_t vision_soft_node = 0; - size_t num_vision_soft_tokens = 0; - size_t audio_soft_node = 0; - size_t num_audio_soft_tokens = 0; - - if (!image_paths.empty()) { - auto preprocessed = vision_encoder_.preprocess_image(image_paths[0]); - size_t vision_output = vision_encoder_.forward_vision(gb, preprocessed, backend); - vision_soft_node = vision_encoder_.build_vision_projector(gb, vision_output, backend); - uint32_t k = config_.vision_pooling_kernel_size; - num_vision_soft_tokens = (preprocessed.patch_width / k) * (preprocessed.patch_height / k); - } - - if (audio_features && !audio_features->empty()) { - size_t audio_output = audio_encoder_.forward_audio(gb, *audio_features, audio_num_frames, backend); - audio_soft_node = audio_encoder_.build_audio_projector(gb, audio_output, backend); - const auto& audio_buf = gb->get_output_buffer(audio_soft_node); - num_audio_soft_tokens = audio_buf.shape[0]; - } - - uint32_t image_token_id = config_.image_token_id; - uint32_t audio_token_id = config_.audio_token_id; - uint32_t pad_token_id = config_.pad_token_id; - - std::vector sequence_nodes; - std::vector current_text; - std::vector pli_tokens; - size_t total_seq_len = 0; - size_t vision_offset = 0; - size_t audio_offset = 0; - - auto flush_text = [&]() { - if (current_text.empty()) return; - size_t seg_len = current_text.size(); - size_t input_node = gb->input({seg_len}, Precision::FP32); - - auto hidden = gb->scalar_multiply( - gb->embedding(language_model_.embedding_node_id_, input_node), - std::sqrt(static_cast(config_.hidden_dim))); - - std::vector input_data(seg_len); - for (size_t i = 0; i < seg_len; i++) - input_data[i] = static_cast(current_text[i]); - gb->set_input(input_node, input_data.data(), Precision::FP32); - - sequence_nodes.push_back(hidden); - for (auto t : current_text) - pli_tokens.push_back(t); - total_seq_len += seg_len; - current_text.clear(); - }; - - auto append_soft_region = [&](size_t soft_node, size_t& soft_offset, size_t total_soft_tokens, - size_t placeholder_count) { - size_t to_insert = std::min(placeholder_count, total_soft_tokens - soft_offset); - if (to_insert > 0) { - sequence_nodes.push_back(gb->slice(soft_node, 0, soft_offset, to_insert)); - for (size_t j = 0; j < to_insert; j++) - pli_tokens.push_back(pad_token_id); - total_seq_len += to_insert; - soft_offset += to_insert; - } - }; - - auto flush_vision_region = [&](size_t placeholder_count) { - append_soft_region(vision_soft_node, vision_offset, num_vision_soft_tokens, placeholder_count); - }; - - auto flush_audio_region = [&](size_t placeholder_count) { - append_soft_region(audio_soft_node, audio_offset, num_audio_soft_tokens, placeholder_count); - }; - - bool in_image_region = false; - bool in_audio_region = false; - size_t region_count = 0; - - for (size_t i = 0; i < tokens.size(); i++) { - uint32_t tok = tokens[i]; - bool is_vision_token = (tok == image_token_id && image_token_id != 0); - bool is_audio_token = (tok == audio_token_id && audio_token_id != 0); - - if (is_vision_token) { - if (in_audio_region) { - flush_audio_region(region_count); - in_audio_region = false; - } - if (!in_image_region) { - flush_text(); - in_image_region = true; - region_count = 0; - } - region_count++; - } else if (is_audio_token) { - if (in_image_region) { - flush_vision_region(region_count); - in_image_region = false; - } - if (!in_audio_region) { - flush_text(); - in_audio_region = true; - region_count = 0; - } - region_count++; - } else { - if (in_image_region) { - flush_vision_region(region_count); - in_image_region = false; - } - if (in_audio_region) { - flush_audio_region(region_count); - in_audio_region = false; - } - current_text.push_back(tok); - } - } - - if (in_image_region) - flush_vision_region(region_count); - if (in_audio_region) - flush_audio_region(region_count); - flush_text(); - - if (sequence_nodes.empty()) - throw std::runtime_error("No embedding nodes built"); - - size_t merged = sequence_nodes[0]; - for (size_t i = 1; i < sequence_nodes.size(); i++) - merged = gb->concat(merged, sequence_nodes[i], 0); - - return MultimodalInputs{ - .hidden_node = merged, - .pli_hidden_source_node = merged, - .pli_tokens = std::move(pli_tokens), - .seq_len = total_seq_len, - }; -} - -uint32_t Gemma4MmModel::decode_multimodal( - const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - - bool has_media = !image_paths.empty() || (audio_features && !audio_features->empty()); - - if (!has_media) { - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); - } - - if (temperature < 0) temperature = config_.default_temperature; - if (top_p < 0) top_p = config_.default_top_p; - if (top_k == 0) top_k = config_.default_top_k; - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - size_t cached_len = language_model_.cache_total_seq_len_; - if (cached_len >= tokens.size()) { - reset_cache(); - cached_len = 0; - } - std::vector forward_tokens = cached_len == 0 - ? tokens - : std::vector(tokens.end() - (tokens.size() - cached_len), tokens.end()); - - bool delta_has_media = std::any_of(forward_tokens.begin(), forward_tokens.end(), [&](uint32_t t) { - return (config_.image_token_id != 0 && t == config_.image_token_id) || - (config_.audio_token_id != 0 && t == config_.audio_token_id); - }); - - size_t final_hidden_node = 0; - size_t seq_len_for_updates = 0; - if (delta_has_media) { - auto result = forward_multimodal(gb, forward_tokens, image_paths, audio_features, - audio_num_frames, backend, true); - final_hidden_node = result.final_hidden_node; - seq_len_for_updates = result.seq_len; - } else { - final_hidden_node = language_model_.forward(forward_tokens, true); - seq_len_for_updates = forward_tokens.size(); - } - - auto last_hidden = gb->index(final_hidden_node, seq_len_for_updates - 1, 0); - const auto& last_buf = gb->get_output_buffer(last_hidden); - last_hidden = gb->reshape(last_hidden, {1, last_buf.shape[0]}); - - auto logits_node = gb->matmul(last_hidden, language_model_.output_weight_node_id_, true, backend); - - if (config_.final_logit_softcapping > 0.0f) { - float inv_cap = 1.0f / config_.final_logit_softcapping; - logits_node = gb->scalar_multiply(logits_node, inv_cap); - logits_node = gb->tanh(logits_node); - logits_node = gb->scalar_multiply(logits_node, config_.final_logit_softcapping); - } - - size_t sampled_token = - language_model_.sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty, nullptr); - - if (!profile_file.empty()) - gb->execute(profile_file); - else - gb->execute(); - - compute_entropy(gb, logits_node, out_entropy); - - language_model_.post_execute_updates(gb, seq_len_for_updates); - language_model_.cache_total_seq_len_ += seq_len_for_updates; - - auto* output_ptr = gb->get_output(sampled_token); - uint32_t result_token = *static_cast(output_ptr); - language_model_.record_sampled_token(result_token); - return result_token; -} - -size_t Gemma4MmModel::forward(const std::vector& tokens, bool use_cache) { - return language_model_.forward(tokens, use_cache); -} - -uint32_t Gemma4MmModel::decode(const std::vector& tokens, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -void Gemma4MmModel::prefill(const std::vector& tokens, size_t chunk_size, - const std::string& profile_file) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - language_model_.prefill(tokens, chunk_size, profile_file); -} - -void Gemma4MmModel::prefill_with_images(const std::vector& tokens, - const std::vector& image_paths, - const std::string& profile_file) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - - if (image_paths.empty()) { - prefill(tokens, get_prefill_chunk_size(), profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - auto result = forward_multimodal(gb, tokens, image_paths, nullptr, 0, backend, true); - - if (!profile_file.empty()) - gb->execute(profile_file); - else - gb->execute(); - - language_model_.post_execute_updates(gb, result.seq_len); - language_model_.cache_total_seq_len_ += result.seq_len; -} - -uint32_t Gemma4MmModel::decode_with_images( - const std::vector& tokens, const std::vector& image_paths, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - return decode_multimodal(tokens, image_paths, nullptr, 0, - temperature, top_p, top_k, profile_file, out_entropy, - min_p, repetition_penalty); -} - -uint32_t Gemma4MmModel::decode_with_audio( - const std::vector& tokens, const std::vector& audio_features, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty, - float* /*out_token_time_start*/, float* /*out_token_time_end*/) { - size_t num_frames = audio_features.size() / config_.audio_input_feat_size; - std::vector empty_images; - return decode_multimodal(tokens, empty_images, &audio_features, num_frames, - temperature, top_p, top_k, profile_file, out_entropy, - min_p, repetition_penalty); -} - -uint32_t Gemma4MmModel::decode_with_media( - const std::vector& tokens, - const std::vector& image_paths, - const std::vector& audio_features, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - size_t num_frames = audio_features.size() / config_.audio_input_feat_size; - return decode_multimodal(tokens, image_paths, &audio_features, num_frames, - temperature, top_p, top_k, profile_file, out_entropy, - min_p, repetition_penalty); -} - -std::vector Gemma4MmModel::get_image_embeddings(const std::string& image_path) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - auto preprocessed = vision_encoder_.preprocess_image(image_path); - size_t vision_output = vision_encoder_.forward_vision(gb, preprocessed, backend); - size_t projected = vision_encoder_.build_vision_projector(gb, vision_output, backend); - - gb->execute(); - - const auto& buf = gb->get_output_buffer(projected); - size_t total = buf.total_size; - std::vector embedding(total); - const __fp16* fp16_data = buf.data_as<__fp16>(); - for (size_t i = 0; i < total; i++) - embedding[i] = static_cast(fp16_data[i]); - return embedding; -} - -std::vector Gemma4MmModel::get_audio_embeddings(const std::vector& audio_features) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - size_t num_frames = audio_features.size() / config_.audio_input_feat_size; - size_t audio_output = audio_encoder_.forward_audio(gb, audio_features, num_frames, backend); - size_t projected = audio_encoder_.build_audio_projector(gb, audio_output, backend); - - gb->execute(); - - const auto& buf = gb->get_output_buffer(projected); - size_t total = buf.total_size; - std::vector embedding(total); - const __fp16* fp16_data = buf.data_as<__fp16>(); - for (size_t i = 0; i < total; i++) - embedding[i] = static_cast(fp16_data[i]); - return embedding; -} - -size_t Gemma4MmModel::build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_attention should not be called directly on Gemma4MmModel"); -} - -size_t Gemma4MmModel::build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const { - throw std::runtime_error("build_mlp should not be called directly on Gemma4MmModel"); -} - -size_t Gemma4MmModel::build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_transformer_block should not be called directly on Gemma4MmModel"); -} - -} -} diff --git a/cactus-engine/models/gemma4/model_gemma4_vision.cpp b/cactus-engine/models/gemma4/model_gemma4_vision.cpp deleted file mode 100644 index a0ded152b..000000000 --- a/cactus-engine/models/gemma4/model_gemma4_vision.cpp +++ /dev/null @@ -1,563 +0,0 @@ -#include "model_gemma4.h" -#include "cactus_graph.h" -#include "cactus_kernels.h" -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -static std::pair, std::vector> compute_2d_rope_tables( - const Gemma4VisionModel::PreprocessedImage& img, size_t max_patches, size_t head_dim, float theta) { - - size_t half_dim = head_dim / 2; - size_t freq_per_dim = half_dim / 2; - - std::vector cos_table(max_patches * head_dim, 1.0f); - std::vector sin_table(max_patches * head_dim, 0.0f); - - for (size_t py = 0; py < img.patch_height; py++) { - for (size_t px = 0; px < img.patch_width; px++) { - size_t patch_idx = py * img.patch_width + px; - - for (size_t i = 0; i < freq_per_dim; i++) { - float inv_freq = 1.0f / std::pow(theta, 2.0f * i / static_cast(half_dim)); - - float x_angle = static_cast(px) * inv_freq; - cos_table[patch_idx * head_dim + i] = std::cos(x_angle); - cos_table[patch_idx * head_dim + freq_per_dim + i] = std::cos(x_angle); - sin_table[patch_idx * head_dim + i] = std::sin(x_angle); - sin_table[patch_idx * head_dim + freq_per_dim + i] = std::sin(x_angle); - - float y_angle = static_cast(py) * inv_freq; - cos_table[patch_idx * head_dim + half_dim + i] = std::cos(y_angle); - cos_table[patch_idx * head_dim + half_dim + freq_per_dim + i] = std::cos(y_angle); - sin_table[patch_idx * head_dim + half_dim + i] = std::sin(y_angle); - sin_table[patch_idx * head_dim + half_dim + freq_per_dim + i] = std::sin(y_angle); - } - } - } - - return {cos_table, sin_table}; -} - -static std::pair, std::vector<__fp16>> compute_npu_rope_tables( - const Gemma4VisionModel::PreprocessedImage& img, size_t max_patches, size_t head_dim, float theta) { - - auto [cos_f32, sin_f32] = compute_2d_rope_tables(img, max_patches, head_dim, theta); - - std::vector<__fp16> cos_full(max_patches * head_dim); - std::vector<__fp16> sin_signed(max_patches * head_dim); - - for (size_t p = 0; p < max_patches; p++) { - size_t base = p * head_dim; - for (size_t d = 0; d < head_dim; d++) { - cos_full[base + d] = static_cast<__fp16>(cos_f32[base + d]); - sin_signed[base + d] = static_cast<__fp16>(sin_f32[base + d]); - } - } - - return {cos_full, sin_signed}; -} - -Gemma4VisionModel::Gemma4VisionModel() : Model() {} -Gemma4VisionModel::Gemma4VisionModel(const Config& config) : Model(config) {} - -void Gemma4VisionModel::load_weights_to_graph(CactusGraph* gb) { - auto resolve = [&](const std::string& name) -> std::string { - return model_folder_path_ + "/" + name; - }; - - if (!disable_npu_ && npu::is_npu_available()) { - std::string npu_path = model_folder_path_ + "/vision_encoder.mlpackage"; - if (std::filesystem::exists(npu_path)) { - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - if (input_shape.empty()) { - input_shape = {static_cast(config_.vision_default_output_length * - config_.vision_pooling_kernel_size * config_.vision_pooling_kernel_size), - static_cast(config_.vision_embed_dim)}; - } - npu_encoder_->preallocate(input_shape, "hidden_states", "output"); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - CACTUS_LOG_DEBUG("npu", "[gemma4-vision] found vision_encoder.mlpackage but failed to enable NPU vision encoder; using CPU"); - } - } - } - - vision_weights_.patch_input_proj = gb->mmap_weights(resolve("vision_patch_embedder_input_proj.weights")); - vision_weights_.position_table = gb->mmap_weights(resolve("vision_patch_embedder_position_embedding_table.weights")); - - if (!use_npu_encoder_) { - vision_weights_.layers.resize(config_.vision_num_layers); - for (uint32_t i = 0; i < config_.vision_num_layers; i++) { - auto& layer = vision_weights_.layers[i]; - std::string prefix = resolve("vision_encoder_layers_" + std::to_string(i) + "_"); - - layer.attn_q_weight = gb->mmap_weights(prefix + "self_attn_q_proj.weights"); - layer.attn_k_weight = gb->mmap_weights(prefix + "self_attn_k_proj.weights"); - layer.attn_v_weight = gb->mmap_weights(prefix + "self_attn_v_proj.weights"); - layer.attn_output_weight = gb->mmap_weights(prefix + "self_attn_o_proj.weights"); - layer.attn_q_norm = gb->mmap_weights(prefix + "self_attn_q_norm.weights"); - layer.attn_k_norm = gb->mmap_weights(prefix + "self_attn_k_norm.weights"); - layer.input_layernorm = gb->mmap_weights(prefix + "input_layernorm.weights"); - layer.post_attention_layernorm = gb->mmap_weights(prefix + "post_attention_layernorm.weights"); - layer.pre_feedforward_layernorm = gb->mmap_weights(prefix + "pre_feedforward_layernorm.weights"); - layer.post_feedforward_layernorm = gb->mmap_weights(prefix + "post_feedforward_layernorm.weights"); - layer.mlp_gate_proj = gb->mmap_weights(prefix + "mlp_gate_proj.weights"); - layer.mlp_up_proj = gb->mmap_weights(prefix + "mlp_up_proj.weights"); - layer.mlp_down_proj = gb->mmap_weights(prefix + "mlp_down_proj.weights"); - layer.layer_scalar = std::filesystem::exists(prefix + "layer_scalar.weights") ? gb->mmap_weights(prefix + "layer_scalar.weights") : 0; - } - } - - vision_weights_.embed_vision_proj = gb->mmap_weights(resolve("embed_vision_proj.weights")); - - size_t vision_head_dim = config_.vision_head_dim; - vision_v_norm_ones_.assign(vision_head_dim, static_cast<__fp16>(1.0f)); - vision_v_norm_ones_node_ = gb->input({vision_head_dim}, Precision::FP16); - gb->set_external_input(vision_v_norm_ones_node_, vision_v_norm_ones_.data(), Precision::FP16); - - std::string post_proj_norm_path = resolve("embed_vision_post_proj_norm.weights"); - if (std::filesystem::exists(post_proj_norm_path)) { - vision_weights_.post_proj_norm = gb->mmap_weights(post_proj_norm_path); - } else { - post_proj_norm_ones_.assign(config_.hidden_dim, static_cast<__fp16>(1.0f)); - vision_weights_.post_proj_norm = gb->input({config_.hidden_dim}, Precision::FP16); - gb->set_external_input(vision_weights_.post_proj_norm, post_proj_norm_ones_.data(), Precision::FP16); - } - - output_weight_node_id_ = 0; -} - -Gemma4VisionModel::PreprocessedImage Gemma4VisionModel::preprocess_image(const std::string& image_path) { - int w, h, channels; - unsigned char* data = cactus_image_load(image_path.c_str(), &w, &h, &channels, 3); - if (!data) - throw std::runtime_error("Failed to load image: " + image_path); - - uint32_t patch_size = config_.vision_patch_size; - uint32_t pooling_k = config_.vision_pooling_kernel_size; - uint32_t max_patches = config_.vision_default_output_length * pooling_k * pooling_k; - uint32_t side_mult = pooling_k * patch_size; - - float total_px = static_cast(h) * static_cast(w); - float target_px = static_cast(max_patches) * static_cast(patch_size * patch_size); - float factor = std::sqrt(target_px / total_px); - - int target_h = static_cast(std::floor(factor * h / side_mult)) * side_mult; - int target_w = static_cast(std::floor(factor * w / side_mult)) * side_mult; - - if (target_h == 0) target_h = side_mult; - if (target_w == 0) target_w = side_mult; - - std::vector resized_data; - unsigned char* src = data; - if (target_h != h || target_w != w) { - resized_data.resize(target_h * target_w * 3); - cactus_image_resize_uint8(data, w, h, resized_data.data(), target_w, target_h, 3); - src = resized_data.data(); - } - - size_t ph = target_h / patch_size; - size_t pw = target_w / patch_size; - size_t num_patches = ph * pw; - - float mean = config_.image_mean; - float inv_std = 1.0f / config_.image_std; - float rescale = config_.rescale_factor; - - std::vector pixels(3 * target_h * target_w); - for (int y = 0; y < target_h; y++) { - for (int x = 0; x < target_w; x++) { - size_t src_idx = (y * target_w + x) * 3; - for (int c = 0; c < 3; c++) { - pixels[c * target_h * target_w + y * target_w + x] = - (static_cast(src[src_idx + c]) * rescale - mean) * inv_std; - } - } - } - - cactus_image_free(data); - - return PreprocessedImage{ - std::move(pixels), - static_cast(target_h), - static_cast(target_w), - ph, pw, num_patches - }; -} - -size_t Gemma4VisionModel::build_vision_patch_embedding(CactusGraph* gb, const PreprocessedImage& img, - ComputeBackend backend) { - uint32_t patch_size = config_.vision_patch_size; - size_t patch_dim = 3 * patch_size * patch_size; - size_t num_patches = img.num_patches; - size_t hidden_size = config_.vision_embed_dim; - size_t pos_embed_size = config_.vision_position_embedding_size; - - size_t pixel_input = gb->input({3, img.height, img.width}, Precision::FP32); - gb->set_input(pixel_input, img.pixel_values.data(), Precision::FP32); - - size_t pixel_fp16 = gb->precision_cast(pixel_input, Precision::FP16); - size_t reshaped = gb->reshape(pixel_fp16, {3, img.patch_height, patch_size, img.patch_width, patch_size}); - size_t permuted = gb->transposeN(reshaped, {1, 3, 2, 4, 0}); - size_t patches = gb->reshape(permuted, {num_patches, patch_dim}); - - size_t projected = gb->matmul(patches, vision_weights_.patch_input_proj, true, backend); - - size_t pos_table = gb->reshape(vision_weights_.position_table, {2, pos_embed_size, hidden_size}); - - std::vector x_positions(num_patches); - std::vector y_positions(num_patches); - for (size_t py = 0; py < img.patch_height; py++) { - for (size_t px = 0; px < img.patch_width; px++) { - size_t idx = py * img.patch_width + px; - x_positions[idx] = static_cast(px); - y_positions[idx] = static_cast(py); - } - } - - size_t x_pos_input = gb->input({num_patches}, Precision::FP32); - gb->set_input(x_pos_input, x_positions.data(), Precision::FP32); - size_t y_pos_input = gb->input({num_patches}, Precision::FP32); - gb->set_input(y_pos_input, y_positions.data(), Precision::FP32); - - size_t x_table = gb->slice(pos_table, 0, 0, 1); - x_table = gb->reshape(x_table, {pos_embed_size, hidden_size}); - size_t y_table = gb->slice(pos_table, 0, 1, 1); - y_table = gb->reshape(y_table, {pos_embed_size, hidden_size}); - - size_t x_embed = gb->embedding(x_table, x_pos_input); - size_t y_embed = gb->embedding(y_table, y_pos_input); - size_t pos_embed = gb->add(x_embed, y_embed); - - size_t pos_embed_fp16 = gb->precision_cast(pos_embed, Precision::FP16); - - return gb->add(projected, pos_embed_fp16); -} - -std::pair Gemma4VisionModel::build_2d_rope_nodes( - CactusGraph* gb, const PreprocessedImage& img, size_t max_patches) { - - size_t head_dim = config_.vision_head_dim; - auto rope_tables = compute_2d_rope_tables(img, max_patches, head_dim, config_.vision_rope_theta); - auto& cos_table = rope_tables.first, sin_table = rope_tables.second; - - size_t cos_input = gb->input({max_patches, head_dim}, Precision::FP32); - gb->set_input(cos_input, cos_table.data(), Precision::FP32); - size_t cos_node = gb->precision_cast(cos_input, Precision::FP16); - cos_node = gb->reshape(cos_node, {1, max_patches, 1, head_dim}); - - size_t sin_input = gb->input({max_patches, head_dim}, Precision::FP32); - gb->set_input(sin_input, sin_table.data(), Precision::FP32); - size_t sin_node = gb->precision_cast(sin_input, Precision::FP16); - sin_node = gb->reshape(sin_node, {1, max_patches, 1, head_dim}); - - return {cos_node, sin_node}; -} - -size_t Gemma4VisionModel::build_padding_mask(CactusGraph* gb, size_t num_real, size_t max_patches) { - if (num_real >= max_patches) - return 0; - - std::vector mask(max_patches * max_patches, 0.0f); - for (size_t q = 0; q < num_real; q++) - for (size_t k = 0; k < num_real; k++) - mask[q * max_patches + k] = 1.0f; - size_t mask_input = gb->input({max_patches, max_patches}, Precision::FP32); - gb->set_input(mask_input, mask.data(), Precision::FP32); - size_t mask_node = gb->precision_cast(mask_input, Precision::FP16); - return gb->reshape(mask_node, {1, max_patches, max_patches}); -} - -size_t Gemma4VisionModel::build_vision_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend) { - const auto& layer = vision_weights_.layers[layer_idx]; - size_t head_dim = config_.vision_head_dim; - size_t num_heads = config_.vision_attention_heads; - size_t kv_heads = config_.vision_kv_heads; - - const auto& input_buf = gb->get_output_buffer(input); - size_t seq_len = input_buf.shape[0]; - - auto q = gb->matmul(input, layer.attn_q_weight, true, backend); - q = gb->reshape(q, {seq_len * num_heads, head_dim}); - q = gb->rms_norm(q, layer.attn_q_norm, config_.layer_norm_eps); - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - - auto k = gb->matmul(input, layer.attn_k_weight, true, backend); - k = gb->reshape(k, {seq_len * kv_heads, head_dim}); - k = gb->rms_norm(k, layer.attn_k_norm, config_.layer_norm_eps); - k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); - - auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); - auto v = gb->rms_norm(gb->reshape(v_proj, {seq_len * kv_heads, head_dim}), - vision_v_norm_ones_node_, config_.layer_norm_eps); - v = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); - - size_t half_dim = head_dim / 2; - - auto apply_2d_rope = [&](size_t tensor) -> size_t { - auto first_half = gb->slice(tensor, 3, 0, half_dim); - auto second_half = gb->slice(tensor, 3, half_dim, half_dim); - - auto rot_first_a = gb->slice(first_half, 3, 0, half_dim / 2); - auto rot_first_b = gb->slice(first_half, 3, half_dim / 2, half_dim / 2); - auto neg_b = gb->scalar_multiply(rot_first_b, -1.0f); - - size_t cos_x = gb->slice(cos_node, 3, 0, half_dim); - size_t sin_x = gb->slice(sin_node, 3, 0, half_dim); - - auto first_rotated = gb->add( - gb->multiply(first_half, cos_x), - gb->multiply(gb->concat(neg_b, rot_first_a, 3), sin_x)); - - auto rot_second_a = gb->slice(second_half, 3, 0, half_dim / 2); - auto rot_second_b = gb->slice(second_half, 3, half_dim / 2, half_dim / 2); - auto neg_sb = gb->scalar_multiply(rot_second_b, -1.0f); - - size_t cos_y = gb->slice(cos_node, 3, half_dim, half_dim); - size_t sin_y = gb->slice(sin_node, 3, half_dim, half_dim); - - auto second_rotated = gb->add( - gb->multiply(second_half, cos_y), - gb->multiply(gb->concat(neg_sb, rot_second_a, 3), sin_y)); - - return gb->concat(first_rotated, second_rotated, 3); - }; - - auto q_rot = apply_2d_rope(q); - auto k_rot = apply_2d_rope(k); - - size_t attn; - if (attn_mask_node != 0) - attn = gb->attention_masked(q_rot, k_rot, v, attn_mask_node, 1.f, false, backend); - else - attn = gb->attention(q_rot, k_rot, v, 1.f, false, backend); - - return gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), - layer.attn_output_weight, true, backend); -} - -size_t Gemma4VisionModel::build_vision_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) { - const auto& layer = vision_weights_.layers[layer_idx]; - auto gate = gb->gelu(gb->matmul(input, layer.mlp_gate_proj, true, backend)); - auto up = gb->matmul(input, layer.mlp_up_proj, true, backend); - return gb->matmul(gb->multiply(gate, up), layer.mlp_down_proj, true, backend); -} - -size_t Gemma4VisionModel::build_vision_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend) { - const auto& layer = vision_weights_.layers[layer_idx]; - - auto normed = gb->rms_norm(hidden, layer.input_layernorm, config_.layer_norm_eps); - auto attn_raw = build_vision_attention(gb, normed, layer_idx, cos_node, sin_node, attn_mask_node, backend); - auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm, config_.layer_norm_eps); - auto residual = gb->add(hidden, attn); - - auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm, config_.layer_norm_eps); - auto mlp_raw = build_vision_mlp(gb, pre_mlp, layer_idx, backend); - auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm, config_.layer_norm_eps); - - auto out = gb->add(residual, mlp); - - if (layer.layer_scalar != 0) - out = gb->multiply(out, layer.layer_scalar); - - return out; -} - -size_t Gemma4VisionModel::build_vision_pooler(CactusGraph* gb, size_t hidden, const PreprocessedImage& img, - ComputeBackend backend) { - size_t k = config_.vision_pooling_kernel_size; - size_t output_length = config_.vision_default_output_length; - size_t num_patches = img.num_patches; - size_t k_squared = k * k; - - if (num_patches == output_length) - return hidden; - - size_t max_x = img.patch_width; - size_t max_y = img.patch_height; - - std::vector pool_weights(num_patches * output_length, 0.0f); - std::vector valid_bins(output_length, false); - for (size_t py = 0; py < max_y; py++) { - for (size_t px = 0; px < max_x; px++) { - size_t patch_idx = py * max_x + px; - size_t kx = px / k; - size_t ky = py / k; - size_t kernel_idx = kx + (max_x / k) * ky; - if (kernel_idx < output_length) { - pool_weights[kernel_idx * num_patches + patch_idx] = 1.0f / static_cast(k_squared); - valid_bins[kernel_idx] = true; - } - } - } - - size_t weights_node = gb->input({output_length, num_patches}, Precision::FP32); - gb->set_input(weights_node, pool_weights.data(), Precision::FP32); - size_t weights_fp16 = gb->precision_cast(weights_node, Precision::FP16); - - size_t pooled = gb->matmul(weights_fp16, hidden, false, backend); - - size_t valid_count = 0; - for (bool v : valid_bins) if (v) valid_count++; - - if (valid_count == output_length) - return pooled; - - std::vector valid_slices; - for (size_t i = 0; i < output_length; i++) { - if (valid_bins[i]) { - auto slice = gb->slice(pooled, 0, i, 1); - valid_slices.push_back(slice); - } - } - - size_t stripped = valid_slices[0]; - for (size_t i = 1; i < valid_slices.size(); i++) - stripped = gb->concat(stripped, valid_slices[i], 0); - - return stripped; -} - -size_t Gemma4VisionModel::forward_vision(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend) { - size_t hidden = build_vision_patch_embedding(gb, img, backend); - - size_t num_real = img.num_patches; - size_t hidden_size = config_.vision_embed_dim; - uint32_t pooling_k = config_.vision_pooling_kernel_size; - size_t max_patches = config_.vision_default_output_length * pooling_k * pooling_k; - bool can_use_npu_path = use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available(); - if (can_use_npu_path) { - std::vector npu_input_shape = npu_encoder_->get_input_shape(); - if (npu_input_shape.size() >= 2) { - size_t a = static_cast(npu_input_shape[0]); - size_t b = static_cast(npu_input_shape[1]); - if (b == hidden_size) { - max_patches = a; - } else if (a == hidden_size) { - max_patches = b; - } else if (a > 0) { - max_patches = a; - } - } - } - if (num_real > max_patches) { - if (can_use_npu_path) { - CACTUS_LOG_WARN("npu", "[gemma4-vision] image has more patches than NPU encoder supports; falling back to CPU vision encoder"); - can_use_npu_path = false; - max_patches = config_.vision_default_output_length * pooling_k * pooling_k; - } - } - if (!can_use_npu_path) { - max_patches = num_real; - } - size_t num_padding = max_patches - num_real; - - if (num_padding > 0) { - std::vector<__fp16> zeros(num_padding * hidden_size, static_cast<__fp16>(0.0f)); - size_t pad_node = gb->input({num_padding, hidden_size}, Precision::FP16); - gb->set_input(pad_node, zeros.data(), Precision::FP16); - hidden = gb->concat(hidden, pad_node, 0); - } - - auto [cos_node, sin_node] = build_2d_rope_nodes(gb, img, max_patches); - size_t attn_mask_node = build_padding_mask(gb, num_real, max_patches); - - if (can_use_npu_path) { - gb->execute(); - - const auto& h_buf = gb->get_output_buffer(hidden); - if (h_buf.precision != Precision::FP16) - throw std::runtime_error("[gemma4-vision] expected FP16 hidden output for NPU path"); - std::vector<__fp16> hidden_fp16(max_patches * hidden_size); - memcpy(hidden_fp16.data(), h_buf.data_as<__fp16>(), max_patches * hidden_size * sizeof(__fp16)); - - size_t head_dim = config_.vision_head_dim; - auto [cos_fp16, sin_fp16] = compute_npu_rope_tables(img, max_patches, head_dim, config_.vision_rope_theta); - - std::vector<__fp16> additive_mask(max_patches * max_patches, static_cast<__fp16>(0.0f)); - if (num_padding > 0) { - for (size_t q = 0; q < max_patches; q++) - for (size_t k = 0; k < max_patches; k++) - if (q >= num_real || k >= num_real) - additive_mask[q * max_patches + k] = static_cast<__fp16>(-65504.0f); - } - - std::vector inputs = { - {"hidden_states", hidden_fp16.data(), {static_cast(max_patches), static_cast(hidden_size)}}, - {"cos_full", cos_fp16.data(), {static_cast(max_patches), 1, static_cast(head_dim)}}, - {"sin_signed", sin_fp16.data(), {static_cast(max_patches), 1, static_cast(head_dim)}}, - {"attention_mask", additive_mask.data(), {1, static_cast(max_patches), static_cast(max_patches)}}, - }; - - std::vector out_shape = npu_encoder_->get_output_shape(); - size_t out_elements = 1; - for (int d : out_shape) out_elements *= d; - size_t required = max_patches * hidden_size; - if (out_elements < required) - out_elements = required; - - std::vector<__fp16> npu_output(out_elements); - size_t written = npu_encoder_->encode_multimodal_input(inputs, npu_output.data(), "output"); - - if (written >= required) { - hidden = gb->input({max_patches, hidden_size}, Precision::FP16); - gb->set_input(hidden, npu_output.data(), Precision::FP16); - } else { - CACTUS_LOG_WARN("npu", "[gemma4-vision] encode_multimodal_input returned insufficient output, falling back to CPU"); - for (uint32_t i = 0; i < config_.vision_num_layers; i++) - hidden = build_vision_transformer_block(gb, hidden, i, cos_node, sin_node, attn_mask_node, backend); - } - } else { - for (uint32_t i = 0; i < config_.vision_num_layers; i++) { - auto [cn, sn] = build_2d_rope_nodes(gb, img, num_real); - hidden = build_vision_transformer_block(gb, hidden, i, cn, sn, 0, backend); - gb->execute(); - - const auto& out_buf = gb->get_output_buffer(hidden); - std::vector tmp(out_buf.byte_size); - std::memcpy(tmp.data(), out_buf.get_data(), out_buf.byte_size); - auto shape = out_buf.shape; - auto prec = out_buf.precision; - - const auto& lw = vision_weights_.layers[i]; - for (size_t wn : {lw.attn_q_weight, lw.attn_k_weight, lw.attn_v_weight, - lw.attn_output_weight, lw.mlp_gate_proj, lw.mlp_up_proj, lw.mlp_down_proj}) - gb->release_weight_pages(wn); - - gb->soft_reset_keep_pool(); - hidden = gb->input(shape, prec); - gb->set_input(hidden, tmp.data(), prec); - } - gb->release_weight_pages(vision_weights_.patch_input_proj); - gb->release_weight_pages(vision_weights_.position_table); - } - - if (num_padding > 0) - hidden = gb->slice(hidden, 0, 0, num_real); - - return build_vision_pooler(gb, hidden, img, backend); -} - -size_t Gemma4VisionModel::build_vision_projector(CactusGraph* gb, size_t vision_features, ComputeBackend backend) { - size_t projected = gb->matmul(vision_features, vision_weights_.embed_vision_proj, true, backend); - return gb->rms_norm(projected, vision_weights_.post_proj_norm, config_.layer_norm_eps); -} - -} -} diff --git a/cactus-engine/models/model.h b/cactus-engine/models/model.h deleted file mode 100644 index 5e60c43e8..000000000 --- a/cactus-engine/models/model.h +++ /dev/null @@ -1,302 +0,0 @@ -#pragma once - -#include "../src/engine.h" - -namespace cactus { -namespace engine { - -class QwenModel : public Model { -public: - QwenModel(); - explicit QwenModel(const Config& config); - ~QwenModel() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - }; - - std::vector layers; - } weight_nodes_; -}; - -class Lfm2VlModel; - -class Siglip2VisionModel : public Model { - friend class Lfm2VlModel; - -public: - struct VisionEmbeddingResult { - size_t combined_embeddings; - std::vector tile_embeddings; - }; - - Siglip2VisionModel(); - explicit Siglip2VisionModel(const Config& cfg); - ~Siglip2VisionModel() override = default; - - virtual size_t forward_vision(const Siglip2Preprocessor::PreprocessedImage& preprocessed_image); - virtual size_t forward_vision(CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend); - std::vector get_image_embedding(const std::string& image_path); - Siglip2Preprocessor& get_preprocessor() { return preprocessor_; } - const Siglip2Preprocessor& get_preprocessor() const { return preprocessor_; } - -protected: - VisionEmbeddingResult build_vision_embeddings(CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend); - size_t build_vision_transformer_layer(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, - ComputeBackend backend); - size_t build_vision_attention(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, - ComputeBackend backend); - size_t build_vision_mlp(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, - ComputeBackend backend); - void ensure_cpu_vision_weights_loaded(CactusGraph* gb); - - void load_weights_to_graph(CactusGraph* gb) override; - size_t forward(const std::vector& tokens, bool use_cache = false) override; - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - struct VisionWeightNodeIDs { - size_t patch_embedding_weight; - size_t patch_embedding_bias; - size_t position_embedding; - size_t post_layernorm_weight; - size_t post_layernorm_bias; - - struct VisionLayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t attn_q_bias; - size_t attn_k_bias; - size_t attn_v_bias; - size_t attn_output_bias; - size_t layer_norm1_weight; - size_t layer_norm1_bias; - size_t layer_norm2_weight; - size_t layer_norm2_bias; - size_t mlp_fc1_weight; - size_t mlp_fc1_bias; - size_t mlp_fc2_weight; - size_t mlp_fc2_bias; - }; - - std::vector vision_layers; - } vision_weight_nodes_; - - Siglip2Preprocessor preprocessor_; - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool cpu_vision_weights_loaded_ = false; -}; - -class LFM2Model : public Model { - friend class Lfm2VlModel; - -public: - LFM2Model(); - explicit LFM2Model(const Config& config); - ~LFM2Model() override = default; - - bool is_cache_empty() const; - void update_kv_cache(CactusGraph* gb, size_t seq_len); - bool init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - -protected: - using Model::forward; - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - size_t build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache); - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - size_t forward(const std::vector& tokens, bool use_cache = false) override; - size_t forward(CactusGraph* gb, const std::vector& tokens, ComputeBackend backend, bool use_cache = false); - size_t forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, ComputeBackend backend, bool use_cache = false); - void post_init() override; - void post_execute_updates(CactusGraph* gb, size_t seq_len) override; - void reset_cache() override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t conv_depthwise_weight; - size_t conv_in_proj_weight; - size_t conv_out_proj_weight; - size_t input_layernorm_weight; - size_t post_attention_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - }; - - enum class LayerType : uint8_t { ATTENTION, CONV }; - - struct LayerEntry { - LayerType type; - LayerWeights weights; - }; - - std::vector layers; - } weight_nodes_; - - ConvCache conv_cache_; - std::vector conv_cache_bx_nodes_; - bool last_forward_used_cache_ = false; -}; - -class Lfm2VlModel : public Model { -public: - Lfm2VlModel(); - explicit Lfm2VlModel(const Config& config); - ~Lfm2VlModel() override = default; - - bool init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - size_t forward(const std::vector& tokens, bool use_cache = false) override; - uint32_t decode(const std::vector& tokens, - float temperature = -1.0f, - float top_p = -1.0f, - size_t top_k = 0, - const std::string& profile_file = "", - float* out_entropy = nullptr, - float min_p = 0.15f, - float repetition_penalty = 1.1f) override; - void prefill(const std::vector& tokens, size_t chunk_size = 256, - const std::string& profile_file = "") override; - void prefill_with_images(const std::vector& tokens, - const std::vector& image_paths, - const std::string& profile_file = "") override; - uint32_t decode_with_images(const std::vector& tokens, - const std::vector& image_paths, - float temperature = -1.0f, - float top_p = -1.0f, - size_t top_k = 0, - const std::string& profile_file = "", - float* out_entropy = nullptr, - float min_p = 0.15f, - float repetition_penalty = 1.1f) override; - void reset_cache() override; - std::vector get_image_embeddings(const std::string& image_path) override; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override; - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct ProjectedTileFeature { - size_t node_id; - size_t token_count; - }; - - struct TextEmbeddingInput { - size_t input_node; - std::vector tokens; - }; - - struct MergedEmbeddingResult { - size_t node_id; - size_t seq_len; - }; - - struct ForwardImageResult { - size_t final_hidden_node; - size_t seq_len; - }; - - std::vector get_image_features( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend); - ForwardImageResult forward_images(CactusGraph* gb, - const std::vector& tokens, - const std::vector& image_paths, - ComputeBackend backend, - bool use_cache); - size_t build_multimodal_projector(CactusGraph* gb, - size_t image_features, - size_t tile_h, - size_t tile_w, - ComputeBackend backend); - size_t pixel_unshuffle(CactusGraph* gb, size_t hidden_states, size_t height, size_t width, size_t channels); - MergedEmbeddingResult merge_image_text_embeddings( - CactusGraph* gb, - const std::vector& tokens, - const std::vector>& image_embedding_nodes, - std::vector& text_embedding_inputs); - - Siglip2VisionModel vision_tower_; - LFM2Model language_model_; - Siglip2Preprocessor preprocessor_; - - struct ProjectorWeights { - size_t layer_norm_weight; - size_t layer_norm_bias; - size_t linear_1_weight; - size_t linear_1_bias; - size_t linear_2_weight; - size_t linear_2_bias; - } projector_weights_; - - bool vision_weights_loaded_ = false; - bool language_weights_loaded_ = false; - bool image_prefill_completed_ = false; - size_t last_token_count_ = 0; -}; - -} // namespace engine -} // namespace cactus - -#include "gemma4/model_gemma4.h" diff --git a/cactus-engine/models/model_lfm2.cpp b/cactus-engine/models/model_lfm2.cpp deleted file mode 100644 index 8be28b4f3..000000000 --- a/cactus-engine/models/model_lfm2.cpp +++ /dev/null @@ -1,384 +0,0 @@ -#include "model.h" -#include "cactus_graph.h" -#include -#include -#include -#include -#include -#include -#include -namespace cactus { -namespace engine { -LFM2Model::LFM2Model() : Model() { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -LFM2Model::LFM2Model(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -void LFM2Model::post_init() { - if (config_.conv_L_cache > 0) { - Precision cache_precision = Precision::FP16; - size_t conv_window = config_.conv_L_cache > 0 ? config_.conv_L_cache - 1 : 0; - conv_cache_.init(config_.num_layers, config_.hidden_dim, conv_window, cache_precision); - - } - last_forward_used_cache_ = false; - -} -void LFM2Model::reset_cache() { - Model::reset_cache(); - - if (conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } -} -bool LFM2Model::is_cache_empty() const { - return cache_total_seq_len_ == 0; -} -void LFM2Model::update_kv_cache(CactusGraph* /*gb*/, size_t seq_len) { - cache_total_seq_len_ += seq_len; -} -bool LFM2Model::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - if (!Model::init(model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -bool LFM2Model::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!Model::init(external_graph, model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -void LFM2Model::load_weights_to_graph(CactusGraph* gb) { - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - } - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - } - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - - } - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer_entry = weight_nodes_.layers[i]; - auto& layer = layer_entry.weights; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - - bool is_conv_layer = false; - if (i < config_.layer_types.size()) { - std::string layer_type = config_.layer_types[i]; - is_conv_layer = (layer_type == "conv" || layer_type == "CONV"); - - } - if (is_conv_layer) { - layer_entry.type = WeightNodeIDs::LayerType::CONV; - layer.conv_in_proj_weight = gb->mmap_weights(layer_prefix + "conv_in_proj.weights"); - layer.conv_out_proj_weight = gb->mmap_weights(layer_prefix + "conv_out_proj.weights"); - layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); - - } else { - layer_entry.type = WeightNodeIDs::LayerType::ATTENTION; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); - - } - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - - } - -} -size_t LFM2Model::build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - size_t in_proj = gb->matmul(input, layer.conv_in_proj_weight, true, backend); - const auto& in_proj_buf = gb->get_output_buffer(in_proj); - if (in_proj_buf.shape.size() != 2 || (in_proj_buf.shape[1] % 3) != 0) { - throw std::runtime_error("Conv in_proj output must be [L, 3*C]"); - } - - const size_t L = in_proj_buf.shape[0]; - const size_t C = in_proj_buf.shape[1] / 3; - - size_t triplet = gb->reshape(in_proj, {L, static_cast(3), C}); - size_t B = gb->slice(triplet, 1, 0, 1); - size_t Cg = gb->slice(triplet, 1, 1, 1); - size_t X = gb->slice(triplet, 1, 2, 1); - - B = gb->reshape(B, {L, C}); - - Cg = gb->reshape(Cg, {L, C}); - - X = gb->reshape(X, {L, C}); - size_t Bx = gb->multiply(B, X); - if (use_cache) { - conv_cache_bx_nodes_[layer_idx] = Bx; - - } else { - conv_cache_bx_nodes_[layer_idx] = 0; - - } - const auto& wbuf = gb->get_output_buffer(layer.conv_depthwise_weight); - size_t K = wbuf.shape.back(); - - size_t conv_w = layer.conv_depthwise_weight; - if (wbuf.shape.size() == 2) { - K = wbuf.shape[1]; - conv_w = gb->reshape(conv_w, {wbuf.shape[0], static_cast(1), K}); - } else if (wbuf.shape.size() != 3){ - throw std::runtime_error("Unexpected depthwise weight rank"); - } - K = wbuf.shape.back(); - - size_t conv_input_lc = Bx; - if (use_cache && conv_cache_.window_size > 0) { - auto view = conv_cache_.get_window(layer_idx); - std::vector segments; - if (view.len2 > 0) { - size_t left_node = gb->input({view.len2, C}, conv_cache_.precision); - gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); - segments.push_back(left_node); - - } - if (view.len1 > 0) { - size_t right_node = gb->input({view.len1, C}, conv_cache_.precision); - gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); - segments.push_back(right_node); - - } - if (!segments.empty()) { - conv_input_lc = segments[0]; - for (size_t idx = 1; idx < segments.size(); ++idx) { - conv_input_lc = gb->concat(conv_input_lc, segments[idx], 0); - - } - conv_input_lc = gb->concat(conv_input_lc, Bx, 0); - - } - } - - const auto& conv_input_buf = gb->get_output_buffer(conv_input_lc); - size_t total_len = conv_input_buf.shape[0]; - - size_t x_nlc = gb->reshape(conv_input_lc, {static_cast(1), total_len, C}); - const size_t dilation = 1; - - size_t y_nlc = gb->conv1d_causal(x_nlc, conv_w, K, dilation); - size_t start = total_len > L ? total_len - L : 0; - size_t y_slice = gb->slice(y_nlc, 1, start, L); - - size_t y_lc = gb->reshape(y_slice, {L, C}); - size_t gated = gb->multiply(Cg, y_lc); - size_t projected = gb->matmul(gated, layer.conv_out_proj_weight, true, backend); - return projected; -} - -size_t LFM2Model::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto q_proj_linear = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj_linear = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj_linear = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - const auto& q_shape = gb->get_output_buffer(q_proj_linear).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - - auto q_proj_reshaped = gb->reshape(q_proj_linear, {batch_seq * num_heads, head_dim}); - - auto q_proj_norm = gb->rms_norm(q_proj_reshaped, layer.attn_q_norm_weight, config_.layer_norm_eps); - - auto q_proj = gb->reshape(q_proj_norm, {batch_seq, num_heads * head_dim}); - size_t num_kv_heads = config_.attention_kv_heads; - auto k_proj_reshaped = gb->reshape(k_proj_linear, {batch_seq * num_kv_heads, head_dim}); - - auto k_proj_norm = gb->rms_norm(k_proj_reshaped, layer.attn_k_norm_weight, config_.layer_norm_eps); - - auto k_proj = gb->reshape(k_proj_norm, {batch_seq, num_kv_heads * head_dim}); - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj_linear, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); - } - size_t attn_output_4d; - - if (use_cache && layer_idx < graph_cache_k_nodes_.size() && graph_cache_k_nodes_[layer_idx] != 0) { - gb->kv_cache_append(k_proj_4d, graph_cache_k_nodes_[layer_idx], cache_window_size_, cache_sink_size_); - gb->kv_cache_append(v_proj_4d, graph_cache_v_nodes_[layer_idx], cache_window_size_, cache_sink_size_); - attn_output_4d = gb->attention_cached( - q_proj_4d, k_proj_4d, v_proj_4d, - graph_cache_k_nodes_[layer_idx], graph_cache_v_nodes_[layer_idx], - attention_scale_, position_offset, cache_window_size_); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - auto projected = gb->matmul(attn_output, layer.attn_output_weight, true, backend); - return projected; -} - -size_t LFM2Model::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, ComputeBackend backend) const { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto gate = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - auto up = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - auto activated = gb->multiply(gb->silu(gate), up); - - auto down = gb->matmul(activated, layer.ffn_down_weight, true, backend); - - return down; -} - -size_t LFM2Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - size_t block_output; - if (layer_entry.type == WeightNodeIDs::LayerType::CONV) { - block_output = build_conv1d(gb, normalized_input, layer_idx, backend, use_cache); - } else { - if (layer_idx < conv_cache_bx_nodes_.size()) { - conv_cache_bx_nodes_[layer_idx] = 0; - } - block_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - } - - auto after_block = gb->add(hidden, block_output); - auto normalized_after_block = gb->rms_norm(after_block, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_block, layer_idx, backend); - auto block_result = gb->add(after_block, mlp_output); - return block_result; -} - -size_t LFM2Model::forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, - ComputeBackend backend, bool use_cache) { - if (seq_len == 0) { - throw std::runtime_error("Sequence length must be greater than zero"); - } - - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - } - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - last_forward_used_cache_ = use_cache; - - if (!use_cache && conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } - size_t position_offset = use_cache ? cache_total_seq_len_ : 0; - - size_t hidden = input_embeddings; - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - return final_hidden; -} -size_t LFM2Model::forward(CactusGraph* gb, const std::vector& tokens, - ComputeBackend backend, bool use_cache) { - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - auto seq_len = static_cast(tokens.size()); - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - auto final_hidden = forward(gb, hidden, seq_len, backend, use_cache); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - return final_hidden; -} -size_t LFM2Model::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - return forward(gb, tokens, backend, use_cache); -} -void LFM2Model::post_execute_updates(CactusGraph* gb, size_t /*seq_len*/) { - if (conv_cache_bx_nodes_.empty()) { - return; - } - - if (!last_forward_used_cache_ || conv_cache_.window_size == 0) { - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - return; - } - size_t layer_count = std::min(conv_cache_bx_nodes_.size(), weight_nodes_.layers.size()); - for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { - if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::CONV) { - conv_cache_bx_nodes_[layer_idx] = 0; - continue; - } - size_t bx_node = conv_cache_bx_nodes_[layer_idx]; - if (bx_node != 0) { - conv_cache_.update(gb, layer_idx, bx_node); - } - conv_cache_bx_nodes_[layer_idx] = 0; - } - last_forward_used_cache_ = false; - -} -} -} diff --git a/cactus-engine/models/model_lfm2vl.cpp b/cactus-engine/models/model_lfm2vl.cpp deleted file mode 100644 index 898f6c2ba..000000000 --- a/cactus-engine/models/model_lfm2vl.cpp +++ /dev/null @@ -1,607 +0,0 @@ -#include "model.h" -#include "cactus_graph.h" -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Lfm2VlModel::Lfm2VlModel() : Model() { - config_.model_type = Config::ModelType::LFM2; -} - -Lfm2VlModel::Lfm2VlModel(const Config& config) - : Model(config), - vision_tower_(config), - language_model_(config) { - Siglip2Preprocessor::Config preprocessor_config; - preprocessor_config.patch_size = static_cast(config.vision_patch_size); - preprocessor_config.downsample_factor = static_cast(config.downsample_factor); - preprocessor_config.min_tiles = static_cast(config.min_tiles); - preprocessor_config.max_tiles = static_cast(config.max_tiles); - preprocessor_config.use_thumbnail = config.use_thumbnail; - preprocessor_config.min_image_tokens = static_cast(config.min_image_tokens); - preprocessor_config.max_image_tokens = static_cast(config.max_image_tokens); - preprocessor_config.max_num_patches = static_cast(config.max_num_patches); - preprocessor_config.tile_size = static_cast(config.tile_size); - preprocessor_config.max_pixels_tolerance = config.max_pixels_tolerance; - preprocessor_config.do_resize = true; - preprocessor_config.do_rescale = true; - preprocessor_config.do_normalize = true; - preprocessor_config.do_convert_rgb = true; - preprocessor_config.do_image_splitting = config.do_image_splitting; - preprocessor_config.rescale_factor = config.rescale_factor; - preprocessor_config.image_mean[0] = config.image_mean; - preprocessor_config.image_mean[1] = config.image_mean; - preprocessor_config.image_mean[2] = config.image_mean; - preprocessor_config.image_std[0] = config.image_std; - preprocessor_config.image_std[1] = config.image_std; - preprocessor_config.image_std[2] = config.image_std; - - preprocessor_ = Siglip2Preprocessor(preprocessor_config); -} - -bool Lfm2VlModel::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - - if (!Model::init(model_folder, context_size, system_prompt, false)) { - return false; - } - auto* shared_graph = static_cast(graph_handle_); - if (!shared_graph) { - throw std::runtime_error("Shared graph was not initialized for Lfm2VlModel"); - } - std::string vision_folder = model_folder; - if (!vision_tower_.init(shared_graph, vision_folder, context_size, "", false)) { - throw std::runtime_error("Failed to initialize vision tower"); - } - - vision_weights_loaded_ = true; - if (!language_model_.init(shared_graph, model_folder, context_size, system_prompt, false)) { - throw std::runtime_error("Failed to initialize language model"); - } - - language_model_.output_weight_node_id_ = output_weight_node_id_; - - language_weights_loaded_ = true; - - if (do_warmup) { - std::string warmup_text = system_prompt.empty() ? "Hello" : system_prompt; - auto warmup_tokens = tokenizer_->encode(warmup_text); - language_model_.decode(warmup_tokens, config_.default_temperature, config_.default_top_p, config_.default_top_k, ""); - language_model_.reset_cache(); - } - - return true; -} - -void Lfm2VlModel::reset_cache() { - Model::reset_cache(); - language_model_.reset_cache(); - image_prefill_completed_ = false; - last_token_count_ = 0; -} - -void Lfm2VlModel::load_weights_to_graph(CactusGraph* gb) { - namespace fs = std::filesystem; - fs::path base(model_folder_path_); - - auto resolve_weight = [&](const std::string& primary, const std::string& fallback = "") -> std::string { - fs::path primary_path = base / primary; - if (fs::exists(primary_path)) { - return primary_path.string(); - } - if (!fallback.empty()) { - fs::path fallback_path = base / fallback; - if (fs::exists(fallback_path)) { - return fallback_path.string(); - } - } - return primary_path.string(); - }; - - try { - projector_weights_.layer_norm_weight = gb->mmap_weights(resolve_weight("projector_layer_norm.weights")); - projector_weights_.layer_norm_bias = gb->mmap_weights(resolve_weight("projector_layer_norm.bias.weights")); - } catch (const std::exception&) { - projector_weights_.layer_norm_weight = 0; - projector_weights_.layer_norm_bias = 0; - } - - projector_weights_.linear_1_weight = gb->mmap_weights(resolve_weight("projector_linear_1.weights", "projector_linear1.weights")); - projector_weights_.linear_1_bias = gb->mmap_weights(resolve_weight("projector_linear_1.bias.weights", "projector_linear1.bias.weights")); - projector_weights_.linear_2_weight = gb->mmap_weights(resolve_weight("projector_linear_2.weights", "projector_linear2.weights")); - projector_weights_.linear_2_bias = gb->mmap_weights(resolve_weight("projector_linear_2.bias.weights", "projector_linear2.bias.weights")); - output_weight_node_id_ = gb->mmap_weights(resolve_weight("token_embeddings.weights")); -} - -size_t Lfm2VlModel::pixel_unshuffle(CactusGraph* gb, size_t hidden_states, - size_t height, size_t width, size_t channels) { - - const size_t factor = config_.downsample_factor; - const size_t new_height = height / factor; - const size_t new_width = width / factor; - size_t step1 = gb->reshape(hidden_states, {1, height, new_width, channels * factor}); - step1 = gb->transposeN(step1, {0, 2, 1, 3}); - size_t step2 = gb->reshape(step1, {1, new_width, new_height, channels * factor * factor}); - size_t result = gb->transposeN(step2, {0, 2, 1, 3}); - return result; -} - -size_t Lfm2VlModel::build_multimodal_projector(CactusGraph* gb, size_t image_features, - size_t tile_h, size_t tile_w, ComputeBackend backend) { - const size_t vision_hidden = config_.vision_embed_dim; - - const auto& input_buf = gb->get_output_buffer(image_features); - size_t image_features_fp16 = (input_buf.precision == Precision::FP16) - ? image_features - : gb->precision_cast(image_features, Precision::FP16); - - size_t unshuffled = pixel_unshuffle(gb, image_features_fp16, tile_h, tile_w, vision_hidden); - const size_t factor = config_.downsample_factor; - const size_t new_h = tile_h / factor; - const size_t new_w = tile_w / factor; - const size_t in_channels = vision_hidden * factor * factor; - const size_t seq_len = new_h * new_w; - size_t flattened = gb->reshape(unshuffled, {seq_len, in_channels}); - size_t normalized = flattened; - if (projector_weights_.layer_norm_weight != 0 && projector_weights_.layer_norm_bias != 0) { - normalized = gb->layernorm(flattened, projector_weights_.layer_norm_weight, - projector_weights_.layer_norm_bias, config_.layer_norm_eps); - } - size_t hidden = gb->matmul(normalized, projector_weights_.linear_1_weight, true, backend); - hidden = gb->add(hidden, projector_weights_.linear_1_bias); - hidden = gb->gelu(hidden); - size_t output = gb->matmul(hidden, projector_weights_.linear_2_weight, true, backend); - output = gb->add(output, projector_weights_.linear_2_bias); - return output; -} - -std::vector Lfm2VlModel::get_image_embeddings(const std::string& image_path) { - return vision_tower_.get_image_embedding(image_path); -} - -std::vector Lfm2VlModel::get_image_features( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend) { - - size_t vision_output = vision_tower_.forward_vision(gb, preprocessed_image, backend); - std::vector projected_features; - projected_features.reserve(preprocessed_image.spatial_shapes.size()); - - size_t offset = 0; - for (size_t tile_idx = 0; tile_idx < preprocessed_image.spatial_shapes.size(); ++tile_idx) { - const auto& shape = preprocessed_image.spatial_shapes[tile_idx]; - const size_t tile_h = shape.first; - const size_t tile_w = shape.second; - const size_t tile_tokens = tile_h * tile_w; - const size_t factor = config_.downsample_factor; - if (factor == 0) { - throw std::runtime_error("Downsample factor must be greater than zero"); - } - if (tile_h % factor != 0 || tile_w % factor != 0) { - throw std::runtime_error("Tile dimensions must be divisible by downsample factor"); - } - const size_t new_h = tile_h / factor; - const size_t new_w = tile_w / factor; - const size_t projected_tokens = new_h * new_w; - - size_t tile_features = gb->slice(vision_output, 0, offset, tile_tokens); - offset += tile_tokens; - - size_t reshaped = gb->reshape(tile_features, {1, tile_h, tile_w, config_.vision_embed_dim}); - size_t projected = build_multimodal_projector(gb, reshaped, tile_h, tile_w, backend); - ProjectedTileFeature feature{}; - feature.node_id = projected; - feature.token_count = projected_tokens; - projected_features.push_back(feature); - - } - - return projected_features; -} - -Lfm2VlModel::MergedEmbeddingResult Lfm2VlModel::merge_image_text_embeddings( - CactusGraph* gb, - const std::vector& tokens, - const std::vector>& image_embedding_nodes, - std::vector& text_embedding_inputs) { - - text_embedding_inputs.clear(); - - Tokenizer* tokenizer = language_model_.get_tokenizer(); - if (!tokenizer) { - throw std::runtime_error("Tokenizer must be initialized before merging embeddings"); - } - - const uint32_t image_token_id = tokenizer->get_image_token_id(); - - auto get_token_id = [tokenizer](const std::string& token_text) -> uint32_t { - auto encoded = tokenizer->encode(token_text); - if (encoded.size() != 1) { - - throw std::runtime_error("Expected single token encoding for " + token_text); - } - return encoded[0]; - }; - - const uint32_t image_start_id = get_token_id("<|image_start|>"); - const uint32_t image_end_id = get_token_id("<|image_end|>"); - - std::vector sequence_nodes; - sequence_nodes.reserve(tokens.size() + image_embedding_nodes.size()); - - std::vector current_segment; - current_segment.reserve(tokens.size()); - - size_t total_seq_len = 0; - - auto flush_segment = [&](void) { - if (current_segment.empty()) { - - return; - } - - const size_t segment_len = current_segment.size(); - TextEmbeddingInput segment; - segment.tokens.swap(current_segment); - segment.input_node = gb->input({segment.tokens.size()}, Precision::FP32); - - const auto& embedding_buffer = gb->get_output_buffer(language_model_.embedding_node_id_); - - for (size_t i = 0; i < embedding_buffer.shape.size(); ++i) { - (void)i; - } - - size_t embedding_node = gb->embedding(language_model_.embedding_node_id_, segment.input_node); - - text_embedding_inputs.push_back(std::move(segment)); - - sequence_nodes.push_back(embedding_node); - - total_seq_len += segment_len; - - current_segment.clear(); - - }; - - size_t token_index = 0; - size_t image_index = 0; - - while (token_index < tokens.size()) { - uint32_t token_id = tokens[token_index]; - - if (token_id == image_start_id) { - flush_segment(); - - if (image_index >= image_embedding_nodes.size()) { - - throw std::runtime_error("Encountered <|image_start|> without corresponding image features"); - } - - current_segment.push_back(token_id); - ++token_index; - - const auto& tiles = image_embedding_nodes[image_index]; - size_t tile_index = 0; - while (token_index < tokens.size()) { - uint32_t inner_token = tokens[token_index]; - if (inner_token == image_token_id) { - flush_segment(); - - if (tile_index >= tiles.size()) { - - throw std::runtime_error("More placeholders than projected tile features"); - } - - const auto& tile = tiles[tile_index++]; - sequence_nodes.push_back(tile.node_id); - - total_seq_len += tile.token_count; - - for (size_t count = 0; count < tile.token_count; ++count) { - if (token_index >= tokens.size()) { - throw std::runtime_error("Insufficient tokens for projected features"); - } - if (tokens[token_index] != image_token_id) { - throw std::runtime_error("Unexpected token encountered within image feature span"); - } - ++token_index; - - } - continue; - } - - current_segment.push_back(inner_token); - - ++token_index; - - if (inner_token == image_end_id) { - flush_segment(); - break; - } - } - - if (tile_index != tiles.size()) { - if (tile_index < tiles.size()) { - for (size_t remaining = tile_index; remaining < tiles.size(); ++remaining) { - (void)tiles[remaining]; - } - } - throw std::runtime_error("Unused projected tile features remain after processing image block"); - } - - ++image_index; - - } else { - current_segment.push_back(token_id); - - ++token_index; - - } - } - - flush_segment(); - if (image_index != image_embedding_nodes.size()) { - throw std::runtime_error("Not all image features were consumed while merging embeddings"); - } - - if (sequence_nodes.empty()) { - throw std::runtime_error("Failed to build embedding sequence from provided tokens"); - } - - size_t merged = sequence_nodes[0]; - for (size_t idx = 1; idx < sequence_nodes.size(); ++idx) { - merged = gb->concat(merged, sequence_nodes[idx], 0); - - } - - return MergedEmbeddingResult{merged, total_seq_len}; -} - -size_t Lfm2VlModel::build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_attention should not be called directly on Lfm2VlModel"); -} - -size_t Lfm2VlModel::build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const { - throw std::runtime_error("build_mlp should not be called directly on Lfm2VlModel"); -} - -size_t Lfm2VlModel::build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_transformer_block should not be called directly on Lfm2VlModel"); -} - -size_t Lfm2VlModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - size_t final_hidden = language_model_.forward(gb, tokens, backend, use_cache); - - return final_hidden; -} - -uint32_t Lfm2VlModel::decode(const std::vector& tokens, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (temperature < 0) { - temperature = config_.default_temperature; - } - if (top_p < 0) { - top_p = config_.default_top_p; - } - if (top_k == 0) { - top_k = config_.default_top_k; - } - - image_prefill_completed_ = false; - last_token_count_ = tokens.size(); - - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -void Lfm2VlModel::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - image_prefill_completed_ = false; - last_token_count_ = tokens.size(); - - language_model_.prefill(tokens, chunk_size, profile_file); -} - -void Lfm2VlModel::prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (tokens.empty()) { - return; - } - if (image_paths.empty()) { - prefill(tokens, get_prefill_chunk_size(), profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto forward_result = forward_images(gb, tokens, image_paths, backend, true); - gb->execute(profile_file); - - language_model_.post_execute_updates(gb, forward_result.seq_len); - language_model_.update_kv_cache(gb, forward_result.seq_len); - - image_prefill_completed_ = true; - last_token_count_ = tokens.size(); -} - -Lfm2VlModel::ForwardImageResult Lfm2VlModel::forward_images( - CactusGraph* gb, - const std::vector& tokens, - const std::vector& image_paths, - ComputeBackend backend, - bool use_cache) { - if (!gb) { - throw std::runtime_error("Graph must be initialized before forwarding"); - } - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - std::vector> all_image_embeddings; - all_image_embeddings.reserve(image_paths.size()); - for (const auto& image_path : image_paths) { - auto preprocessed = preprocessor_.preprocess_from_file(image_path); - - auto image_features = get_image_features(gb, preprocessed, backend); - - all_image_embeddings.push_back(std::move(image_features)); - } - - std::vector text_embedding_inputs; - text_embedding_inputs.reserve(tokens.size() / 4 + 1); - - auto merged_embeddings = merge_image_text_embeddings(gb, tokens, all_image_embeddings, text_embedding_inputs); - if (merged_embeddings.seq_len == 0) { - throw std::runtime_error("Merged embedding sequence length cannot be zero"); - } - - for (const auto& embedding_input : text_embedding_inputs) { - if (embedding_input.tokens.empty()) { - continue; - } - - std::vector segment_data(embedding_input.tokens.size()); - for (size_t i = 0; i < embedding_input.tokens.size(); ++i) { - segment_data[i] = static_cast(embedding_input.tokens[i]); - } - gb->set_input(embedding_input.input_node, segment_data.data(), Precision::FP32); - } - size_t final_hidden = language_model_.forward(gb, merged_embeddings.node_id, merged_embeddings.seq_len, backend, use_cache); - return ForwardImageResult{final_hidden, merged_embeddings.seq_len}; -} - -uint32_t Lfm2VlModel::decode_with_images( - const std::vector& tokens, - const std::vector& image_paths, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty) { - - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (image_paths.empty()) { - - image_prefill_completed_ = false; - last_token_count_ = tokens.size(); - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); - } - - if (temperature < 0) { - temperature = config_.default_temperature; - } - if (top_p < 0) { - top_p = config_.default_top_p; - } - if (top_k == 0) { - top_k = config_.default_top_k; - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - bool cache_empty = language_model_.is_cache_empty(); - bool need_prefill = cache_empty || !image_prefill_completed_; - - if (!need_prefill && tokens.size() <= last_token_count_) { - - reset_cache(); - need_prefill = true; - } - - size_t seq_len_for_updates = 0; - size_t final_hidden_node = 0; - - if (need_prefill) { - auto forward_result = forward_images(gb, tokens, image_paths, backend, true); - - final_hidden_node = forward_result.final_hidden_node; - seq_len_for_updates = forward_result.seq_len; - image_prefill_completed_ = true; - last_token_count_ = tokens.size(); - } else { - size_t delta = tokens.size() - last_token_count_; - if (delta > tokens.size()) { - delta = tokens.size(); - } - if (delta == 0) { - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty for cached decode step"); - } - delta = 1; - - } - std::vector incremental_tokens(tokens.end() - delta, tokens.end()); - - final_hidden_node = language_model_.forward(gb, incremental_tokens, backend, true); - - seq_len_for_updates = incremental_tokens.size(); - last_token_count_ = tokens.size(); - } - - auto logits_node_id = gb->matmul(final_hidden_node, language_model_.output_weight_node_id_, true, backend); - size_t sampled_token_id = - language_model_.sample_token(gb, logits_node_id, temperature, top_p, top_k, min_p, repetition_penalty, nullptr); - if (!profile_file.empty()) { - gb->execute(profile_file); - - } else { - gb->execute(); - - } - - compute_entropy(gb, logits_node_id, out_entropy); - - language_model_.post_execute_updates(gb, seq_len_for_updates); - language_model_.update_kv_cache(gb, seq_len_for_updates); - - auto* output_ptr = gb->get_output(sampled_token_id); - uint32_t result = *static_cast(output_ptr); - language_model_.record_sampled_token(result); - return result; -} - -} -} diff --git a/cactus-engine/models/model_qwen.cpp b/cactus-engine/models/model_qwen.cpp deleted file mode 100644 index d21fb19f1..000000000 --- a/cactus-engine/models/model_qwen.cpp +++ /dev/null @@ -1,162 +0,0 @@ -#include "model.h" -#include "cactus_graph.h" -#include -#include - - -namespace cactus { -namespace engine { - -QwenModel::QwenModel() : Model() {} - -QwenModel::QwenModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -void QwenModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - } - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - load_npu_prefill(npu_prefill_path); - } -} - -size_t QwenModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); - q_proj = gb->rms_norm(q_proj, layer.attn_q_norm_weight, config_.layer_norm_eps); - q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); - - size_t num_kv_heads = config_.attention_kv_heads; - k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); - k_proj = gb->rms_norm(k_proj, layer.attn_k_norm_weight, config_.layer_norm_eps); - k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); - - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); - } - - size_t attn_output_4d; - - if (use_cache && layer_idx < graph_cache_k_nodes_.size() && graph_cache_k_nodes_[layer_idx] != 0) { - gb->kv_cache_append(k_proj_4d, graph_cache_k_nodes_[layer_idx], cache_window_size_, cache_sink_size_); - gb->kv_cache_append(v_proj_4d, graph_cache_v_nodes_[layer_idx], cache_window_size_, cache_sink_size_); - attn_output_4d = gb->attention_cached( - q_proj_4d, k_proj_4d, v_proj_4d, - graph_cache_k_nodes_[layer_idx], graph_cache_v_nodes_[layer_idx], - attention_scale_, position_offset, cache_window_size_); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - return gb->matmul(attn_output, layer.attn_output_weight, true, backend); -} - - -size_t QwenModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - size_t gate_silu = gb->silu(gate_output); - size_t gated = gb->multiply(gate_silu, up_output); - return gb->matmul(gated, layer.ffn_down_weight, true, backend); -} - - -size_t QwenModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - auto after_attention = gb->add(hidden, attn_output); - auto normalized_after_attention = gb->rms_norm(after_attention, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); - return gb->add(after_attention, mlp_output); -} - - -size_t QwenModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto seq_len = static_cast(tokens.size()); - - size_t position_offset = use_cache ? cache_total_seq_len_ : 0; - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - return final_hidden; -} - -} -} diff --git a/cactus-engine/models/model_siglip2.cpp b/cactus-engine/models/model_siglip2.cpp deleted file mode 100644 index 9eca06752..000000000 --- a/cactus-engine/models/model_siglip2.cpp +++ /dev/null @@ -1,442 +0,0 @@ -#include "model.h" -#include "cactus_graph.h" -#include "cactus_kernels.h" -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -namespace { - -size_t infer_npu_patch_capacity(const std::vector& input_shape, size_t hidden_dim) { - if (input_shape.size() < 2) { - return 0; - } - - const int first = input_shape[0]; - const int second = input_shape[1]; - if (first <= 0 || second <= 0) { - return 0; - } - - if (static_cast(second) == hidden_dim) { - return static_cast(first); - } - if (static_cast(first) == hidden_dim) { - return static_cast(second); - } - - return static_cast(first); -} - -} // namespace - -Siglip2VisionModel::Siglip2VisionModel() : Model() { - config_.model_type = Config::ModelType::SIGLIP2; -} - -Siglip2VisionModel::Siglip2VisionModel(const Config& cfg) : Model(cfg) { - Siglip2Preprocessor::Config preprocessor_config; - preprocessor_config.patch_size = static_cast(config_.vision_patch_size); - preprocessor_config.downsample_factor = static_cast(config_.downsample_factor); - preprocessor_config.min_tiles = static_cast(config_.min_tiles); - preprocessor_config.max_tiles = static_cast(config_.max_tiles); - preprocessor_config.use_thumbnail = config_.use_thumbnail; - preprocessor_config.min_image_tokens = static_cast(config_.min_image_tokens); - preprocessor_config.max_image_tokens = static_cast(config_.max_image_tokens); - preprocessor_config.max_num_patches = static_cast(config_.max_num_patches); - preprocessor_config.tile_size = static_cast(config_.tile_size); - preprocessor_config.max_pixels_tolerance = config_.max_pixels_tolerance; - preprocessor_config.do_resize = true; - preprocessor_config.do_rescale = true; - preprocessor_config.do_normalize = true; - preprocessor_config.do_convert_rgb = true; - preprocessor_config.do_image_splitting = config_.do_image_splitting; - preprocessor_config.rescale_factor = config_.rescale_factor; - preprocessor_config.image_mean[0] = config_.image_mean; - preprocessor_config.image_mean[1] = config_.image_mean; - preprocessor_config.image_mean[2] = config_.image_mean; - preprocessor_config.image_std[0] = config_.image_std; - preprocessor_config.image_std[1] = config_.image_std; - preprocessor_config.image_std[2] = config_.image_std; - - preprocessor_ = Siglip2Preprocessor(preprocessor_config); -} - -void Siglip2VisionModel::ensure_cpu_vision_weights_loaded(CactusGraph* gb) { - if (cpu_vision_weights_loaded_) { - return; - } - - std::string base = model_folder_path_ + "/"; - vision_weight_nodes_.vision_layers.resize(config_.vision_num_layers); - - vision_weight_nodes_.post_layernorm_weight = gb->mmap_weights(base + "vision_post_layernorm.weights"); - vision_weight_nodes_.post_layernorm_bias = gb->mmap_weights(base + "vision_post_layernorm.bias.weights"); - - for (uint32_t i = 0; i < vision_weight_nodes_.vision_layers.size(); ++i) { - auto& layer = vision_weight_nodes_.vision_layers[i]; - std::string prefix = base + "vision_layer_" + std::to_string(i) + "_"; - - layer.attn_q_weight = gb->mmap_weights(prefix + "self_attn_q.weights"); - layer.attn_q_bias = gb->mmap_weights(prefix + "self_attn_q.bias.weights"); - layer.attn_k_weight = gb->mmap_weights(prefix + "self_attn_k.weights"); - layer.attn_k_bias = gb->mmap_weights(prefix + "self_attn_k.bias.weights"); - layer.attn_v_weight = gb->mmap_weights(prefix + "self_attn_v.weights"); - layer.attn_v_bias = gb->mmap_weights(prefix + "self_attn_v.bias.weights"); - layer.attn_output_weight = gb->mmap_weights(prefix + "self_attn_out.weights"); - layer.attn_output_bias = gb->mmap_weights(prefix + "self_attn_out.bias.weights"); - - layer.layer_norm1_weight = gb->mmap_weights(prefix + "layer_norm1.weights"); - layer.layer_norm1_bias = gb->mmap_weights(prefix + "layer_norm1.bias.weights"); - layer.layer_norm2_weight = gb->mmap_weights(prefix + "layer_norm2.weights"); - layer.layer_norm2_bias = gb->mmap_weights(prefix + "layer_norm2.bias.weights"); - - layer.mlp_fc1_weight = gb->mmap_weights(prefix + "ffn_fc1.weights"); - layer.mlp_fc1_bias = gb->mmap_weights(prefix + "ffn_fc1.bias.weights"); - layer.mlp_fc2_weight = gb->mmap_weights(prefix + "ffn_fc2.weights"); - layer.mlp_fc2_bias = gb->mmap_weights(prefix + "ffn_fc2.bias.weights"); - } - - cpu_vision_weights_loaded_ = true; -} - -void Siglip2VisionModel::load_weights_to_graph(CactusGraph* gb) { - std::string base = model_folder_path_ + "/"; - - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - - std::vector input_shape = npu_encoder_->get_input_shape(); - if (input_shape.empty()) { - input_shape = { - static_cast(config_.max_num_patches), - static_cast(config_.vision_embed_dim) - }; - } - npu_encoder_->preallocate(input_shape, "x", ""); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - - // Always load patch embedding and position embedding weights (needed for both CPU and NPU paths) - vision_weight_nodes_.patch_embedding_weight = gb->mmap_weights(base + "vision_patch_embedding.weights"); - vision_weight_nodes_.patch_embedding_bias = gb->mmap_weights(base + "vision_patch_embedding.bias.weights"); - vision_weight_nodes_.position_embedding = gb->mmap_weights(base + "vision_position_embedding.weights"); - - if (!use_npu_encoder_) { - ensure_cpu_vision_weights_loaded(gb); - } -} - -Siglip2VisionModel::VisionEmbeddingResult Siglip2VisionModel::build_vision_embeddings( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend) { - const int num_tiles = preprocessed_image.num_tiles; - const int max_patches = preprocessed_image.max_patches_per_tile; - const int patch_dim = preprocessed_image.patch_dim; - - const size_t expected_size = static_cast(num_tiles) * static_cast(max_patches) * - static_cast(patch_dim); - if (preprocessed_image.pixel_values.size() != expected_size) { - throw std::runtime_error( - "Pixel values size mismatch: expected " + std::to_string(expected_size) + - " (tiles=" + std::to_string(num_tiles) + " * max_patches=" + std::to_string(max_patches) + - " * patch_dim=" + std::to_string(patch_dim) + ") but got " + - std::to_string(preprocessed_image.pixel_values.size())); - } - for (size_t i = 0; i < std::min(100, preprocessed_image.pixel_values.size()); ++i) { - float val = preprocessed_image.pixel_values[i]; - if (std::isnan(val) || std::isinf(val)) { - throw std::runtime_error( - "Invalid value in pixel_values at index " + std::to_string(i) + ": " + std::to_string(val)); - } - } - - size_t reshaped_weight = gb->reshape( - vision_weight_nodes_.patch_embedding_weight, - {static_cast(config_.vision_embed_dim), static_cast(patch_dim)}); - - size_t patch_bias = vision_weight_nodes_.patch_embedding_bias; - std::vector tile_embeddings; - tile_embeddings.reserve(static_cast(num_tiles)); - - for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { - const auto& shape = preprocessed_image.spatial_shapes[tile_idx]; - const int tile_h = shape.first; - const int tile_w = shape.second; - const int actual_patches = tile_h * tile_w; - if (actual_patches <= 0) { - continue; - } - - const float* tile_data = preprocessed_image.pixel_values.data() + - static_cast(tile_idx) * static_cast(max_patches) * - static_cast(patch_dim); - - size_t tile_input_fp32 = gb->input( - {static_cast(actual_patches), static_cast(patch_dim)}, Precision::FP32); - gb->set_input(tile_input_fp32, tile_data, Precision::FP32); - size_t tile_input = gb->precision_cast(tile_input_fp32, Precision::FP16); - size_t tile_patch = gb->matmul(tile_input, reshaped_weight, true, backend); - size_t tile_bias = gb->add(tile_patch, patch_bias); - size_t tile_pos = gb->bilinear_interpolation( - vision_weight_nodes_.position_embedding, - static_cast(tile_h), - static_cast(tile_w), - /*align_corners=*/false); - size_t tile_pos_cast = gb->precision_cast(tile_pos, Precision::FP16); - size_t tile_embed = gb->add(tile_bias, tile_pos_cast); - tile_embeddings.push_back(tile_embed); - } - - if (tile_embeddings.empty()) { - throw std::runtime_error("No valid tiles produced embeddings in build_vision_embeddings"); - } - auto concat_nodes = [&](const std::vector& nodes) { - if (nodes.empty()) { - throw std::runtime_error("Attempted to concatenate an empty node list"); - } - size_t combined = nodes.front(); - for (size_t i = 1; i < nodes.size(); ++i) { - combined = gb->concat(combined, nodes[i], /*axis=*/0); - } - return combined; - }; - - size_t embeddings = concat_nodes(tile_embeddings); - return VisionEmbeddingResult{embeddings, std::move(tile_embeddings)}; -} - -size_t Siglip2VisionModel::build_vision_attention(CactusGraph* gb, size_t hidden_states, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; - - size_t q = gb->matmul(hidden_states, layer.attn_q_weight, true, backend); - q = gb->add(q, layer.attn_q_bias); - size_t k = gb->matmul(hidden_states, layer.attn_k_weight, true, backend); - k = gb->add(k, layer.attn_k_bias); - size_t v = gb->matmul(hidden_states, layer.attn_v_weight, true, backend); - v = gb->add(v, layer.attn_v_bias); - const size_t num_heads = static_cast(config_.vision_attention_heads); - const size_t head_dim = static_cast(config_.vision_embed_dim / config_.vision_attention_heads); - const auto& q_buf = gb->get_output_buffer(q); - size_t seq_len = q_buf.shape[0]; - - size_t q_4d = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - size_t k_4d = gb->reshape(k, {1, seq_len, num_heads, head_dim}); - size_t v_4d = gb->reshape(v, {1, seq_len, num_heads, head_dim}); - - float scale = 1.0f / std::sqrt(static_cast(head_dim)); - size_t attn_output = gb->attention(q_4d, k_4d, v_4d, scale, false, backend); - size_t attn_2d = gb->reshape(attn_output, {seq_len, num_heads * head_dim}); - size_t output = gb->matmul(attn_2d, layer.attn_output_weight, true, backend); - output = gb->add(output, layer.attn_output_bias); - return output; -} - -size_t Siglip2VisionModel::build_vision_mlp(CactusGraph* gb, size_t hidden_states, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; - - size_t fc1_output = gb->matmul(hidden_states, layer.mlp_fc1_weight, true, backend); - fc1_output = gb->add(fc1_output, layer.mlp_fc1_bias); - size_t activated = gb->gelu(fc1_output); - size_t fc2_output = gb->matmul(activated, layer.mlp_fc2_weight, true, backend); - fc2_output = gb->add(fc2_output, layer.mlp_fc2_bias); - return fc2_output; -} - -size_t Siglip2VisionModel::build_vision_transformer_layer(CactusGraph* gb, size_t hidden_states, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; - - size_t residual = hidden_states; - size_t normalized = gb->layernorm(hidden_states, layer.layer_norm1_weight, - layer.layer_norm1_bias, config_.layer_norm_eps); - size_t attn_output = build_vision_attention(gb, normalized, layer_idx, backend); - hidden_states = gb->add(residual, attn_output); - residual = hidden_states; - normalized = gb->layernorm(hidden_states, layer.layer_norm2_weight, - layer.layer_norm2_bias, config_.layer_norm_eps); - size_t mlp_output = build_vision_mlp(gb, normalized, layer_idx, backend); - hidden_states = gb->add(residual, mlp_output); - return hidden_states; -} - -size_t Siglip2VisionModel::forward_vision( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend) { - - bool can_use_npu_encoder = use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available(); - size_t total_patches = 0; - for (const auto& shape : preprocessed_image.spatial_shapes) { - total_patches += shape.first * shape.second; - } - - if (can_use_npu_encoder) { - size_t max_npu_patches = - infer_npu_patch_capacity(npu_encoder_->get_input_shape(), config_.vision_embed_dim); - if (max_npu_patches > 0 && total_patches > max_npu_patches) { - CACTUS_LOG_WARN("npu", - "[siglip2-vision] image has " - << total_patches << " patches but the NPU encoder supports " - << max_npu_patches << "; falling back to CPU vision encoder"); - can_use_npu_encoder = false; - } - } - - if (can_use_npu_encoder) { - // NPU path: build patch embeddings + position embeddings on CPU, then run transformer on NPU - auto embedding_result = build_vision_embeddings(gb, preprocessed_image, backend); - - gb->execute(); - - const auto& embed_buffer = gb->get_output_buffer(embedding_result.combined_embeddings); - void* embed_ptr = gb->get_output(embedding_result.combined_embeddings); - - std::vector<__fp16> embed_f16; - const __fp16* input_ptr; - if (embed_buffer.precision == Precision::FP16) { - input_ptr = static_cast(embed_ptr); - } else { - embed_f16.resize(total_patches * config_.vision_embed_dim); - cactus_fp32_to_fp16(static_cast(embed_ptr), - embed_f16.data(), embed_f16.size()); - input_ptr = embed_f16.data(); - } - - std::vector input_shape = { - static_cast(total_patches), - static_cast(config_.vision_embed_dim) - }; - - __fp16* output_buffer = npu_encoder_->get_output_buffer(); - if (output_buffer) { - size_t elements = npu_encoder_->encode( - input_ptr, output_buffer, input_shape, "x", ""); - - if (elements > 0) { - gb->soft_reset(); - size_t vision_output = gb->input({total_patches, config_.vision_embed_dim}, - Precision::FP16); - gb->set_external_input(vision_output, output_buffer, Precision::FP16); - return vision_output; - } - } else { - std::vector<__fp16> npu_output(total_patches * config_.vision_embed_dim); - size_t elements = npu_encoder_->encode( - input_ptr, npu_output.data(), input_shape, "x", ""); - - if (elements > 0) { - gb->soft_reset(); - size_t vision_output = gb->input({total_patches, config_.vision_embed_dim}, - Precision::FP16); - gb->set_input(vision_output, npu_output.data(), Precision::FP16); - return vision_output; - } - } - - CACTUS_LOG_WARN("npu", "[siglip2-vision] NPU encoder failed; falling back to CPU vision encoder"); - gb->soft_reset(); - } - - ensure_cpu_vision_weights_loaded(gb); - - // CPU path: full forward pass through transformer layers - auto embedding_result = build_vision_embeddings(gb, preprocessed_image, backend); - - auto concat_nodes = [&](const std::vector& nodes) { - if (nodes.empty()) { - throw std::runtime_error("Attempted to concatenate an empty node list in forward_vision"); - } - size_t combined = nodes.front(); - for (size_t i = 1; i < nodes.size(); ++i) { - combined = gb->concat(combined, nodes[i], /*axis=*/0); - } - return combined; - }; - - std::vector tile_outputs; - tile_outputs.reserve(embedding_result.tile_embeddings.size()); - - for (size_t tile_idx = 0; tile_idx < embedding_result.tile_embeddings.size(); ++tile_idx) { - size_t hidden_states = embedding_result.tile_embeddings[tile_idx]; - - for (uint32_t layer_idx = 0; layer_idx < config_.vision_num_layers; ++layer_idx) { - hidden_states = build_vision_transformer_layer(gb, hidden_states, layer_idx, backend); - } - - hidden_states = gb->layernorm(hidden_states, - vision_weight_nodes_.post_layernorm_weight, - vision_weight_nodes_.post_layernorm_bias, - config_.layer_norm_eps); - tile_outputs.push_back(hidden_states); - } - - if (tile_outputs.empty()) { - throw std::runtime_error("No tile outputs generated in forward_vision"); - } - - size_t combined_output = concat_nodes(tile_outputs); - - return combined_output; -} - -size_t Siglip2VisionModel::forward_vision(const Siglip2Preprocessor::PreprocessedImage& preprocessed_image) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - return forward_vision(gb, preprocessed_image, backend); -} - -std::vector Siglip2VisionModel::get_image_embedding(const std::string& image_path) { - auto preprocessed = preprocessor_.preprocess_from_file(image_path); - size_t last_hidden_state = forward_vision(preprocessed); - - auto* gb = static_cast(graph_handle_); - size_t pooled = gb->mean(last_hidden_state, 0); - gb->execute(); - - const auto& output_buf = gb->get_output_buffer(pooled); - size_t hidden_dim = output_buf.total_size; - - std::vector embedding(hidden_dim); - void* output_data = gb->get_output(pooled); - const float* output_ptr = static_cast(output_data); - std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); - return embedding; -} - -size_t Siglip2VisionModel::forward(const std::vector&, bool) {return 0;} - -size_t Siglip2VisionModel::build_attention(CactusGraph*, size_t, uint32_t, - ComputeBackend, bool, size_t) { - return 0; -} - -size_t Siglip2VisionModel::build_mlp(CactusGraph*, size_t, uint32_t, - ComputeBackend) const { - return 0; -} - -size_t Siglip2VisionModel::build_transformer_block(CactusGraph*, size_t, uint32_t, - ComputeBackend, bool, size_t) { - return 0; -} - -} -} diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index c79bd794a..a9ab196b5 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -660,18 +660,11 @@ int cactus_complete( bool has_audio = prompt.has_audio(); const bool has_gemma4_mixed_media = prompt.model_type == Config::ModelType::GEMMA4 && has_images && has_audio; auto decode_gemma4_mixed_media = [&](const std::vector& tokens, float* out_entropy) -> uint32_t { - auto* gemma4_mm = dynamic_cast(handle->model.get()); - if (!gemma4_mm) { - throw std::runtime_error("Gemma4 mixed-media decode requested on non-Gemma4 multimodal model"); - } - return gemma4_mm->decode_with_media( - tokens, - prompt.image_paths, - prompt.audio_features, - prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, - "", out_entropy, - prompt.options.min_p, prompt.options.repetition_penalty - ); + (void)tokens; + (void)out_entropy; + throw std::runtime_error( + "Gemma4 mixed-media native decode is not present in this build. " + "Use cactus run-transpiled for transpiled graph bundles."); }; auto stop_token_sequences = build_stop_sequences(tokenizer, prompt.options.stop_sequences, prompt.model_type, !prompt.tools.empty()); diff --git a/cactus-engine/src/graph_ffi.cpp b/cactus-engine/src/graph_ffi.cpp index 8e61d5fe0..6de9f17fb 100644 --- a/cactus-engine/src/graph_ffi.cpp +++ b/cactus-engine/src/graph_ffi.cpp @@ -515,6 +515,17 @@ int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus } } +int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_cumsum"); + try { + *out = static_cast(as_graph(graph)->graph.cumsum(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + int cactus_graph_concat(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, cactus_node_t* out) { if (!graph || !out) { last_error_message = "Invalid args to cactus_graph_concat"; @@ -614,6 +625,17 @@ int cactus_graph_mmap_weights(cactus_graph_t graph, const char* filename, cactus } } +int cactus_graph_bind_mmap_weights(cactus_graph_t graph, cactus_node_t node, const char* filename) { + if (!graph || !filename) return fail_invalid("Invalid args to cactus_graph_bind_mmap_weights"); + try { + as_graph(graph)->graph.bind_mmap_weights(static_cast(node), std::string(filename)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + int cactus_graph_bilinear_interpolation(cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out) { if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_bilinear_interpolation"); try { @@ -1081,6 +1103,17 @@ int cactus_graph_conv1d_pointwise(cactus_graph_t graph, cactus_node_t input, cac } } +int cactus_graph_clamp(cactus_graph_t graph, cactus_node_t input, float lo, float hi, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_clamp"); + try { + *out = static_cast(as_graph(graph)->graph.clamp(static_cast(input), lo, hi)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + int cactus_graph_conv2d_k3s2p1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_k3s2p1"); try { @@ -1226,6 +1259,25 @@ int cactus_graph_moe_layer_gated(cactus_graph_t graph, cactus_node_t hidden, cac } } +int cactus_graph_dense_mlp_tq_fused(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t gate_weight, cactus_node_t up_weight, cactus_node_t down_weight, float product_scale, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_dense_mlp_tq_fused"); + try { + *out = static_cast( + as_graph(graph)->graph.dense_mlp_tq_fused( + static_cast(hidden), + static_cast(gate_weight), + static_cast(up_weight), + static_cast(down_weight), + product_scale + ) + ); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + int cactus_graph_moe_layer_ungated(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out) { diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 76d557d6f..e5f45f9fc 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -1,5 +1,4 @@ #include "engine.h" -#include "../models/model.h" #include "cactus_graph.h" #include #include @@ -238,7 +237,9 @@ bool Model::init_internal(CactusGraph* gb, const std::string& model_folder, size initialized_ = true; - if (do_warmup) { + if (do_warmup && + config_.model_type != Config::ModelType::WHISPER && + config_.model_type != Config::ModelType::PARAKEET_TDT) { std::vector warmup_tokens = {2}; forward(warmup_tokens); auto* gb = static_cast(graph_handle_); @@ -662,6 +663,8 @@ bool Config::from_json(const std::string& config_path) { else if (mt == "gemma") model_type = ModelType::GEMMA; else if (mt == "gemma3n") model_type = ModelType::GEMMA3N; else if (mt == "lfm2") model_type = ModelType::LFM2; + else if (mt == "whisper") model_type = ModelType::WHISPER; + else if (mt == "parakeet_tdt" || mt == "parakeet-tdt") model_type = ModelType::PARAKEET_TDT; else if (mt == "youtu") model_type = ModelType::YOUTU; else if (mt == "needle") model_type = ModelType::NEEDLE; else model_type = ModelType::GEMMA4; @@ -860,35 +863,10 @@ std::unique_ptr create_model(const std::string& model_folder) { return nullptr; } - const bool has_vision_support = - config.use_image_tokens || - config.vision_num_layers > 0 || - config.vision_embed_dim > 0 || - config.vision_hidden_dim > 0 || - config.visual_tokens_per_img > 0; - - const bool has_audio_support = - config.audio_num_layers > 0 || - config.audio_hidden_dim > 0; - - if (config.model_type == Config::ModelType::LFM2 && has_vision_support) { - return std::make_unique(config); - } - - if (config.model_type == Config::ModelType::GEMMA4 && (has_vision_support || has_audio_support)) { - return std::make_unique(config); - } - - switch (config.model_type) { - case Config::ModelType::QWEN: - return std::make_unique(config); - case Config::ModelType::LFM2: - return std::make_unique(config); - case Config::ModelType::GEMMA4: - return std::make_unique(config); - default: - return std::make_unique(config); - } + CACTUS_LOG_ERROR("model", + "Native model subclasses are not present in this build. " + "Use cactus run-transpiled for transpiled graph bundles."); + return nullptr; } void Model::capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const { diff --git a/cactus-engine/src/transcribe.cpp b/cactus-engine/src/transcribe.cpp index aeae47377..6f889d475 100644 --- a/cactus-engine/src/transcribe.cpp +++ b/cactus-engine/src/transcribe.cpp @@ -2,12 +2,107 @@ #include "utils.h" #include "cactus_kernels.h" #include "wav.h" +#include +#include +#include #include +#include using namespace cactus::ffi; extern "C" { +int cactus_preprocess_audio_features( + const char* audio_file_path, + const char* model_type, + size_t mel_bins, + float* features_buffer, + size_t buffer_size, + size_t* feature_count, + size_t* out_mel_bins, + size_t* out_frames +) { + if (!audio_file_path || !model_type || !features_buffer || !feature_count || !out_mel_bins || !out_frames) { + last_error_message = "Invalid parameters for audio feature preprocessing"; + CACTUS_LOG_ERROR("audio_preprocess", last_error_message); + return -1; + } + + try { + AudioFP32 audio = load_wav(audio_file_path); + std::vector audio_samples = resample_to_16k_fp32(audio.samples, audio.sample_rate); + if (audio_samples.empty()) { + last_error_message = "No audio samples available for preprocessing"; + CACTUS_LOG_ERROR("audio_preprocess", last_error_message); + return -1; + } + + const size_t bins = std::max(1, mel_bins); + std::string lowered(model_type); + std::transform(lowered.begin(), lowered.end(), lowered.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + + std::vector features; + size_t frames = 0; + + if (lowered.find("gemma4") != std::string::npos || lowered.find("gemma-4") != std::string::npos) { + cactus::engine::Config cfg; + cfg.model_type = cactus::engine::Config::ModelType::GEMMA4; + cfg.audio_input_feat_size = static_cast(bins); + cfg.audio_soft_tokens = 188; + cfg.audio_fft_length = 512; + auto prepared = cactus::audio::preprocess_audio_for_gemma4(std::move(audio_samples), cfg); + features = std::move(prepared.features); + frames = prepared.num_frames; + } else if (lowered.find("parakeet") != std::string::npos) { + auto cfg = cactus::audio::get_parakeet_spectrogram_config(); + const size_t waveform_samples = audio_samples.size(); + cactus::audio::apply_preemphasis(audio_samples, 0.97f); + features = cactus::audio::compute_spectrogram_graph( + audio_samples, cfg, bins, 0.0f, 8000.0f, + cactus::audio::WHISPER_SAMPLE_RATE, 0, 0); + cactus::audio::normalize_parakeet_log_mel(features, bins); + size_t valid_frames = waveform_samples / cfg.hop_length; + if (valid_frames == 0) valid_frames = 1; + cactus::audio::trim_mel_frames(features, bins, valid_frames); + frames = features.size() / bins; + } else { + auto cfg = cactus::audio::get_whisper_spectrogram_config(); + const bool is_whisper_v3 = bins > 80; + if (is_whisper_v3) cactus::audio::apply_whisper_v3_overrides(cfg); + int norm_type = 1; // Whisper HF feature extractor uses Slaney-normalized mel filters. + int scale_type = 2; // Whisper HF feature extractor uses the Slaney mel scale. + std::vector mel = cactus::audio::compute_spectrogram_graph( + audio_samples, cfg, bins, 0.0f, 8000.0f, + cactus::audio::WHISPER_SAMPLE_RATE, norm_type, scale_type); + features = cactus::audio::normalize_whisper_mel(mel, bins, true); + frames = features.size() / bins; + } + + const size_t bytes_needed = features.size() * sizeof(float); + if (bytes_needed > buffer_size) { + last_error_message = "Audio feature output buffer too small"; + CACTUS_LOG_ERROR("audio_preprocess", last_error_message); + return -2; + } + + std::memcpy(features_buffer, features.data(), bytes_needed); + *feature_count = features.size(); + *out_mel_bins = bins; + *out_frames = frames; + return static_cast(features.size()); + } catch (const std::exception& e) { + last_error_message = e.what(); + CACTUS_LOG_ERROR("audio_preprocess", last_error_message); + return -1; + } catch (...) { + last_error_message = "Unknown error during audio feature preprocessing"; + CACTUS_LOG_ERROR("audio_preprocess", last_error_message); + return -1; + } +} + int cactus_transcribe( cactus_model_t model, const char* audio_file_path, @@ -20,62 +115,229 @@ int cactus_transcribe( const uint8_t* pcm_buffer, size_t pcm_buffer_size ) { - if (!model || !response_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("transcribe", "Invalid parameters"); - handle_error_response("Invalid parameters", response_buffer, buffer_size); + if (validate_audio_params("transcribe", model, response_buffer, buffer_size, + audio_file_path, pcm_buffer, pcm_buffer_size) != 0) { return -1; } - if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { - CACTUS_LOG_ERROR("transcribe", "No audio input provided"); - handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); - return -1; - } + try { + auto* handle = static_cast(model); + const auto model_type = handle->model->get_config().model_type; + const bool is_whisper = model_type == cactus::engine::Config::ModelType::WHISPER; + const bool is_parakeet = model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - // Load audio file into PCM if a file path was given - const uint8_t* pcm_data = pcm_buffer; - size_t pcm_size = pcm_buffer_size; - std::vector file_pcm; + if (!is_whisper && !is_parakeet) { + const uint8_t* pcm_data = pcm_buffer; + size_t pcm_size = pcm_buffer_size; + std::vector file_pcm; - if (audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { - AudioFP32 audio = load_wav(audio_file_path); - if (audio.samples.empty()) { - CACTUS_LOG_ERROR("transcribe", "Failed to load audio file: " << audio_file_path); - handle_error_response("Failed to load audio file", response_buffer, buffer_size); + if (audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { + AudioFP32 audio = load_wav(audio_file_path); + if (audio.samples.empty()) { + CACTUS_LOG_ERROR("transcribe", "Failed to load audio file: " << audio_file_path); + handle_error_response("Failed to load audio file", response_buffer, buffer_size); + return -1; + } + file_pcm.resize(audio.samples.size() * sizeof(int16_t)); + int16_t* out = reinterpret_cast(file_pcm.data()); + for (size_t i = 0; i < audio.samples.size(); i++) { + float clamped = std::max(-1.0f, std::min(1.0f, audio.samples[i])); + out[i] = static_cast(clamped * 32767.0f); + } + pcm_data = file_pcm.data(); + pcm_size = file_pcm.size(); + } + + std::string user_content = "Transcribe the following audio."; + if (prompt && prompt[0] != '\0') { + user_content = prompt; + } + + std::string messages_json = "[{\"role\": \"user\", \"content\": \"" + + escape_json_string(user_content) + "\"}]"; + + return cactus_complete( + model, + messages_json.c_str(), + response_buffer, + buffer_size, + options_json, + nullptr, + callback, + user_data, + pcm_data, + pcm_size + ); + } + + std::lock_guard lock(handle->model_mutex); + handle->should_stop = false; + auto start_time = std::chrono::high_resolution_clock::now(); + InferenceOptions options = parse_inference_options_json(options_json ? options_json : ""); + + std::vector audio_samples; + if (audio_file_path == nullptr) { + auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); + audio_samples = resample_to_16k_fp32(waveform_fp32, cactus::audio::WHISPER_SAMPLE_RATE); + } else { + AudioFP32 audio = load_wav(audio_file_path); + audio_samples = resample_to_16k_fp32(audio.samples, audio.sample_rate); + } + if (audio_samples.empty()) { + handle_error_response("No audio input provided", response_buffer, buffer_size); return -1; } - // Convert float samples to int16 PCM - file_pcm.resize(audio.samples.size() * sizeof(int16_t)); - int16_t* out = reinterpret_cast(file_pcm.data()); - for (size_t i = 0; i < audio.samples.size(); i++) { - float clamped = std::max(-1.0f, std::min(1.0f, audio.samples[i])); - out[i] = static_cast(clamped * 32767.0f); + + const size_t mel_bins = std::max(1, static_cast(handle->model->get_config().num_mel_bins)); + std::vector audio_features; + if (is_parakeet) { + auto cfg = cactus::audio::get_parakeet_spectrogram_config(); + const size_t waveform_samples = audio_samples.size(); + cactus::audio::apply_preemphasis(audio_samples, 0.97f); + audio_features = cactus::audio::compute_spectrogram_graph( + audio_samples, cfg, mel_bins, 0.0f, 8000.0f, + cactus::audio::WHISPER_SAMPLE_RATE, 0, 0); + cactus::audio::normalize_parakeet_log_mel(audio_features, mel_bins); + size_t valid_frames = waveform_samples / cfg.hop_length; + if (valid_frames == 0) valid_frames = 1; + cactus::audio::trim_mel_frames(audio_features, mel_bins, valid_frames); + } else { + auto cfg = cactus::audio::get_whisper_spectrogram_config(); + const bool is_whisper_v3 = mel_bins > 80; + if (is_whisper_v3) cactus::audio::apply_whisper_v3_overrides(cfg); + int norm_type = 1; // Whisper HF feature extractor uses Slaney-normalized mel filters. + int scale_type = 2; // Whisper HF feature extractor uses the Slaney mel scale. + std::vector mel = cactus::audio::compute_spectrogram_graph( + audio_samples, cfg, mel_bins, 0.0f, 8000.0f, + cactus::audio::WHISPER_SAMPLE_RATE, norm_type, scale_type); + audio_features = cactus::audio::normalize_whisper_mel(mel, mel_bins, true); + } + if (audio_features.empty()) { + handle_error_response("Computed audio features are empty", response_buffer, buffer_size); + return -1; } - pcm_data = file_pcm.data(); - pcm_size = file_pcm.size(); - } - // Build a transcription message and route through cactus_complete - std::string user_content = "Transcribe the following audio."; - if (prompt && prompt[0] != '\0') { - user_content = prompt; - } + auto* tokenizer = handle->model->get_tokenizer(); + if (!tokenizer) { + handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); + return -1; + } + + std::vector tokens; + if (prompt && prompt[0] != '\0') { + tokens = tokenizer->encode(prompt); + } else if (is_whisper) { + tokens = tokenizer->encode("<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"); + } + + if (tokens.empty() && is_whisper) { + handle_error_response("Decoder input tokens empty", response_buffer, buffer_size); + return -1; + } + + std::vector> stop_token_sequences = {{tokenizer->get_eos_token()}}; + auto append_exact_stop_sequence = [&](const char* stop_text) { + std::vector seq = tokenizer->encode(stop_text); + if (!seq.empty() && tokenizer->decode(seq) == stop_text) { + stop_token_sequences.push_back(std::move(seq)); + } + }; + append_exact_stop_sequence("<|endoftext|>"); + append_exact_stop_sequence("<|endoftranscript|>"); + append_exact_stop_sequence(""); + append_exact_stop_sequence(""); + + const float audio_length_sec = + static_cast(audio_samples.size()) / static_cast(cactus::audio::WHISPER_SAMPLE_RATE); + if (options.max_tokens == 100 && (!options_json || std::string(options_json).find("\"max_tokens\"") == std::string::npos)) { + options.max_tokens = std::max(100, static_cast(audio_length_sec * (is_parakeet ? 30.0f : 20.0f))); + } + + constexpr size_t WHISPER_MAX_DECODER_POSITIONS = 448; + if (is_whisper && tokens.size() < WHISPER_MAX_DECODER_POSITIONS) { + options.max_tokens = std::min(options.max_tokens, WHISPER_MAX_DECODER_POSITIONS - tokens.size()); + } + + std::string final_text; + std::vector generated_tokens; + generated_tokens.reserve(options.max_tokens); + const size_t prompt_tokens = tokens.size(); + double time_to_first_token = 0.0; + float total_entropy_sum = 0.0f; - std::string messages_json = "[{\"role\": \"user\", \"content\": \"" - + escape_json_string(user_content) + "\"}]"; - - return cactus_complete( - model, - messages_json.c_str(), - response_buffer, - buffer_size, - options_json, - nullptr, // no tools - callback, - user_data, - pcm_data, - pcm_size - ); + for (size_t i = 0; i < options.max_tokens; ++i) { + if (handle->should_stop) break; + + float token_entropy = 0.0f; + float tok_time_start = 0.0f; + float tok_time_end = 0.0f; + uint32_t next_token = handle->model->decode_with_audio( + tokens, audio_features, + options.temperature, options.top_p, options.top_k, + "", &token_entropy, + options.min_p, options.repetition_penalty, + is_parakeet ? &tok_time_start : nullptr, + is_parakeet ? &tok_time_end : nullptr); + + if (generated_tokens.empty()) { + auto t_first = std::chrono::high_resolution_clock::now(); + time_to_first_token = + std::chrono::duration_cast(t_first - start_time).count() / 1000.0; + } + + total_entropy_sum += token_entropy; + generated_tokens.push_back(next_token); + if (matches_stop_sequence(generated_tokens, stop_token_sequences)) { + break; + } + + std::string piece = tokenizer->decode({next_token}); + if (piece == "<|endoftext|>" || piece == "<|endoftranscript|>" || piece == "" || piece == "") { + break; + } + + tokens.push_back(next_token); + final_text += piece; + if (callback) callback(piece.c_str(), next_token, user_data); + } + + handle->model->reset_cache(); + + if (!final_text.empty() && final_text[0] == ' ') { + final_text.erase(0, 1); + } + + const size_t completion_tokens = generated_tokens.size(); + float mean_entropy = completion_tokens > 0 + ? total_entropy_sum / static_cast(completion_tokens) + : 0.0f; + float confidence = 1.0f - mean_entropy; + + auto end_time = std::chrono::high_resolution_clock::now(); + double total_time_ms = + std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + double prefill_tps = time_to_first_token > 0 ? (prompt_tokens * 1000.0) / time_to_first_token : 0.0; + double decode_time_ms = std::max(0.0, total_time_ms - time_to_first_token); + double decode_tps = + (completion_tokens > 1 && decode_time_ms > 0.0) + ? ((completion_tokens - 1) * 1000.0) / decode_time_ms + : 0.0; + + std::string json = construct_response_json( + final_text, {}, time_to_first_token, total_time_ms, + prefill_tps, decode_tps, prompt_tokens, completion_tokens, + confidence); + if (json.size() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + std::strcpy(response_buffer, json.c_str()); + return static_cast(json.size()); + } catch (const std::exception& e) { + CACTUS_LOG_ERROR("transcribe", "Exception: " << e.what()); + handle_error_response(e.what(), response_buffer, buffer_size); + return -1; + } } } // extern "C" diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index ab0b60823..4ffdac3a8 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -2,7 +2,6 @@ #define CACTUS_UTILS_H #include "engine.h" -#include "../models/model.h" #include #include #include @@ -113,7 +112,7 @@ inline cactus::engine::SpectrogramConfig get_whisper_spectrogram_config() { cfg.log_mel = "log10"; cfg.reference = 1.0f; cfg.min_value = 1e-10f; - cfg.remove_dc_offset = true; + cfg.remove_dc_offset = false; return cfg; } diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index eaefbc584..a9c67a8b8 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -91,7 +91,7 @@ enum class OpType { ABS, POW, FLATTEN, VIEW, MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, BILINEAR_INTERPOLATION, - SUM, MEAN, VARIANCE, MIN, MAX, + SUM, MEAN, VARIANCE, MIN, MAX, CUMSUM, RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, @@ -497,6 +497,7 @@ class CactusGraph { size_t variance(size_t input, int axis); size_t min(size_t input, int axis); size_t max(size_t input, int axis); + size_t cumsum(size_t input, int axis); size_t softmax(size_t input, int axis = -1); size_t topk(size_t input, size_t k); @@ -648,7 +649,7 @@ class CactusGraph { size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, Activation activation); - size_t dense_mlp_tq_fused(size_t hidden, size_t gate_weight, size_t up_weight, size_t down_weight); + size_t dense_mlp_tq_fused(size_t hidden, size_t gate_weight, size_t up_weight, size_t down_weight, float product_scale = 1.0f); size_t stats_pool(size_t input); size_t weighted_stats_pool(size_t input, size_t weights); @@ -665,6 +666,7 @@ class CactusGraph { size_t embedding(size_t embedding_tensor, size_t indices); size_t mmap_embeddings(const std::string& filename); size_t mmap_weights(const std::string& filename); + void bind_mmap_weights(size_t node_id, const std::string& filename); void release_weight_pages(size_t node_id); void prefetch_weight_pages(size_t node_id); void release_all_weight_pages(); diff --git a/cactus-graph/src/builder.cpp b/cactus-graph/src/builder.cpp index c0e2f92b7..5a7ce4c54 100644 --- a/cactus-graph/src/builder.cpp +++ b/cactus-graph/src/builder.cpp @@ -213,6 +213,23 @@ size_t CactusGraph::variance(size_t input, int axis) { return reduction_op(OpTyp size_t CactusGraph::min(size_t input, int axis) { return reduction_op(OpType::MIN, input, axis); } size_t CactusGraph::max(size_t input, int axis) { return reduction_op(OpType::MAX, input, axis); } +size_t CactusGraph::cumsum(size_t input, int axis) { + const auto& input_buffer = get_output_buffer(input); + if (input_buffer.shape.empty()) { + throw std::runtime_error("Cumsum requires at least one dimension"); + } + + int actual_axis = axis; + if (actual_axis < 0) { + actual_axis += static_cast(input_buffer.shape.size()); + } + if (actual_axis < 0 || static_cast(actual_axis) >= input_buffer.shape.size()) { + throw std::runtime_error("Invalid axis for cumsum operation"); + } + + return add_node(OpType::CUMSUM, {input}, input_buffer.shape, {.axis = actual_axis, .output_precision = input_buffer.precision}); +} + size_t CactusGraph::rms_norm(size_t input, size_t weight, float epsilon) { OpParams params{.epsilon = epsilon}; return add_node(OpType::RMS_NORM, {input, weight}, {}, params); @@ -294,7 +311,7 @@ size_t CactusGraph::moe_layer(size_t hidden, return add_node(OpType::MOE_LAYER, input_ids, hidden_buffer.shape, params); } -size_t CactusGraph::dense_mlp_tq_fused(size_t hidden, size_t gate_weight, size_t up_weight, size_t down_weight) { +size_t CactusGraph::dense_mlp_tq_fused(size_t hidden, size_t gate_weight, size_t up_weight, size_t down_weight, float product_scale) { const auto& hidden_buffer = get_output_buffer(hidden); const auto& down_buffer = get_output_buffer(down_weight); if (hidden_buffer.shape.empty()) { @@ -309,6 +326,7 @@ size_t CactusGraph::dense_mlp_tq_fused(size_t hidden, size_t gate_weight, size_t OpParams params; params.output_precision = Precision::FP16; + params.scalar = product_scale; return add_node(OpType::DENSE_MLP_TQ_FUSED, {hidden, gate_weight, up_weight, down_weight}, output_shape, params); diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index f2f7bc471..72f44c6f5 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -118,6 +118,7 @@ static bool init_dispatch() { dispatch_flat[static_cast(OpType::VARIANCE)] = compute_reduce_node; dispatch_flat[static_cast(OpType::MIN)] = compute_reduce_node; dispatch_flat[static_cast(OpType::MAX)] = compute_reduce_node; + dispatch_flat[static_cast(OpType::CUMSUM)] = compute_reduce_node; dispatch_flat[static_cast(OpType::FLATTEN)] = compute_reshape_node; dispatch_flat[static_cast(OpType::VIEW)] = compute_reshape_node; dispatch_flat[static_cast(OpType::RESHAPE)] = compute_reshape_node; @@ -200,7 +201,7 @@ static const char* op_type_names[] = { "ABS", "POW", "FLATTEN", "VIEW", "MATMUL", "TRANSPOSE", "RESHAPE", "SLICE", "GATHER", "EMBEDDING", "BILINEAR_INTERPOLATION", - "SUM", "MEAN", "VARIANCE", "MIN", "MAX", + "SUM", "MEAN", "VARIANCE", "MIN", "MAX", "CUMSUM", "RMS_NORM", "ROPE", "ROPE_GPTJ", "SOFTMAX", "ATTENTION", "ATTENTION_INT8_HYBRID", "REL_POS_BIAS", "CONV1D_CAUSAL", "CONV1D_K3", "CONV1D_K7S3", "CONV1D", @@ -297,11 +298,62 @@ void CactusGraph::execute(const std::string& profile_file) { BufferPool& pool = buffer_pool_; const size_t n = nodes_.size(); + auto get_env_int = [](const char* name, int fallback) -> int { + const char* val = std::getenv(name); + return val ? std::atoi(val) : fallback; + }; + + bool trace_execution = get_env_int("CACTUS_TRACE_EXECUTE", 0) != 0; + bool trace_nan = get_env_int("CACTUS_TRACE_NAN", 0) != 0; bool need_debug = !profile_file.empty(); if (!need_debug) { static const bool env_debug = check_debug_env(); need_debug = env_debug; } + if (trace_execution) { + need_debug = true; + } + + auto trace_nonfinite = [&](size_t node_idx, const GraphNode& node) { + if (!trace_nan) return; + const BufferDesc& buffer = node.output_buffer; + const void* data = buffer.get_data(); + if (!data || buffer.total_size == 0) return; + + auto report = [&](size_t elem_idx, float value) { + std::cerr << "[cactus:nan] idx=" << node_idx + << " id=" << node.id + << " op=" << get_op_name(node.op_type) + << " elem=" << elem_idx + << " value=" << value + << " shape=["; + for (size_t dim_idx = 0; dim_idx < buffer.shape.size(); ++dim_idx) { + if (dim_idx > 0) std::cerr << ","; + std::cerr << buffer.shape[dim_idx]; + } + std::cerr << "]" << std::endl; + }; + + if (buffer.precision == Precision::FP16) { + const __fp16* values = buffer.data_as<__fp16>(); + for (size_t i = 0; i < buffer.total_size; ++i) { + float value = static_cast(values[i]); + if (!std::isfinite(value)) { + report(i, value); + return; + } + } + } else if (buffer.precision == Precision::FP32) { + const float* values = buffer.data_as(); + for (size_t i = 0; i < buffer.total_size; ++i) { + float value = values[i]; + if (!std::isfinite(value)) { + report(i, value); + return; + } + } + } + }; if (!need_debug) { for (size_t i = 0; i < n; ++i) { @@ -314,6 +366,7 @@ void CactusGraph::execute(const std::string& profile_file) { } node->output_buffer.allocate_from_pool(pool); dispatch_node(*node, nodes_, node_index_map_); + trace_nonfinite(i, *node); if (node->op_type == OpType::PERSISTENT) { populated_node_ids_.insert(node->id); } @@ -331,11 +384,6 @@ void CactusGraph::execute(const std::string& profile_file) { } } - auto get_env_int = [](const char* name, int fallback) -> int { - const char* val = std::getenv(name); - return val ? std::atoi(val) : fallback; - }; - auto get_env_str = [](const char* name) -> std::string { const char* val = std::getenv(name); return val ? std::string(val) : std::string(); @@ -390,13 +438,32 @@ void CactusGraph::execute(const std::string& profile_file) { for (size_t node_idx = 0; node_idx < n; ++node_idx) { auto& node = nodes_[node_idx]; - if (node->op_type != OpType::INPUT) { - node->output_buffer.allocate_from_pool(pool); + if (node->op_type == OpType::INPUT) { + continue; } - if (enable_profiling && node->op_type != OpType::INPUT) { + node->output_buffer.allocate_from_pool(pool); + + if (trace_execution) { + std::cerr << "[cactus:execute] begin idx=" << node_idx + << " id=" << node->id + << " op=" << get_op_name(node->op_type) + << " shape=["; + for (size_t dim_idx = 0; dim_idx < node->output_buffer.shape.size(); ++dim_idx) { + if (dim_idx > 0) std::cerr << ","; + std::cerr << node->output_buffer.shape[dim_idx]; + } + std::cerr << "]" << std::endl; + } + + if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) { + dispatch_node(*node, nodes_, node_index_map_); + trace_nonfinite(node_idx, *node); + populated_node_ids_.insert(node->id); + } else if (enable_profiling) { auto start = std::chrono::high_resolution_clock::now(); dispatch_node(*node, nodes_, node_index_map_); + trace_nonfinite(node_idx, *node); if (node->op_type == OpType::PERSISTENT) { populated_node_ids_.insert(node->id); } @@ -415,10 +482,18 @@ void CactusGraph::execute(const std::string& profile_file) { << std::setw(20) << shape_str << std::endl; } else { dispatch_node(*node, nodes_, node_index_map_); + trace_nonfinite(node_idx, *node); if (node->op_type == OpType::PERSISTENT) { populated_node_ids_.insert(node->id); } } + + if (trace_execution) { + std::cerr << "[cactus:execute] done idx=" << node_idx + << " id=" << node->id + << " op=" << get_op_name(node->op_type) + << std::endl; + } } std::unique_ptr capture_file_stream; diff --git a/cactus-graph/src/io.cpp b/cactus-graph/src/io.cpp index 6a646943e..b295e328d 100644 --- a/cactus-graph/src/io.cpp +++ b/cactus-graph/src/io.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace { constexpr uint32_t fourcc(char a, char b, char c, char d) { @@ -21,6 +22,7 @@ namespace { constexpr uint32_t CACTUS_GRAPH_MAGIC = fourcc('C', 'G', 'R', 'F'); constexpr uint32_t FLAG_ORTHOGONAL_ROTATION = 1 << 1; constexpr uint32_t FLAG_INTERLEAVED_4ROW = 1 << 2; + constexpr uint32_t FLAG_EXTENDED_SHAPE = 1 << 4; constexpr size_t HEADER_SIZE = 84; inline size_t align_offset(size_t offset, size_t alignment) { @@ -29,6 +31,25 @@ namespace { return offset + (alignment - remainder); } + std::string resolve_quantized_weight_file(const std::string& filename) { + constexpr const char* suffix = ".weights"; + if (filename.size() <= std::strlen(suffix) || + filename.compare(filename.size() - std::strlen(suffix), std::strlen(suffix), suffix) != 0) { + return filename; + } + + const std::string stem = filename.substr(0, filename.size() - std::strlen(suffix)); + const std::string cq4 = stem + ".cq4.weights"; + if (std::filesystem::exists(cq4)) { + return cq4; + } + const std::string cq2 = stem + ".cq2.weights"; + if (std::filesystem::exists(cq2)) { + return cq2; + } + return filename; + } + inline void write_u32(std::ostream& out, uint32_t v) { out.write(reinterpret_cast(&v), sizeof(v)); } @@ -54,23 +75,30 @@ namespace { } std::vector compute_leaf_outputs(const GraphFile::SerializedGraph& sg) { - std::vector referenced(sg.nodes.size(), false); + std::unordered_set node_ids; + std::unordered_set referenced; + node_ids.reserve(sg.nodes.size()); + referenced.reserve(sg.nodes.size()); + + for (const auto& node : sg.nodes) { + node_ids.insert(node.index); + } for (const auto& node : sg.nodes) { for (uint32_t input_idx : node.inputs) { - if (input_idx >= referenced.size()) { - throw std::runtime_error("Graph save failed: input index out of range"); + if (node_ids.find(input_idx) == node_ids.end()) { + throw std::runtime_error("Graph save failed: input node id not found"); } - referenced[input_idx] = true; + referenced.insert(input_idx); } } std::vector outputs; outputs.reserve(sg.nodes.size()); - for (uint32_t i = 0; i < sg.nodes.size(); ++i) { - if (!referenced[i]) { - outputs.push_back(i); + for (const auto& node : sg.nodes) { + if (referenced.find(node.index) == referenced.end()) { + outputs.push_back(node.index); } } @@ -188,7 +216,7 @@ namespace { GraphFile::NodeEntry node; node.index = read_u32(in); uint32_t op_type_val = read_u32(in); - if (op_type_val > static_cast(OpType::IMAGE_PREPROCESS)) { + if (op_type_val > static_cast(OpType::DENSE_MLP_TQ_FUSED)) { throw std::runtime_error("Graph file corrupted: invalid op type"); } node.op_type = static_cast(op_type_val); @@ -214,31 +242,52 @@ CactusGraph CactusGraph::from_serialized(const GraphFile::SerializedGraph& sg) { CactusGraph graph; std::vector runtime_ids; runtime_ids.reserve(sg.nodes.size()); + std::unordered_map serialized_id_to_runtime_id; + serialized_id_to_runtime_id.reserve(sg.nodes.size()); if (sg.nodes.size() != sg.header.node_count) { throw std::runtime_error("Graph file corrupted: node count mismatch"); } + bool dense_legacy_indices = true; + for (size_t i = 0; i < sg.nodes.size(); ++i) { + if (sg.nodes[i].index != i) { + dense_legacy_indices = false; + break; + } + } + for (size_t i = 0; i < sg.nodes.size(); ++i) { const auto& node_entry = sg.nodes[i]; - if (node_entry.index != i) { - throw std::runtime_error("Graph file corrupted: node indices must be dense and ordered"); - } std::vector runtime_inputs; runtime_inputs.reserve(node_entry.inputs.size()); - for (uint32_t serialized_input_idx : node_entry.inputs) { - if (serialized_input_idx >= runtime_ids.size()) { - throw std::runtime_error( - "Graph file corrupted: input refers to a node that has not been reconstructed yet" - ); + for (uint32_t serialized_input_id : node_entry.inputs) { + if (dense_legacy_indices) { + if (serialized_input_id >= runtime_ids.size()) { + throw std::runtime_error( + "Graph file corrupted: input refers to a node that has not been reconstructed yet" + ); + } + runtime_inputs.push_back(runtime_ids[serialized_input_id]); + } else { + auto it = serialized_id_to_runtime_id.find(serialized_input_id); + if (it == serialized_id_to_runtime_id.end()) { + throw std::runtime_error( + "Graph file corrupted: input refers to a node that has not been reconstructed yet" + ); + } + runtime_inputs.push_back(it->second); } - runtime_inputs.push_back(runtime_ids[serialized_input_idx]); } size_t new_node_id = 0; + if (!dense_legacy_indices) { + graph.next_node_id_ = static_cast(node_entry.index); + } + if (node_entry.op_type == OpType::INPUT) { new_node_id = graph.input(node_entry.output_shape, node_entry.precision); } @@ -253,6 +302,7 @@ CactusGraph CactusGraph::from_serialized(const GraphFile::SerializedGraph& sg) { } } runtime_ids.push_back(new_node_id); + serialized_id_to_runtime_id[node_entry.index] = new_node_id; } return graph; } @@ -263,7 +313,8 @@ CactusGraph CactusGraph::load(const std::string& path) { } size_t CactusGraph::mmap_embeddings(const std::string& filename) { - auto mapped_file = std::make_unique(filename); + const std::string resolved_filename = resolve_quantized_weight_file(filename); + auto mapped_file = std::make_unique(resolved_filename); const auto& shape = mapped_file->shape(); if (shape.size() != 2) { @@ -325,12 +376,13 @@ size_t CactusGraph::mmap_embeddings(const std::string& filename) { } size_t CactusGraph::mmap_weights(const std::string& filename) { - auto it = weight_cache_.find(filename); + const std::string resolved_filename = resolve_quantized_weight_file(filename); + auto it = weight_cache_.find(resolved_filename); if (it != weight_cache_.end()) { return it->second; } - auto mapped_file = std::make_unique(filename); + auto mapped_file = std::make_unique(resolved_filename); const auto& shape = mapped_file->shape(); Precision precision = mapped_file->precision(); @@ -380,10 +432,96 @@ size_t CactusGraph::mmap_weights(const std::string& filename) { size_t file_idx = mapped_files_.size(); mapped_files_.push_back(std::move(mapped_file)); node_to_mapped_file_[node_id] = file_idx; - weight_cache_[filename] = node_id; + weight_cache_[resolved_filename] = node_id; return node_id; } +void CactusGraph::bind_mmap_weights(size_t node_id, const std::string& filename) { + const std::string resolved_filename = resolve_quantized_weight_file(filename); + auto node_it = node_index_map_.find(node_id); + if (node_it == node_index_map_.end()) { + throw std::out_of_range("Unknown input node id: " + std::to_string(node_id)); + } + + auto& node = *nodes_[node_it->second]; + if (node.op_type != OpType::INPUT) { + throw std::invalid_argument("Can only bind mmap weights to input nodes"); + } + + auto mapped_file = std::make_unique(resolved_filename); + const auto& shape = mapped_file->shape(); + Precision precision = mapped_file->precision(); + auto& buffer = node.output_buffer; + if (buffer.shape != shape) { + throw std::runtime_error("mmap weight shape mismatch for node " + std::to_string(node_id)); + } + if (buffer.precision != precision) { + throw std::runtime_error("mmap weight precision mismatch for node " + std::to_string(node_id)); + } + + set_external_input(node_id, const_cast(mapped_file->data()), precision); + buffer.group_size = 0; + buffer.num_groups = 0; + buffer.activation_scales_data = nullptr; + buffer.owned_activation_scales.reset(); + buffer.num_rows_for_activation_scales = 0; + buffer.cq_codebook = nullptr; + buffer.cq_input_scale = nullptr; + buffer.cq_input_scale_recip = nullptr; + buffer.cq_norms = nullptr; + buffer.cq_left_signs = nullptr; + buffer.cq_right_signs = nullptr; + buffer.cq_permutation = nullptr; + buffer.cq_rotation = nullptr; + buffer.cq_flags = 0; + + if (PrecisionTraits::is_cq(precision) && mapped_file->group_size() > 0) { + buffer.group_size = mapped_file->group_size(); + buffer.num_groups = mapped_file->num_groups(); + + const char* scales_base = static_cast(mapped_file->scales_data()); + uint32_t bits = PrecisionTraits::cq_bits(precision); + uint32_t cb_size = 1u << bits; + uint32_t gs = static_cast(mapped_file->group_size()); + uint32_t K = gs * static_cast(mapped_file->num_groups()); + uint32_t N = static_cast(shape.size() >= 2 ? shape[0] : 1); + + size_t off = 0; + buffer.cq_codebook = reinterpret_cast(scales_base + off); + off += cb_size * sizeof(__fp16); + buffer.cq_input_scale = reinterpret_cast(scales_base + off); + off += K * sizeof(__fp16); + buffer.cq_input_scale_recip = reinterpret_cast(scales_base + off); + off += K * sizeof(__fp16); + buffer.cq_norms = reinterpret_cast(scales_base + off); + off += static_cast(N) * mapped_file->num_groups() * sizeof(__fp16); + + if (mapped_file->is_orthogonal_rotation()) { + buffer.cq_rotation = reinterpret_cast(scales_base + off); + buffer.cq_flags = CACTUS_QUANT_FLAG_ORTHOGONAL; + } else { + buffer.cq_left_signs = reinterpret_cast(scales_base + off); + off += gs; + buffer.cq_right_signs = reinterpret_cast(scales_base + off); + off += gs; + buffer.cq_permutation = reinterpret_cast(scales_base + off); + buffer.cq_flags = 0; + } + if (mapped_file->is_interleaved_4row()) { + buffer.cq_flags |= CACTUS_QUANT_FLAG_INTERLEAVED_4ROW; + } + } else if (precision == Precision::INT8 && mapped_file->group_size() > 0) { + buffer.group_size = mapped_file->group_size(); + buffer.num_groups = mapped_file->num_groups(); + buffer.set_activation_scales(const_cast(mapped_file->scales_data()), shape.empty() ? 0 : shape[0]); + } + + size_t file_idx = mapped_files_.size(); + mapped_files_.push_back(std::move(mapped_file)); + node_to_mapped_file_[node_id] = file_idx; + weight_cache_[resolved_filename] = node_id; +} + void CactusGraph::release_weight_pages(size_t node_id) { auto it = node_to_mapped_file_.find(node_id); if (it != node_to_mapped_file_.end() && it->second < mapped_files_.size()) { @@ -438,13 +576,6 @@ void save_graph(const CactusGraph& graph, throw std::runtime_error("Cannot open file for writing: " + filename); } - std::unordered_map id_to_index; - id_to_index.reserve(graph.nodes_.size()); - - for (uint32_t i = 0; i < graph.nodes_.size(); ++i) { - id_to_index[graph.nodes_[i]->id] = i; - } - SerializedGraph sg; sg.header.magic = CACTUS_GRAPH_MAGIC; sg.header.version = 4; @@ -457,7 +588,7 @@ void save_graph(const CactusGraph& graph, const auto& node = graph.nodes_[i]; NodeEntry entry; - entry.index = i; + entry.index = static_cast(node->id); entry.op_type = node->op_type; entry.output_shape = node->output_buffer.shape; entry.precision = node->output_buffer.precision; @@ -465,11 +596,7 @@ void save_graph(const CactusGraph& graph, entry.inputs.reserve(node->input_ids.size()); for (size_t input_id : node->input_ids) { - auto it = id_to_index.find(input_id); - if (it == id_to_index.end()) { - throw std::runtime_error("Graph save failed: missing input id mapping"); - } - entry.inputs.push_back(it->second); + entry.inputs.push_back(static_cast(input_id)); } if (node->op_type == OpType::INPUT) { @@ -506,29 +633,46 @@ SerializedGraph load_graph(const std::string& filename) { throw std::runtime_error("Graph file corrupted: node count mismatch"); } + bool dense_legacy_indices = true; + std::unordered_set node_ids; + node_ids.reserve(sg.nodes.size()); for (uint32_t i = 0; i < sg.nodes.size(); ++i) { const auto& node = sg.nodes[i]; - if (node.index != i) { - throw std::runtime_error("Graph file corrupted: node indices must be dense and ordered"); + dense_legacy_indices = false; } + node_ids.insert(node.index); + } + for (const auto& node : sg.nodes) { for (uint32_t input_idx : node.inputs) { - if (input_idx >= sg.nodes.size()) { - throw std::runtime_error("Graph file corrupted: input index out of range"); + if (dense_legacy_indices) { + if (input_idx >= sg.nodes.size()) { + throw std::runtime_error("Graph file corrupted: input index out of range"); + } + } else if (node_ids.find(input_idx) == node_ids.end()) { + throw std::runtime_error("Graph file corrupted: input node id out of range"); } } } for (uint32_t input_idx : sg.graph_inputs) { - if (input_idx >= sg.nodes.size()) { - throw std::runtime_error("Graph file corrupted: graph input index out of range"); + if (dense_legacy_indices) { + if (input_idx >= sg.nodes.size()) { + throw std::runtime_error("Graph file corrupted: graph input index out of range"); + } + } else if (node_ids.find(input_idx) == node_ids.end()) { + throw std::runtime_error("Graph file corrupted: graph input node id out of range"); } } for (uint32_t output_idx : sg.graph_outputs) { - if (output_idx >= sg.nodes.size()) { - throw std::runtime_error("Graph file corrupted: graph output index out of range"); + if (dense_legacy_indices) { + if (output_idx >= sg.nodes.size()) { + throw std::runtime_error("Graph file corrupted: graph output index out of range"); + } + } else if (node_ids.find(output_idx) == node_ids.end()) { + throw std::runtime_error("Graph file corrupted: graph output node id out of range"); } } @@ -748,13 +892,12 @@ void MappedFile::parse_header() { uint32_t ndim = *reinterpret_cast(ptr + offset); offset += sizeof(uint32_t); - shape_.clear(); + std::vector dims; + dims.reserve(8); for (uint32_t i = 0; i < 4; i++) { uint64_t dim_val = *reinterpret_cast(ptr + offset); offset += sizeof(uint64_t); - if (i < ndim && dim_val > 0) { - shape_.push_back(static_cast(dim_val)); - } + dims.push_back(dim_val); } uint32_t prec_val = *reinterpret_cast(ptr + offset); @@ -776,7 +919,30 @@ void MappedFile::parse_header() { original_N_ = *reinterpret_cast(ptr + offset); offset += sizeof(uint64_t); - size_t aligned_header = align_offset(HEADER_SIZE, alignment_); + size_t header_size = HEADER_SIZE; + if ((flags & FLAG_EXTENDED_SHAPE) != 0) { + header_size += 4 * sizeof(uint64_t); + if (file_size_ < header_size) { + throw std::runtime_error("File too small: insufficient data for extended shape header"); + } + for (uint32_t i = 0; i < 4; i++) { + uint64_t dim_val = *reinterpret_cast(ptr + offset); + offset += sizeof(uint64_t); + dims.push_back(dim_val); + } + } + + shape_.clear(); + if (ndim > dims.size()) { + throw std::runtime_error("Invalid tensor file: ndim exceeds encoded shape rank"); + } + for (uint32_t i = 0; i < ndim; i++) { + if (dims[i] > 0) { + shape_.push_back(static_cast(dims[i])); + } + } + + size_t aligned_header = align_offset(header_size, alignment_); if (scales_bytes_ > 0) { scales_offset_ = aligned_header; diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 38830a6b6..f44816a69 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -2,6 +2,7 @@ #include "cactus_kernels.h" #include #include +#include namespace { @@ -187,6 +188,10 @@ void compute_attention_cached_node( size_t new_seq_len = key_new_buf.total_size / (kv_heads * hdim); size_t history_len = (cache_len >= new_seq_len) ? cache_len - new_seq_len : 0; + size_t position_offset = node.params.position_offset; + if (position_offset == std::numeric_limits::max()) { + position_offset = history_len; + } cactus_attention_hybrid_int8_fp16( query_buf.data_as<__fp16>(), @@ -200,7 +205,7 @@ void compute_attention_cached_node( batch_size, seq_len, history_len, seq_len, num_q_heads, kv_heads, hdim, node.params.scale, - node.params.position_offset, + position_offset, true, node.params.window_size, KV_QUANT_GROUP_SIZE, diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index bda707085..401603f52 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -220,6 +220,28 @@ void compute_reduce_node(GraphNode& node, const std::vector(axis) >= input_buffer.shape.size()) { + throw std::runtime_error("Cumsum axis out of range"); + } + + auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); + const __fp16* input = input_buffer.data_as<__fp16>(); + __fp16* output = node.output_buffer.data_as<__fp16>(); + + for (size_t outer = 0; outer < dims.outer; ++outer) { + for (size_t inner = 0; inner < dims.inner; ++inner) { + float running = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + size_t flat_index = ((outer * dims.axis_size) + axis_index) * dims.inner + inner; + running += static_cast(input[flat_index]); + output[flat_index] = static_cast<__fp16>(running); + } + } + } + return; + } + if (axis == -1) { switch (node.op_type) { case OpType::SUM: { diff --git a/cactus-graph/src/ops_nn.cpp b/cactus-graph/src/ops_nn.cpp index 8f53156d2..23afc2f20 100644 --- a/cactus-graph/src/ops_nn.cpp +++ b/cactus-graph/src/ops_nn.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace { thread_local std::vector<__fp16> transpose_buffer_fp16; @@ -344,12 +345,94 @@ void compute_dense_mlp_tq_fused_node(GraphNode& node, const std::vector(M), gate); cactus_gelu_f16(gate, gate, inter_size); + float max_gate_abs = 0.0f; + size_t gate_nonfinite = 0; + if (use_safe_product_scale) { + const __fp16 scale = static_cast<__fp16>(node.params.scalar); + for (size_t i = 0; i < inter_size; ++i) { + float value = static_cast(gate[i]); + if (!std::isfinite(value)) { + gate_nonfinite += 1; + value = std::copysign(65504.0f, value); + } + value *= static_cast(scale); + max_gate_abs = std::max(max_gate_abs, std::abs(value)); + gate[i] = static_cast<__fp16>(value); + } + } cactus_quant_matmul(&up_mat, hidden, static_cast(M), up); + float max_up_abs = 0.0f; + size_t up_nonfinite = 0; + if (use_safe_product_scale) { + for (size_t i = 0; i < inter_size; ++i) { + float value = static_cast(up[i]); + if (!std::isfinite(value)) { + up_nonfinite += 1; + value = std::copysign(65504.0f, value); + } + max_up_abs = std::max(max_up_abs, std::abs(value)); + up[i] = static_cast<__fp16>(value); + } + const float product_bound = max_gate_abs * max_up_abs; + constexpr float kSafeHadamardProductBound = 1024.0f; + if (std::isfinite(product_bound) && product_bound > kSafeHadamardProductBound) { + const __fp16 extra_scale = static_cast<__fp16>(kSafeHadamardProductBound / product_bound); + for (size_t i = 0; i < inter_size; ++i) { + gate[i] = static_cast<__fp16>(gate[i] * extra_scale); + } + max_gate_abs *= static_cast(extra_scale); + } + } cactus_multiply_f16(gate, up, prod, inter_size); + if (trace_dense_mlp) { + size_t prod_nonfinite = 0; + float max_prod_abs = 0.0f; + for (size_t i = 0; i < inter_size; ++i) { + float value = static_cast(prod[i]); + if (!std::isfinite(value)) { + prod_nonfinite += 1; + continue; + } + max_prod_abs = std::max(max_prod_abs, std::abs(value)); + } + std::cerr << "[cactus:dense_mlp] id=" << node.id + << " scale=" << node.params.scalar + << " gate_nonfinite=" << gate_nonfinite + << " up_nonfinite=" << up_nonfinite + << " prod_nonfinite=" << prod_nonfinite + << " max_gate=" << max_gate_abs + << " max_up=" << max_up_abs + << " max_prod=" << max_prod_abs + << " shape=["; + for (size_t i = 0; i < node.output_buffer.shape.size(); ++i) { + if (i) std::cerr << ","; + std::cerr << node.output_buffer.shape[i]; + } + std::cerr << "]" << std::endl; + } cactus_quant_matmul(&down_mat, prod, static_cast(M), output); + if (trace_dense_mlp) { + size_t output_nonfinite = 0; + float max_output_abs = 0.0f; + const size_t output_count = node.output_buffer.total_size; + for (size_t i = 0; i < output_count; ++i) { + float value = static_cast(output[i]); + if (!std::isfinite(value)) { + output_nonfinite += 1; + continue; + } + max_output_abs = std::max(max_output_abs, std::abs(value)); + } + std::cerr << "[cactus:dense_mlp_out] id=" << node.id + << " output_nonfinite=" << output_nonfinite + << " max_output=" << max_output_abs + << std::endl; + } } void compute_rms_norm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { @@ -848,9 +931,3 @@ void compute_groupnorm_node(GraphNode& node, const std::vector(params.max_cache_seq_len)); break; + case ParamField::CacheSinkSize: write_u64(out, static_cast(params.cache_sink_size)); break; } } @@ -334,6 +360,8 @@ void read_field(std::istream& in, ParamField field, OpParams& params) { case ParamField::CachedKScalesPtr: case ParamField::CachedVScalesPtr: throw std::runtime_error("Graph file corrupted: runtime-only field serialized"); + case ParamField::MaxCacheSeqLen: params.max_cache_seq_len = static_cast(read_u64(in)); break; + case ParamField::CacheSinkSize: params.cache_sink_size = static_cast(read_u64(in)); break; } } diff --git a/cactus/cactus.h b/cactus/cactus.h index 13aab6e09..7b032b6e2 100644 --- a/cactus/cactus.h +++ b/cactus/cactus.h @@ -5,6 +5,5 @@ #include "../cactus-graph/cactus_graph.h" #include "../cactus-engine/cactus_engine.h" #include "../cactus-engine/src/engine.h" -#include "../cactus-engine/models/model.h" #endif // CACTUS_H diff --git a/python/README.md b/python/README.md index 322801e69..7fe2500f9 100644 --- a/python/README.md +++ b/python/README.md @@ -32,8 +32,8 @@ cactus auth ```python -from src.downloads import ensure_model -from src.cactus import cactus_init, cactus_complete, cactus_destroy +from cactus import ensure_model +from cactus import cactus_init, cactus_complete, cactus_destroy import json # Downloads weights from HuggingFace if not already present @@ -56,14 +56,13 @@ All functions are module-level and mirror the C FFI directly. Handles are plain Download pre-converted weights programmatically (no CLI needed): ```python -from src.downloads import ensure_model, get_weights_dir, download_from_hf +from cactus import ensure_model, get_weights_dir # ensure_model downloads if missing, returns Path to weights dir weights = ensure_model("openai/whisper-tiny") # Or check / download manually weights_dir = get_weights_dir("openai/whisper-tiny") # -> Path("weights/whisper-tiny") -download_from_hf("openai/whisper-tiny", weights_dir, precision="INT4") # -> bool ``` ### Init / Lifecycle @@ -397,7 +396,7 @@ result = json.loads(cactus_complete(model, messages, None, None, None)) The `Graph` API provides a tensor computation graph for building and executing dataflow pipelines on the Cactus kernel layer: ```python -from src.graph import Graph +from cactus.bindings.graph import Graph import numpy as np g = Graph() diff --git a/python/cactus/bindings/cactus.py b/python/cactus/bindings/cactus.py index 92222fb2a..e3929275b 100644 --- a/python/cactus/bindings/cactus.py +++ b/python/cactus/bindings/cactus.py @@ -1,12 +1,16 @@ """Cactus Python FFI bindings.""" import ctypes +import os import platform from pathlib import Path TokenCallback = ctypes.CFUNCTYPE(None, ctypes.c_char_p, ctypes.c_uint32, ctypes.c_void_p) _DIR = Path(__file__).parent.parent.parent.parent -if platform.system() == "Darwin": +_OVERRIDE_LIB_PATH = os.environ.get("CACTUS_LIB_PATH") +if _OVERRIDE_LIB_PATH: + _LIB_PATH = Path(_OVERRIDE_LIB_PATH).expanduser().resolve() +elif platform.system() == "Darwin": _LIB_PATH = _DIR / "cactus" / "build" / "libcactus.dylib" else: _LIB_PATH = _DIR / "cactus" / "build" / "libcactus.so" @@ -19,6 +23,16 @@ _lib = ctypes.CDLL(str(_LIB_PATH)) + +def _bind_optional(name, argtypes, restype): + try: + fn = getattr(_lib, name) + except AttributeError: + return None + fn.argtypes = argtypes + fn.restype = restype + return fn + cactus_graph_t = ctypes.c_void_p cactus_node_t = ctypes.c_uint64 @@ -88,11 +102,22 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_divide.restype = ctypes.c_int +_bind_optional( + "cactus_graph_not_equal", + [cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_precision_cast.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_precision_cast.restype = ctypes.c_int +_bind_optional( + "cactus_graph_quantize_activations", + [cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) + _lib.cactus_graph_scalar_add.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] @@ -109,6 +134,16 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_scalar_divide.restype = ctypes.c_int +_bind_optional( + "cactus_graph_scalar_floor_divide", + [cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_scalar_not_equal", + [cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_scalar_exp.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] @@ -129,6 +164,16 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_scalar_log.restype = ctypes.c_int +_bind_optional( + "cactus_graph_masked_select_prefix", + [cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_masked_scatter", + [cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_abs.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) @@ -154,6 +199,11 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_reshape.restype = ctypes.c_int +_bind_optional( + "cactus_graph_expand", + [cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_transpose.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] @@ -182,11 +232,11 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_graph_cat.restype = ctypes.c_int _lib.cactus_graph_matmul.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, ctypes.c_int32, ctypes.POINTER(cactus_node_t) + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, ctypes.c_int32, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_matmul.restype = ctypes.c_int _lib.cactus_graph_gather.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_gather.restype = ctypes.c_int _lib.cactus_graph_embedding_from_tensor.argtypes = [ @@ -205,10 +255,25 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, ctypes.c_char_p, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_mmap_weights.restype = ctypes.c_int +_bind_optional( + "cactus_graph_bind_mmap_weights", + [cactus_graph_t, cactus_node_t, ctypes.c_char_p], + ctypes.c_int, +) _lib.cactus_graph_bilinear_interpolation.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_bilinear_interpolation.restype = ctypes.c_int +_bind_optional( + "cactus_graph_set_grouped_scales", + [cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_void_p], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_set_interleaved", + [cactus_graph_t, cactus_node_t, ctypes.c_bool, ctypes.c_size_t], + ctypes.c_int, +) _lib.cactus_graph_release_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] _lib.cactus_graph_release_weight_pages.restype = ctypes.c_int _lib.cactus_graph_prefetch_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] @@ -236,6 +301,10 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_max.restype = ctypes.c_int +_lib.cactus_graph_cumsum.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_cumsum.restype = ctypes.c_int _lib.cactus_graph_relu.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) @@ -304,6 +373,53 @@ class cactus_tensor_info_t(ctypes.Structure): ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_attention.restype = ctypes.c_int +_lib.cactus_graph_kv_cache_state.argtypes = [ + cactus_graph_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.POINTER(cactus_node_t), +] +_lib.cactus_graph_kv_cache_state.restype = ctypes.c_int +_lib.cactus_graph_kv_cache_append.argtypes = [ + cactus_graph_t, + cactus_node_t, + cactus_node_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.POINTER(cactus_node_t), +] +_lib.cactus_graph_kv_cache_append.restype = ctypes.c_int +_lib.cactus_graph_attention_cached.argtypes = [ + cactus_graph_t, + cactus_node_t, + cactus_node_t, + cactus_node_t, + cactus_node_t, + cactus_node_t, + ctypes.c_float, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.POINTER(cactus_node_t), +] +_lib.cactus_graph_attention_cached.restype = ctypes.c_int +_lib.cactus_graph_conv_cache_state.argtypes = [ + cactus_graph_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.POINTER(cactus_node_t), +] +_lib.cactus_graph_conv_cache_state.restype = ctypes.c_int +_lib.cactus_graph_conv_cache_append.argtypes = [ + cactus_graph_t, + cactus_node_t, + cactus_node_t, + ctypes.POINTER(cactus_node_t), +] +_lib.cactus_graph_conv_cache_append.restype = ctypes.c_int _lib.cactus_graph_rel_pos_bias.argtypes = [ cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] @@ -339,6 +455,10 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_conv1d_pointwise.restype = ctypes.c_int +_lib.cactus_graph_clamp.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.c_float, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_clamp.restype = ctypes.c_int _lib.cactus_graph_conv2d_k3s2p1.argtypes = [ cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) ] @@ -351,6 +471,27 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_conv2d_pointwise_1x1.restype = ctypes.c_int +_bind_optional( + "cactus_graph_conv2d_k3s1p1", + [cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_conv2d", + [ + cactus_graph_t, + cactus_node_t, + cactus_node_t, + ctypes.c_bool, + cactus_node_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.POINTER(cactus_node_t), + ], + ctypes.c_int, +) _lib.cactus_graph_lstm_cell.argtypes = [ cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] @@ -385,6 +526,11 @@ class cactus_tensor_info_t(ctypes.Structure): ctypes.c_size_t, ctypes.c_size_t, ctypes.c_bool, ctypes.c_float, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_moe_layer_gated.restype = ctypes.c_int +_lib.cactus_graph_dense_mlp_tq_fused.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, + ctypes.c_float, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_dense_mlp_tq_fused.restype = ctypes.c_int _lib.cactus_graph_moe_layer_ungated.argtypes = [ cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t), ctypes.POINTER(cactus_node_t), @@ -443,11 +589,14 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_transcribe.restype = ctypes.c_int -_lib.cactus_detect_language.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t -] -_lib.cactus_detect_language.restype = ctypes.c_int +_bind_optional( + "cactus_detect_language", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ], + ctypes.c_int, +) _lib.cactus_embed.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_float), @@ -467,24 +616,44 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_audio_embed.restype = ctypes.c_int -_lib.cactus_vad.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t -] -_lib.cactus_vad.restype = ctypes.c_int - -_lib.cactus_diarize.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t -] -_lib.cactus_diarize.restype = ctypes.c_int - -_lib.cactus_embed_speaker.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, - ctypes.POINTER(ctypes.c_float), ctypes.c_size_t -] -_lib.cactus_embed_speaker.restype = ctypes.c_int +try: + _lib.cactus_preprocess_audio_features.argtypes = [ + ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, + ctypes.POINTER(ctypes.c_size_t), ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_size_t) + ] + _lib.cactus_preprocess_audio_features.restype = ctypes.c_int +except AttributeError: + pass + +_bind_optional( + "cactus_vad", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ], + ctypes.c_int, +) + +_bind_optional( + "cactus_diarize", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ], + ctypes.c_int, +) + +_bind_optional( + "cactus_embed_speaker", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, + ], + ctypes.c_int, +) _lib.cactus_reset.argtypes = [ctypes.c_void_p] _lib.cactus_reset.restype = None @@ -507,6 +676,20 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_tokenize.restype = ctypes.c_int +_bind_optional( + "cactus_decode_tokens", + [ + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_uint32), + ctypes.c_size_t, + ctypes.c_float, + ctypes.c_float, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_uint32), + ], + ctypes.c_int, +) + _lib.cactus_score_window.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32), @@ -525,19 +708,22 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_rag_query.restype = ctypes.c_int -_lib.cactus_stream_transcribe_start.argtypes = [ctypes.c_void_p, ctypes.c_char_p] -_lib.cactus_stream_transcribe_start.restype = ctypes.c_void_p +_bind_optional("cactus_stream_transcribe_start", [ctypes.c_void_p, ctypes.c_char_p], ctypes.c_void_p) -_lib.cactus_stream_transcribe_process.argtypes = [ - ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, - ctypes.c_char_p, ctypes.c_size_t -] -_lib.cactus_stream_transcribe_process.restype = ctypes.c_int +_bind_optional( + "cactus_stream_transcribe_process", + [ + ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ctypes.c_char_p, ctypes.c_size_t, + ], + ctypes.c_int, +) -_lib.cactus_stream_transcribe_stop.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t -] -_lib.cactus_stream_transcribe_stop.restype = ctypes.c_int +_bind_optional( + "cactus_stream_transcribe_stop", + [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t], + ctypes.c_int, +) _lib.cactus_index_init.argtypes = [ctypes.c_char_p, ctypes.c_size_t] _lib.cactus_index_init.restype = ctypes.c_void_p @@ -710,8 +896,6 @@ def cactus_prefill(model, messages_json, options_json, tools_json, pcm_data=None ) if rc < 0: raise RuntimeError(_err("Prefill failed")) - return buf.value.decode("utf-8", errors="ignore") - def cactus_detect_language(model, audio_path, options_json, pcm_data): """Detects the spoken language in audio. Returns JSON string.""" buf = ctypes.create_string_buffer(65536) @@ -785,6 +969,29 @@ def cactus_audio_embed(model, audio_path): return list(buf[:dim.value]) +def cactus_preprocess_audio_features(audio_path, model_type, mel_bins, capacity): + """Returns Cactus-native audio frontend features as (values, mel_bins, frames).""" + if not hasattr(_lib, "cactus_preprocess_audio_features"): + raise RuntimeError("cactus_preprocess_audio_features is unavailable; rebuild with cactus build --python") + buf = (ctypes.c_float * int(capacity))() + feature_count = ctypes.c_size_t() + out_mels = ctypes.c_size_t() + out_frames = ctypes.c_size_t() + rc = _lib.cactus_preprocess_audio_features( + _enc(audio_path), + _enc(model_type), + ctypes.c_size_t(int(mel_bins)), + buf, + ctypes.sizeof(buf), + ctypes.byref(feature_count), + ctypes.byref(out_mels), + ctypes.byref(out_frames), + ) + if rc < 0: + raise RuntimeError(_err("Audio feature preprocessing failed")) + return list(buf[:feature_count.value]), int(out_mels.value), int(out_frames.value) + + def cactus_vad(model, audio_path, options_json, pcm_data): """Runs voice activity detection. Returns JSON string.""" buf = ctypes.create_string_buffer(65536) @@ -858,6 +1065,26 @@ def cactus_tokenize(model, text): return list(arr[:n.value]) +def cactus_decode_tokens(model, tokens, temperature=0.0, top_p=1.0, top_k=1): + """Returns the next token ID from native Cactus for exact raw token IDs.""" + if not tokens: + raise ValueError("tokens must be non-empty") + arr = (ctypes.c_uint32 * len(tokens))(*[int(token) for token in tokens]) + out = ctypes.c_uint32(0) + rc = _lib.cactus_decode_tokens( + model, + arr, + len(tokens), + ctypes.c_float(float(temperature)), + ctypes.c_float(float(top_p)), + ctypes.c_size_t(int(top_k)), + ctypes.byref(out), + ) + if rc < 0: + raise RuntimeError(_err("Raw token decode failed")) + return int(out.value) + + def cactus_score_window(model, tokens, start, end, context): """Scores a window of tokens. Returns JSON string.""" buf = ctypes.create_string_buffer(65536) diff --git a/python/cactus/bindings/graph.py b/python/cactus/bindings/graph.py index 52f706c6e..6aba55ac1 100644 --- a/python/cactus/bindings/graph.py +++ b/python/cactus/bindings/graph.py @@ -1,7 +1,8 @@ import ctypes +from os import wait import numpy as np -from .cactus import _lib, cactus_node_t, cactus_tensor_info_t +from .cactus import _err, _lib, cactus_node_t, cactus_tensor_info_t class Graph: @@ -12,6 +13,7 @@ class Graph: CQ2 = 4 CQ3 = 5 CQ4 = 6 + INT4 = CQ1 CPU = 0 NPU = 1 ACT_SILU = 0 @@ -52,7 +54,7 @@ def input(self, shape, dtype=FP16): out = cactus_node_t() rc = _lib.cactus_graph_input(self.h, arr, len(shape), int(dtype), ctypes.byref(out)) if rc != 0: - raise RuntimeError("graph_input failed") + raise RuntimeError(_err("graph_input failed")) return self._tensor_from_node(out.value) def set_input(self, tensor, data, dtype=None): @@ -62,6 +64,25 @@ def set_input(self, tensor, data, dtype=None): raise ValueError("tensor belongs to a different graph") target_dtype = int(tensor.dtype if dtype is None else dtype) arr = self._coerce_input_array(data, target_dtype) + info = self._get_output_info(tensor.id) + expected_shape = tuple(int(x) for x in info["shape"]) + expected_num_elements = int(info["num_elements"]) + expected_byte_size = int(info["byte_size"]) + if tuple(int(x) for x in arr.shape) != expected_shape: + raise ValueError( + "graph input shape mismatch for node " + f"{tensor.id}: expected {expected_shape}, got {tuple(int(x) for x in arr.shape)}" + ) + if int(arr.size) != expected_num_elements: + raise ValueError( + "graph input element-count mismatch for node " + f"{tensor.id}: expected {expected_num_elements}, got {int(arr.size)}" + ) + if int(arr.nbytes) != expected_byte_size: + raise ValueError( + "graph input byte-size mismatch for node " + f"{tensor.id}: expected {expected_byte_size}, got {int(arr.nbytes)}" + ) rc = _lib.cactus_graph_set_input( self.h, cactus_node_t(tensor.id), @@ -69,7 +90,7 @@ def set_input(self, tensor, data, dtype=None): target_dtype, ) if rc != 0: - raise RuntimeError("graph_set_input failed") + raise RuntimeError(_err("graph_set_input failed")) def set_external_input(self, tensor, data_ptr, dtype=None): if not isinstance(tensor, Tensor): @@ -85,7 +106,7 @@ def set_external_input(self, tensor, data_ptr, dtype=None): target_dtype, ) if rc != 0: - raise RuntimeError("graph_set_external_input failed") + raise RuntimeError(_err("graph_set_external_input failed")) def hard_reset(self): rc = _lib.cactus_graph_hard_reset(self.h) @@ -95,7 +116,7 @@ def hard_reset(self): def execute(self): rc = _lib.cactus_graph_execute(self.h) if rc != 0: - raise RuntimeError("graph_execute failed") + raise RuntimeError(_err("graph_execute failed")) def add(self, a, b): return self._binary("cactus_graph_add", a, b) @@ -112,6 +133,9 @@ def multiply(self, a, b): def divide(self, a, b): return self._binary("cactus_graph_divide", a, b) + def not_equal(self, a, b): + return self._binary("cactus_graph_not_equal", a, b) + def abs(self, x): x = self._ensure_tensor(x) out = cactus_node_t() @@ -136,6 +160,13 @@ def precision_cast(self, x, dtype): raise RuntimeError("graph_precision_cast failed") return self._tensor_from_node(out.value) + def quantize_activations(self, x): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_quantize_activations(self.h, cactus_node_t(x.id), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_quantize_activations failed") + return self._tensor_from_node(out.value) def _scalar(self, fn_name, x, value=None): x = self._ensure_tensor(x) @@ -161,6 +192,12 @@ def scalar_multiply(self, x, value): def scalar_divide(self, x, value): return self._scalar("cactus_graph_scalar_divide", x, value) + def scalar_floor_divide(self, x, value): + return self._scalar("cactus_graph_scalar_floor_divide", x, value) + + def scalar_not_equal(self, x, value): + return self._scalar("cactus_graph_scalar_not_equal", x, value) + def scalar_exp(self, x): return self._scalar("cactus_graph_scalar_exp", x) @@ -176,6 +213,39 @@ def scalar_sin(self, x): def scalar_log(self, x): return self._scalar("cactus_graph_scalar_log", x) + def clamp(self, x, lo, hi): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_clamp( + self.h, + cactus_node_t(x.id), + ctypes.c_float(float(lo)), + ctypes.c_float(float(hi)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_clamp failed")) + return self._tensor_from_node(out.value) + + def masked_select_prefix(self, x, mask): + return self._binary("cactus_graph_masked_select_prefix", x, mask) + + def masked_scatter(self, x, mask, source): + x = self._ensure_tensor(x) + mask = self._ensure_tensor(mask) + source = self._ensure_tensor(source) + out = cactus_node_t() + rc = _lib.cactus_graph_masked_scatter( + self.h, + cactus_node_t(x.id), + cactus_node_t(mask.id), + cactus_node_t(source.id), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_masked_scatter failed")) + return self._tensor_from_node(out.value) + def view(self, x, shape): x = self._ensure_tensor(x) shape = tuple(int(v) for v in shape) @@ -196,6 +266,16 @@ def reshape(self, x, shape): raise RuntimeError("graph_reshape failed") return self._tensor_from_node(out.value) + def expand(self, x, shape): + x = self._ensure_tensor(x) + shape = tuple(int(v) for v in shape) + arr = (ctypes.c_size_t * len(shape))(*shape) + out = cactus_node_t() + rc = _lib.cactus_graph_expand(self.h, cactus_node_t(x.id), arr, len(shape), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_expand failed") + return self._tensor_from_node(out.value) + def flatten(self, x, start_dim=0, end_dim=-1): x = self._ensure_tensor(x) out = cactus_node_t() @@ -269,27 +349,35 @@ def permute(self, x, permutation, backend=CPU): raise RuntimeError("graph_transpose_n failed") return self._tensor_from_node(out.value) - def matmul(self, a, b, pretransposed_rhs=False, backend=CPU): + def matmul(self, a, b, pretransposed_rhs=False, backend=CPU, output_dtype=None): a = self._ensure_tensor(a) b = self._ensure_tensor(b) out = cactus_node_t() + target_dtype = int(a.dtype if output_dtype is None else output_dtype) rc = _lib.cactus_graph_matmul( self.h, cactus_node_t(a.id), cactus_node_t(b.id), ctypes.c_bool(bool(pretransposed_rhs)), ctypes.c_int32(int(backend)), + ctypes.c_int32(target_dtype), ctypes.byref(out), ) if rc != 0: - raise RuntimeError("graph_matmul failed") + raise RuntimeError(_err("graph_matmul failed")) return self._tensor_from_node(out.value) - def gather(self, tensor, indices): + def gather(self, tensor, indices, axis=0): tensor = self._ensure_tensor(tensor) indices = self._ensure_tensor(indices) out = cactus_node_t() - rc = _lib.cactus_graph_gather(self.h, cactus_node_t(tensor.id), cactus_node_t(indices.id), ctypes.byref(out)) + rc = _lib.cactus_graph_gather( + self.h, + cactus_node_t(tensor.id), + cactus_node_t(indices.id), + ctypes.c_int32(int(axis)), + ctypes.byref(out), + ) if rc != 0: raise RuntimeError("graph_gather failed") return self._tensor_from_node(out.value) @@ -327,6 +415,12 @@ def mmap_weights(self, filename): raise RuntimeError("graph_mmap_weights failed") return self._tensor_from_node(out.value) + def bind_mmap_weights(self, tensor, filename): + tensor = self._ensure_tensor(tensor) + rc = _lib.cactus_graph_bind_mmap_weights(self.h, cactus_node_t(tensor.id), str(filename).encode()) + if rc != 0: + raise RuntimeError("graph_bind_mmap_weights failed") + def bilinear_interpolation(self, pos_embeds, dst_height, dst_width): pos_embeds = self._ensure_tensor(pos_embeds) out = cactus_node_t() @@ -337,6 +431,27 @@ def bilinear_interpolation(self, pos_embeds, dst_height, dst_width): raise RuntimeError("graph_bilinear_interpolation failed") return self._tensor_from_node(out.value) + def set_grouped_scales(self, tensor, group_size, num_groups, scales): + tensor = self._ensure_tensor(tensor) + arr = np.ascontiguousarray(scales, dtype=np.float16) + rc = _lib.cactus_graph_set_grouped_scales( + self.h, + cactus_node_t(tensor.id), + ctypes.c_size_t(int(group_size)), + ctypes.c_size_t(int(num_groups)), + arr.ctypes.data_as(ctypes.c_void_p), + ) + if rc != 0: + raise RuntimeError("graph_set_grouped_scales failed") + + def set_interleaved(self, tensor, interleaved=True, original_n=0): + tensor = self._ensure_tensor(tensor) + rc = _lib.cactus_graph_set_interleaved( + self.h, cactus_node_t(tensor.id), ctypes.c_bool(bool(interleaved)), ctypes.c_size_t(int(original_n)) + ) + if rc != 0: + raise RuntimeError("graph_set_interleaved failed") + def release_weight_pages(self, tensor): tensor = self._ensure_tensor(tensor) rc = _lib.cactus_graph_release_weight_pages(self.h, cactus_node_t(tensor.id)) @@ -373,18 +488,13 @@ def cat(self, tensors, axis=0): tensors = [self._ensure_tensor(t) for t in tensors] if not tensors: raise ValueError("cat requires at least one tensor") - ids = (cactus_node_t * len(tensors))(*(cactus_node_t(t.id) for t in tensors)) - out = cactus_node_t() - rc = _lib.cactus_graph_cat( - self.h, - ids, - ctypes.c_size_t(len(tensors)), - ctypes.c_int32(int(axis)), - ctypes.byref(out), - ) - if rc != 0: - raise RuntimeError("graph_cat failed") - return self._tensor_from_node(out.value) + if len(tensors) == 1: + return tensors[0] + + result = tensors[0] + for tensor in tensors[1:]: + result = self.concat(result, tensor, axis=axis) + return result def groupnorm(self, x, weight, bias, num_groups, eps=1e-5): x = self._ensure_tensor(x) @@ -524,6 +634,9 @@ def min(self, x, axis): def max(self, x, axis): return self._reduce("cactus_graph_max", x, axis) + + def cumsum(self, x, axis): + return self._reduce("cactus_graph_cumsum", x, axis) def softmax(self, x, axis=-1): x = self._ensure_tensor(x) @@ -567,6 +680,98 @@ def attention(self, query, key, value, scale, is_causal=True, position_offset=0, raise RuntimeError("graph_attention failed") return self._tensor_from_node(out.value) + def kv_cache_state(self, max_seq_len, num_kv_heads, head_dim, window_size=0, sink_size=0): + out = cactus_node_t() + rc = _lib.cactus_graph_kv_cache_state( + self.h, + ctypes.c_size_t(int(max_seq_len)), + ctypes.c_size_t(int(num_kv_heads)), + ctypes.c_size_t(int(head_dim)), + ctypes.c_size_t(int(window_size)), + ctypes.c_size_t(int(sink_size)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_kv_cache_state failed")) + return self._tensor_from_node(out.value) + + def kv_cache_append(self, new_kv, cache_state, window_size=0, sink_size=0): + new_kv = self._ensure_tensor(new_kv) + cache_state = self._ensure_tensor(cache_state) + out = cactus_node_t() + rc = _lib.cactus_graph_kv_cache_append( + self.h, + cactus_node_t(new_kv.id), + cactus_node_t(cache_state.id), + ctypes.c_size_t(int(window_size)), + ctypes.c_size_t(int(sink_size)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_kv_cache_append failed")) + return self._tensor_from_node(out.value) + + def attention_cached( + self, + query, + key_new, + value_new, + k_cache_state, + v_cache_state, + scale, + position_offset=0, + window_size=0, + v_head_dim=0, + ): + query = self._ensure_tensor(query) + key_new = self._ensure_tensor(key_new) + value_new = self._ensure_tensor(value_new) + k_cache_state = self._ensure_tensor(k_cache_state) + v_cache_state = self._ensure_tensor(v_cache_state) + out = cactus_node_t() + rc = _lib.cactus_graph_attention_cached( + self.h, + cactus_node_t(query.id), + cactus_node_t(key_new.id), + cactus_node_t(value_new.id), + cactus_node_t(k_cache_state.id), + cactus_node_t(v_cache_state.id), + ctypes.c_float(float(scale)), + ctypes.c_size_t(int(position_offset)), + ctypes.c_size_t(int(window_size)), + ctypes.c_size_t(int(v_head_dim)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_attention_cached failed")) + return self._tensor_from_node(out.value) + + def conv_cache_state(self, window_size, hidden_dim): + out = cactus_node_t() + rc = _lib.cactus_graph_conv_cache_state( + self.h, + ctypes.c_size_t(int(window_size)), + ctypes.c_size_t(int(hidden_dim)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_conv_cache_state failed")) + return self._tensor_from_node(out.value) + + def conv_cache_append(self, new_data, cache_state): + new_data = self._ensure_tensor(new_data) + cache_state = self._ensure_tensor(cache_state) + out = cactus_node_t() + rc = _lib.cactus_graph_conv_cache_append( + self.h, + cactus_node_t(new_data.id), + cactus_node_t(cache_state.id), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_conv_cache_append failed")) + return self._tensor_from_node(out.value) + def rel_pos_bias(self, query, relative_key, scale): query = self._ensure_tensor(query) relative_key = self._ensure_tensor(relative_key) @@ -719,6 +924,21 @@ def conv2d_depthwise_k3s2p1(self, x, weight, bias=None): def conv2d_pointwise_1x1(self, x, weight, bias=None): return self._conv_with_optional_bias("cactus_graph_conv2d_pointwise_1x1", x, weight, bias) + def conv2d_k3s1p1(self, x, weight, bias=None): + return self._conv_with_optional_bias("cactus_graph_conv2d_k3s1p1", x, weight, bias) + + def conv2d(self, x, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): + return self._conv_with_optional_bias( + "cactus_graph_conv2d", + x, + weight, + bias, + ctypes.c_size_t(int(stride)), + ctypes.c_size_t(int(padding)), + ctypes.c_size_t(int(dilation)), + ctypes.c_size_t(int(groups)), + ) + def _conv_with_optional_bias(self, fn_name, x, weight, bias=None, *extra): x = self._ensure_tensor(x) weight = self._ensure_tensor(weight) @@ -822,6 +1042,25 @@ def moe_layer_gated(self, hidden, routing_probs, topk_indices, w1_weights, w3_we raise RuntimeError("graph_moe_layer_gated failed") return self._tensor_from_node(out.value) + def dense_mlp_tq_fused(self, hidden, gate_weight, up_weight, down_weight, product_scale=1.0): + hidden = self._ensure_tensor(hidden) + gate_weight = self._ensure_tensor(gate_weight) + up_weight = self._ensure_tensor(up_weight) + down_weight = self._ensure_tensor(down_weight) + out = cactus_node_t() + rc = _lib.cactus_graph_dense_mlp_tq_fused( + self.h, + cactus_node_t(hidden.id), + cactus_node_t(gate_weight.id), + cactus_node_t(up_weight.id), + cactus_node_t(down_weight.id), + ctypes.c_float(float(product_scale)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_dense_mlp_tq_fused failed")) + return self._tensor_from_node(out.value) + def moe_layer_ungated(self, hidden, routing_probs, topk_indices, w1_weights, w2_weights, num_experts, num_experts_per_tok, normalize_routing=True, epsilon=1e-6, routed_scaling_factor=1.0, activation=ACT_GELU): @@ -934,6 +1173,8 @@ def _coerce_input_array(self, data, precision): arr = np.ascontiguousarray(arr, dtype=np.float16) elif precision == self.FP32: arr = np.ascontiguousarray(arr, dtype=np.float32) + elif precision in (self.INT4, self.CQ2, self.CQ3, self.CQ4): + arr = np.ascontiguousarray(arr, dtype=np.uint8) else: raise RuntimeError("unsupported precision") return arr @@ -958,6 +1199,11 @@ def __mul__(self, other): def __truediv__(self, other): return self.g.divide(self, other) + def __ne__(self, other): + if isinstance(other, Tensor): + return self.g.not_equal(self, other) + return self.g.scalar_not_equal(self, other) + def abs(self): return self.g.abs(self) @@ -967,6 +1213,8 @@ def pow(self, exponent): def precision_cast(self, dtype): return self.g.precision_cast(self, dtype) + def quantize_activations(self): + return self.g.quantize_activations(self) def scalar_add(self, value): return self.g.scalar_add(self, value) @@ -980,6 +1228,12 @@ def scalar_multiply(self, value): def scalar_divide(self, value): return self.g.scalar_divide(self, value) + def scalar_floor_divide(self, value): + return self.g.scalar_floor_divide(self, value) + + def scalar_not_equal(self, value): + return self.g.scalar_not_equal(self, value) + def scalar_exp(self): return self.g.scalar_exp(self) @@ -995,6 +1249,9 @@ def scalar_sin(self): def scalar_log(self): return self.g.scalar_log(self) + def clamp(self, lo, hi): + return self.g.clamp(self, lo, hi) + def relu(self): return self.g.relu(self) @@ -1019,6 +1276,9 @@ def view(self, shape): def reshape(self, shape): return self.g.reshape(self, shape) + def expand(self, shape): + return self.g.expand(self, shape) + def flatten(self, start_dim=0, end_dim=-1): return self.g.flatten(self, start_dim=start_dim, end_dim=end_dim) @@ -1067,8 +1327,14 @@ def softmax(self, axis=-1): def glu(self, axis=-1): return self.g.glu(self, axis=axis) - def matmul(self, other, pretransposed_rhs=False, backend=Graph.CPU): - return self.g.matmul(self, other, pretransposed_rhs=pretransposed_rhs, backend=backend) + def matmul(self, other, pretransposed_rhs=False, backend=Graph.CPU, output_dtype=None): + return self.g.matmul( + self, + other, + pretransposed_rhs=pretransposed_rhs, + backend=backend, + output_dtype=output_dtype, + ) def sum(self, axis): return self.g.sum(self, axis) @@ -1085,6 +1351,9 @@ def min(self, axis): def max(self, axis): return self.g.max(self, axis) + def cumsum(self, axis): + return self.g.cumsum(self, axis) + def numpy(self): info = cactus_tensor_info_t() rc = _lib.cactus_graph_get_output_info(self.g.h, cactus_node_t(self.id), ctypes.byref(info)) @@ -1107,6 +1376,9 @@ def numpy(self): arr = np.ctypeslib.as_array((ctypes.c_float * num_elements).from_address(out_ptr.value)) elif precision == Graph.INT8: arr = np.ctypeslib.as_array((ctypes.c_int8 * num_elements).from_address(out_ptr.value)) + elif precision in (Graph.INT4, Graph.CQ2, Graph.CQ3, Graph.CQ4): + arr = np.ctypeslib.as_array((ctypes.c_uint8 * int(info.byte_size)).from_address(out_ptr.value)) + return arr.copy() else: raise RuntimeError("unsupported precision") diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index ab40a4605..a526946d6 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -13,6 +13,7 @@ from .convert import cmd_convert from .eval import cmd_eval from .misc import cmd_auth, cmd_clean, cmd_list +from .transpile import cmd_run_transpiled def create_parser(): @@ -62,7 +63,7 @@ def create_parser(): ----------------------------------------------------------------- - cactus download downloads CQ model weights + cactus download downloads CQ model files auto-resolves Cactus-Compute CQ repo Optional flags: @@ -75,6 +76,7 @@ def create_parser(): ----------------------------------------------------------------- cactus convert [output_dir] converts HuggingFace model to CQ format + and transpiles the graph into the same folder Optional flags: --bits 1|2|3|4 CQ quantization bits (default: 4) @@ -82,6 +84,11 @@ def create_parser(): ----------------------------------------------------------------- + cactus run runs a transpiled Cactus bundle + cactus run-transpiled explicit transpiled bundle runner + + ----------------------------------------------------------------- + cactus build builds cactus for ARM chips output: build/libcactus.a @@ -155,7 +162,7 @@ def create_parser(): parser._action_groups = [] - download_parser = subparsers.add_parser('download', help='Download CQ model weights') + download_parser = subparsers.add_parser('download', help='Download CQ model files') download_parser.add_argument('model_id', nargs='?', default=DEFAULT_MODEL_ID, help=f'HuggingFace model ID or Cactus-Compute CQ repo (default: {DEFAULT_MODEL_ID})') download_parser.add_argument('--language-bits', type=int, choices=[1, 2, 3, 4], default=4, @@ -192,13 +199,55 @@ def create_parser(): help='Path to image file for VLM inference (attached to first message)') run_parser.add_argument('--audio', help='Path to audio file (WAV) for audio chat (attached to first message)') + run_parser.add_argument('--file', dest='audio_file', default=None, + help='Audio file for transpiled bundles when model_id points to a transpiled folder') + run_parser.add_argument('--image-file', action='append', default=[], + help='Repeatable image input for transpiled bundles') + run_parser.add_argument('--weights-dir', default=None, + help='Converted weights directory for transpiled bundles with bound weights') run_parser.add_argument('--system', help='System prompt to prepend to all messages') run_parser.add_argument('--prompt', help='Initial prompt to send immediately') + run_parser.add_argument('--input-ids', default=None, + help='Comma-separated token ids for transpiled causal-LM bundles') + run_parser.add_argument('--max-new-tokens', type=int, default=None, + help='Maximum tokens to generate for transpiled causal-LM bundles') + run_parser.add_argument('--stop-sequence', action='append', default=[], + help='Optional stop sequence for transpiled causal-LM bundles. Repeatable.') + run_parser.add_argument('--result-json', default=None, + help='Optional path to save transpiled bundle results as JSON') run_parser.add_argument('--thinking', action='store_true', help='Enable thinking/reasoning for models that support it') + run_transpiled_parser = subparsers.add_parser('run-transpiled', help='Run a saved transpiled component bundle') + run_transpiled_parser.add_argument('bundle_dir', + help='Bundle root directory or manifest.json path from the transpiler') + run_transpiled_parser.add_argument('--file', dest='audio_file', default=None, + help='Audio file for transcription bundles such as parakeet-tdt') + run_transpiled_parser.add_argument('--audio', default=None, + help='Alias for --file, matching cactus run') + run_transpiled_parser.add_argument('--image', default=None, + help='Image file for multimodal transpiled bundles') + run_transpiled_parser.add_argument('--image-file', action='append', default=[], + help='Repeatable image input for multimodal transpiled bundles') + run_transpiled_parser.add_argument('--weights-dir', default=None, + help='Optional converted weights directory for bound-weight bundles') + run_transpiled_parser.add_argument('--prompt', default=None, + help='Prompt for causal-LM bundles') + run_transpiled_parser.add_argument('--input-ids', default=None, + help='Comma-separated token ids for causal-LM bundles') + run_transpiled_parser.add_argument('--max-new-tokens', type=int, default=None, + help='Maximum tokens to generate for causal-LM bundles') + run_transpiled_parser.add_argument('--stop-sequence', action='append', default=[], + help='Optional stop sequence for causal-LM bundles. Repeatable.') + run_transpiled_parser.add_argument('--result-json', default=None, + help='Optional path to save the result payload as JSON') + run_transpiled_parser.add_argument('--system', default=None, + help='Optional system prompt for multimodal transpiled bundles') + run_transpiled_parser.add_argument('--thinking', action='store_true', + help='Enable thinking/reasoning markers when supported by the transpiled bundle') + transcribe_parser = subparsers.add_parser('transcribe', help='Download ASR model and run transcription') transcribe_parser.add_argument('model_id', nargs='?', default=DEFAULT_ASR_MODEL_ID, help=f'HuggingFace model ID (default: {DEFAULT_ASR_MODEL_ID})') @@ -275,6 +324,29 @@ def create_parser(): help='CQ quantization bits (default: 4)') convert_parser.add_argument('--cache-dir', help='Cache directory for HuggingFace models') convert_parser.add_argument('--token', help='HuggingFace API token') + convert_parser.add_argument('--task', default='auto', + choices=['auto', 'causal_lm_logits', 'multimodal_causal_lm_logits', + 'ctc_logits', 'encoder_hidden_states', + 'seq2seq_transcription', 'tdt_transcription'], + help='Transpile task after conversion (default: auto)') + convert_parser.add_argument('--prompt', + help='Prompt used for causal or multimodal graph shape capture') + convert_parser.add_argument('--system-prompt', default='', + help='Optional system prompt for multimodal prompt construction') + convert_parser.add_argument('--image-file', action='append', default=[], + help='Representative image file for multimodal transpile') + convert_parser.add_argument('--audio-file', + help='Representative audio file for audio/multimodal transpile') + convert_parser.add_argument('--max-new-tokens', type=int, default=32, + help='Generation room to preallocate for causal decode graphs') + convert_parser.add_argument('--component-pipeline', default='auto', choices=['auto', 'on', 'off'], + help='Use split component graph transpilation when supported') + convert_parser.add_argument('--components', + help='Comma-separated component subset for component-pipeline models') + convert_parser.add_argument('--trust-remote-code', action='store_true', + help='Allow HF remote code during the transpile phase') + convert_parser.add_argument('--local-files-only', action='store_true', + help='Require HF model/processor files to already be local during transpile') return parser @@ -305,6 +377,8 @@ def main(): sys.exit(cmd_build(args)) elif args.command == 'run': sys.exit(cmd_run(args)) + elif args.command == 'run-transpiled': + sys.exit(cmd_run_transpiled(args)) elif args.command == 'transcribe': sys.exit(cmd_transcribe(args)) elif args.command == 'test': diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index 1b1ac418b..6af0c17ec 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -1,30 +1,248 @@ +from __future__ import annotations + +import argparse +import shutil +from dataclasses import dataclass +from pathlib import Path + from .common import ( + GREEN, + PROJECT_ROOT, + RED, + YELLOW, get_weights_dir, print_color, - RED, GREEN, YELLOW, ) +from .transpile import cmd_transpile, resolve_model_id_alias +from cactus.transpile.component_plan import infer_component_plan_from_output + + +_DEFAULT_MULTIMODAL_PROMPT = ( + "Respond with 2 lines. The first should be a description of the image, " + "and the second should be a transcription of the audio" +) +_DEFAULT_TEXT_PROMPT = "Hello" + + +@dataclass(frozen=True) +class _ConvertTranspileSpec: + task: str + components: tuple[str, ...] = () + needs_image: bool = False + needs_audio: bool = False + force_component_pipeline: bool = False + + +def _default_multimodal_asset_args() -> tuple[list[str], str | None]: + assets_dir = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" + image_file = assets_dir / "test_monkey.png" + audio_file = assets_dir / "test.wav" + image_args = [str(image_file)] if image_file.exists() else [] + audio_arg = str(audio_file) if audio_file.exists() else None + return image_args, audio_arg + + +def _default_audio_asset_arg() -> str | None: + _, audio_file = _default_multimodal_asset_args() + return audio_file + + +def _transpile_spec_for_convert(*, task: str, plan) -> _ConvertTranspileSpec: + if task != "auto": + if plan is not None and task == plan.task: + return _ConvertTranspileSpec( + task=task, + components=tuple(plan.components or ()), + needs_image=bool(plan.needs_image), + needs_audio=bool(plan.needs_audio), + force_component_pipeline=bool(plan.force_component_pipeline), + ) + return _ConvertTranspileSpec( + task=task, + needs_image=task == "multimodal_causal_lm_logits", + needs_audio=task + in {"tdt_transcription", "seq2seq_transcription", "ctc_logits", "encoder_hidden_states", "multimodal_causal_lm_logits"}, + force_component_pipeline=task in {"tdt_transcription", "seq2seq_transcription", "multimodal_causal_lm_logits"}, + ) + + if plan is None: + return _ConvertTranspileSpec(task="causal_lm_logits") + + return _ConvertTranspileSpec( + task=plan.task, + components=tuple(plan.components or ()), + needs_image=bool(plan.needs_image), + needs_audio=bool(plan.needs_audio), + force_component_pipeline=bool(plan.force_component_pipeline), + ) + + +def _remove_stale_transpile_artifacts(output_dir: str | Path) -> None: + root = Path(output_dir) + for relative in ( + "components", + "transpile_entrypoints.json", + "raw_ir.json", + "optimized_ir.json", + "graph.cactus", + "graph_bindings.json", + "result.json", + ): + path = root / relative + if path.is_dir(): + shutil.rmtree(path) + elif path.exists(): + path.unlink() + for pattern in ("raw_ir_*.json", "optimized_ir_*.json"): + for path in root.glob(pattern): + if path.is_file(): + path.unlink() def cmd_convert(args): - """Convert a HuggingFace model to CQ format using the convert pipeline.""" - model_id = args.model_name + """Convert a HuggingFace model to CQ format and transpile it in place.""" + model_id = resolve_model_id_alias(args.model_name) output_dir = args.output_dir if output_dir is None: output_dir = str(get_weights_dir(model_id)) - bits = getattr(args, 'bits', 4) or 4 - token = getattr(args, 'token', None) - cache_dir = getattr(args, 'cache_dir', None) + + bits = getattr(args, "bits", 4) or 4 + token = getattr(args, "token", None) + cache_dir = getattr(args, "cache_dir", None) try: from ..convert.cli import main as cq_main - cq_args = ['convert', '--model', model_id, '--out', str(output_dir), '--bits', str(bits)] + + cq_args = [ + "convert", + "--model", + model_id, + "--out", + str(output_dir), + "--bits", + str(bits), + ] if token: - cq_args.extend(['--token', token]) + cq_args.extend(["--token", token]) if cache_dir: - cq_args.extend(['--cache-dir', cache_dir]) - cq_args.append('--force') + cq_args.extend(["--cache-dir", cache_dir]) + cq_args.append("--force") + cq_main(cq_args) - print_color(GREEN, f"Model converted to {output_dir}") + + task = getattr(args, "task", "auto") or "auto" + prompt = getattr(args, "prompt", None) + image_files = [str(path) for path in (getattr(args, "image_file", None) or []) if str(path).strip()] + audio_file = getattr(args, "audio_file", None) + + plan = infer_component_plan_from_output(str(output_dir), model_id=model_id) + spec = _transpile_spec_for_convert(task=task, plan=plan) + + _remove_stale_transpile_artifacts(output_dir) + + spec_prompt = prompt + spec_image_files = list(image_files) + spec_audio_file = audio_file + component_pipeline = getattr(args, "component_pipeline", "auto") or "auto" + components = getattr(args, "components", None) + + if spec_prompt is None and spec.task == "multimodal_causal_lm_logits": + spec_prompt = _DEFAULT_MULTIMODAL_PROMPT + elif spec_prompt is None and spec.task == "causal_lm_logits": + spec_prompt = _DEFAULT_TEXT_PROMPT + + needs_image = False + needs_audio = False + if spec.task == "multimodal_causal_lm_logits": + needs_image = bool(spec.needs_image) + needs_audio = bool(spec.needs_audio) + if not needs_image and not needs_audio: + needs_image = bool(spec_image_files) + needs_audio = bool(spec_audio_file) + if (needs_image and not spec_image_files) or (needs_audio and not spec_audio_file): + default_images, default_audio = _default_multimodal_asset_args() + if needs_image and not spec_image_files: + spec_image_files = default_images + if needs_audio and not spec_audio_file: + spec_audio_file = default_audio + print_color( + YELLOW, + "Multimodal transpile needs representative media shapes; " + "using bundled tiny test assets.", + ) + if needs_image and not spec_image_files: + print_color( + RED, + "Multimodal transpile requires --image-file for this model.", + ) + return 1 + if needs_audio and not spec_audio_file: + print_color( + RED, + "Multimodal transpile requires --audio-file for this model.", + ) + return 1 + if component_pipeline == "auto" and spec.force_component_pipeline: + component_pipeline = "on" + if components is None and spec.components: + components = ",".join(spec.components) + + used_default_audio = False + if spec.task in {"tdt_transcription", "seq2seq_transcription", "ctc_logits", "encoder_hidden_states"} and not spec_audio_file: + spec_audio_file = _default_audio_asset_arg() + used_default_audio = spec_audio_file is not None + if spec.task in {"tdt_transcription", "seq2seq_transcription", "ctc_logits", "encoder_hidden_states"} and used_default_audio: + print_color( + YELLOW, + f"{spec.task} transpile needs a representative audio shape; " + "using bundled tiny test audio asset.", + ) + elif spec.task in {"tdt_transcription", "seq2seq_transcription", "ctc_logits", "encoder_hidden_states"} and not spec_audio_file: + print_color(RED, f"{spec.task} transpile requires --audio-file.") + return 1 + + artifact_dir = Path(output_dir) + + extra_args = [ + "--weights-dir", + str(output_dir), + "--artifact-dir", + str(artifact_dir), + "--task", + spec.task, + "--max-new-tokens", + str(getattr(args, "max_new_tokens", 32) or 32), + "--component-pipeline", + component_pipeline, + ] + if spec_prompt is not None: + extra_args.extend(["--prompt", spec_prompt]) + if components: + extra_args.extend(["--components", str(components)]) + for image_file in spec_image_files: + extra_args.extend(["--image-file", image_file]) + if spec_audio_file: + extra_args.extend(["--audio-file", str(spec_audio_file)]) + if getattr(args, "system_prompt", None): + extra_args.extend(["--system-prompt", str(args.system_prompt)]) + if token: + extra_args.extend(["--token", token]) + if getattr(args, "trust_remote_code", False) or spec.task == "multimodal_causal_lm_logits": + extra_args.append("--trust-remote-code") + if getattr(args, "local_files_only", False): + extra_args.append("--local-files-only") + + transpile_args = argparse.Namespace( + model_id=model_id, + execute_after_transpile=False, + allow_unconverted_weights=False, + extra_args=extra_args, + ) + rc = cmd_transpile(transpile_args) + if rc != 0: + return rc + + print_color(GREEN, f"Model converted and transpiled to {output_dir}") return 0 except SystemExit as e: return e.code if e.code else 0 diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 7f74ad154..62f05021f 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -1,5 +1,6 @@ import os import platform +import sys from pathlib import Path from .common import ( @@ -12,8 +13,87 @@ from .download import cmd_download +def _transpiled_bundle_root_from_manifest(manifest_path: Path) -> Path: + if manifest_path.parent.name == "components": + return manifest_path.parent.parent + return manifest_path.parent + + +def _clear_terminal_for_chat() -> None: + if sys.stdout.isatty(): + os.system('clear' if platform.system() != 'Windows' else 'cls') + + +def _validate_image_path(image_path: str) -> str | None: + resolved = str(Path(image_path).expanduser().resolve()) + path = Path(resolved) + if not path.exists(): + print_color(RED, f"Error: Image file not found: {resolved}") + return None + valid_exts = {'.png', '.jpg', '.jpeg', '.bmp'} + if path.suffix.lower() not in valid_exts: + print_color(RED, f"Error: Unsupported image format. Supported: {', '.join(valid_exts)}") + return None + return resolved + + +def _validate_audio_path(audio_path: str) -> str | None: + resolved = str(Path(audio_path).expanduser().resolve()) + if not Path(resolved).exists(): + print_color(RED, f"Error: Audio file not found: {resolved}") + return None + return resolved + + +def _prepare_transpiled_run_args(args, *, manifest_path: Path) -> int: + args.bundle_dir = str(_transpiled_bundle_root_from_manifest(manifest_path)) + args._transpiled_from_run = True + + image_path = getattr(args, 'image', None) + if image_path: + resolved = _validate_image_path(image_path) + if resolved is None: + return 1 + args.image = resolved + + image_files = [] + for image_file in getattr(args, 'image_file', []) or []: + if not image_file: + continue + resolved = _validate_image_path(str(image_file)) + if resolved is None: + return 1 + image_files.append(resolved) + args.image_file = image_files + + audio_path = getattr(args, 'audio', None) or getattr(args, 'audio_file', None) + if audio_path: + resolved = _validate_audio_path(str(audio_path)) + if resolved is None: + return 1 + args.audio = resolved + args.audio_file = resolved + + return 0 + + def cmd_run(args): """Download model if needed and start interactive chat.""" + model_id = args.model_id + + from .transpile import _resolve_transpiled_manifest, cmd_run_transpiled + + manifest_path = _resolve_transpiled_manifest(model_id) + if manifest_path is not None: + if getattr(args, 'no_cloud_tele', False): + os.environ["CACTUS_NO_CLOUD_TELE"] = "1" + if _prepare_transpiled_run_args(args, manifest_path=manifest_path) != 0: + return 1 + _clear_terminal_for_chat() + print_color(GREEN, f"Starting Cactus Chat with model: {model_id}") + print() + return cmd_run_transpiled(args) + from .config_utils import CactusConfig from .common import prompt_for_api_key @@ -23,8 +103,6 @@ def cmd_run(args): if api_key: os.environ["CACTUS_CLOUD_KEY"] = api_key - model_id = args.model_id - if getattr(args, 'no_cloud_tele', False): os.environ["CACTUS_NO_CLOUD_TELE"] = "1" @@ -60,7 +138,7 @@ def cmd_run(args): print_color(RED, f"Error: {exc}") return 1 - os.system('clear' if platform.system() != 'Windows' else 'cls') + _clear_terminal_for_chat() print_color(GREEN, f"Starting Cactus Chat with model: {model_id}") print() diff --git a/python/cactus/cli/transpile.py b/python/cactus/cli/transpile.py new file mode 100644 index 000000000..afceb3359 --- /dev/null +++ b/python/cactus/cli/transpile.py @@ -0,0 +1,566 @@ +from __future__ import annotations + +import json +import os +import platform +import shutil +import subprocess +import sys +import time +from pathlib import Path + +from .common import ( + DEFAULT_MODEL_ID, + PROJECT_ROOT, + GREEN, + RED, + YELLOW, + print_color, +) +from .download import get_weights_dir + + +MODEL_ID_ALIASES = { + "gemma4": DEFAULT_MODEL_ID, + "gemma4-e2b": DEFAULT_MODEL_ID, + "parakeet": "nvidia/parakeet-tdt-0.6b-v3", + "parakeet-tdt": "nvidia/parakeet-tdt-0.6b-v3", + "whisper": "openai/whisper-small", + "qwen": "Qwen/Qwen3-1.7B", + "lfm": "LiquidAI/LFM2-VL-450M", +} + + +def resolve_model_id_alias(model_id: str) -> str: + normalized = (model_id or "").strip() + return MODEL_ID_ALIASES.get(normalized.lower(), normalized) + + +def _python_runtime_library_path() -> Path: + suffix = ".dylib" if platform.system() == "Darwin" else ".so" + return PROJECT_ROOT / "cactus" / "build" / f"libcactus{suffix}" + + +def _static_cactus_library_path() -> Path: + return PROJECT_ROOT / "cactus" / "build" / "libcactus.a" + + +def _build_static_cactus_library() -> Path: + build_script = PROJECT_ROOT / "cactus" / "build.sh" + if not build_script.exists(): + raise RuntimeError(f"The Cactus build script is missing: {build_script}") + + build = subprocess.run([str(build_script)], cwd=PROJECT_ROOT / "cactus") + if build.returncode != 0: + raise RuntimeError("Failed to build the Cactus static runtime") + + static_library_path = _static_cactus_library_path() + if not static_library_path.exists(): + raise RuntimeError( + "The Cactus build completed, but the static library was not produced.\n" + f"Expected: {static_library_path}" + ) + return static_library_path + + +def _public_cactus_api_symbols(static_library_path: Path) -> list[str]: + if platform.system() == "Darwin": + command = ["nm", "-gU", str(static_library_path)] + else: + command = ["nm", "-g", "--defined-only", str(static_library_path)] + + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError( + "Failed to inspect the Cactus static runtime symbols.\n" + f"Command: {' '.join(command)}\n" + f"{result.stderr.strip()}" + ) + + symbols: list[str] = [] + for line in result.stdout.splitlines(): + parts = line.split() + if not parts: + continue + symbol = parts[-1].strip() + normalized = symbol[1:] if symbol.startswith("_") else symbol + if normalized.startswith("cactus_") and symbol not in symbols: + symbols.append(symbol) + if not symbols: + raise RuntimeError(f"Could not find any public cactus_* symbols in {static_library_path}") + return symbols + + +def _link_python_runtime_library(*, static_library_path: Path, library_path: Path) -> None: + build_dir = library_path.parent + build_dir.mkdir(parents=True, exist_ok=True) + if library_path.exists(): + library_path.unlink() + + exported_symbols = _public_cactus_api_symbols(static_library_path) + if platform.system() == "Darwin": + compiler = shutil.which("clang++") or shutil.which("c++") + if not compiler: + raise RuntimeError("Failed to find a C++ compiler for linking libcactus.dylib") + command = [ + compiler, + "-dynamiclib", + "-o", + str(library_path), + *[f"-Wl,-u,{symbol}" for symbol in exported_symbols], + str(static_library_path), + "-Wl,-install_name,@rpath/libcactus.dylib", + "-lcurl", + "-framework", + "Accelerate", + "-framework", + "CoreML", + "-framework", + "Foundation", + "-framework", + "Security", + "-framework", + "SystemConfiguration", + "-framework", + "CFNetwork", + ] + else: + compiler = shutil.which("g++") or shutil.which("c++") + if not compiler: + raise RuntimeError("Failed to find a C++ compiler for linking libcactus.so") + command = [ + compiler, + "-shared", + "-o", + str(library_path), + *[f"-Wl,--undefined={symbol}" for symbol in exported_symbols], + str(static_library_path), + "-lcurl", + "-pthread", + "-ldl", + "-lm", + ] + + result = subprocess.run(command, cwd=build_dir) + if result.returncode != 0 or not library_path.exists(): + raise RuntimeError(f"Failed to link the Cactus shared runtime: {library_path}") + + +def _ensure_python_runtime_library() -> Path: + library_path = _python_runtime_library_path() + static_library_path = _static_cactus_library_path() + if ( + library_path.exists() + and static_library_path.exists() + and library_path.stat().st_mtime >= static_library_path.stat().st_mtime + ): + return library_path + + print_color(YELLOW, "Preparing Cactus shared runtime for transpiler...") + if not static_library_path.exists(): + static_library_path = _build_static_cactus_library() + _link_python_runtime_library(static_library_path=static_library_path, library_path=library_path) + return library_path + + +def _weights_dir_looks_transpile_ready(weights_dir: Path) -> bool: + root = Path(weights_dir).expanduser().resolve() + if not root.exists() or not root.is_dir(): + return False + if (root / "weights_manifest.json").exists(): + return True + if any(root.glob("*.cq[1-4].weights")): + return True + # Legacy converted Cactus weight folders may not have a manifest yet, but + # they should still carry runtime config plus Cactus tensor files. + return (root / "config.txt").exists() and any(root.glob("*.weights")) + + +def _extra_args_has_option(extra_args: list[str], option: str) -> bool: + prefix = f"{option}=" + return any(arg == option or arg.startswith(prefix) for arg in extra_args) + + +def _resolve_transpiled_manifest(path_value: str | os.PathLike[str] | None) -> Path | None: + if not path_value: + return None + candidate = Path(path_value).expanduser().resolve() + if not candidate.exists(): + return None + if candidate.is_file() and candidate.name == "manifest.json": + return candidate + for manifest in ( + candidate / "components" / "manifest.json", + candidate / "manifest.json", + ): + if manifest.exists(): + return manifest + return None + + +def _prepend_python_path(env: dict[str, str]) -> None: + python_root = str(PROJECT_ROOT / "python") + existing = env.get("PYTHONPATH") + env["PYTHONPATH"] = python_root if not existing else f"{python_root}{os.pathsep}{existing}" + + +def cmd_transpile(args) -> int: + """Invoke the saved-model transpiler entrypoint.""" + model_id = resolve_model_id_alias(args.model_id) + extra_args = list(getattr(args, "extra_args", []) or []) + allow_unconverted = bool(getattr(args, "allow_unconverted_weights", False)) + + command = [sys.executable, "-m", "cactus.transpile.hf_model", "--model-id", model_id] + if _extra_args_has_option(extra_args, "--weights-dir"): + pass + else: + default_weights_dir = get_weights_dir(model_id) + if _weights_dir_looks_transpile_ready(default_weights_dir): + command.extend(["--weights-dir", str(default_weights_dir)]) + elif not allow_unconverted: + print_color( + RED, + "Error: transpilation requires converted Cactus CQ weights.", + ) + print_color( + YELLOW, + "Run conversion first, then retry:\n" + f" cactus convert {model_id} {default_weights_dir} --bits 4\n" + f" cactus transpile {model_id} --weights-dir {default_weights_dir}", + ) + return 1 + + if allow_unconverted: + command.append("--allow-unconverted-weights") + if not getattr(args, "execute_after_transpile", False) and "--skip-execute" not in extra_args: + command.append("--skip-execute") + command.extend(extra_args) + + try: + transpile_lib = _ensure_python_runtime_library() + except RuntimeError as exc: + print_color(RED, f"Error: {exc}") + return 1 + + env = os.environ.copy() + env["CACTUS_LIB_PATH"] = str(transpile_lib) + _prepend_python_path(env) + result = subprocess.run(command, cwd=PROJECT_ROOT, env=env) + return result.returncode + + +def cmd_run_transpiled(args) -> int: + """Run a saved transpiled component bundle.""" + try: + transpile_lib = _ensure_python_runtime_library() + except RuntimeError as exc: + print_color(RED, f"Error: {exc}") + return 1 + + os.environ["CACTUS_LIB_PATH"] = str(transpile_lib) + python_root = str(PROJECT_ROOT / "python") + if python_root not in sys.path: + sys.path.insert(0, python_root) + + from cactus.transpile.component_bundle_runtime import run_transpiled_bundle + + if getattr(args, "_transpiled_from_run", False): + return _run_transpiled_interactive_chat(args, run_transpiled_bundle) + + bundle_dir = getattr(args, "bundle_dir", None) or getattr(args, "model_id", None) + result = _run_transpiled_once(args, run_transpiled_bundle, bundle_dir=bundle_dir) + print(json.dumps(result, indent=2, sort_keys=True)) + if getattr(args, "result_json", None): + result_path = Path(args.result_json).expanduser().resolve() + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print_color(GREEN, f"Saved result to {result_path}") + return 0 + + +def _run_transpiled_once(args, run_transpiled_bundle, *, bundle_dir: str | os.PathLike[str] | None) -> dict[str, object]: + image_values: list[str] = [] + seen_images: set[str] = set() + + def add_image(value: object) -> None: + if not isinstance(value, str) or not value.strip(): + return + normalized = str(Path(value.strip()).expanduser().resolve()) + if normalized in seen_images: + return + seen_images.add(normalized) + image_values.append(normalized) + + for attr_name in ("image_file", "image_files"): + value = getattr(args, attr_name, None) + if isinstance(value, str) and value.strip(): + add_image(value) + elif isinstance(value, (list, tuple)): + for item in value: + add_image(str(item)) + image_arg = getattr(args, "image", None) + if isinstance(image_arg, str) and image_arg.strip(): + add_image(image_arg) + + return run_transpiled_bundle( + bundle_dir, + audio_file=getattr(args, "audio_file", None) or getattr(args, "audio", None), + image_files=tuple(image_values), + prompt=getattr(args, "prompt", None), + input_ids=getattr(args, "input_ids", None), + weights_dir=getattr(args, "weights_dir", None), + system_prompt=getattr(args, "system", None), + enable_thinking=bool(getattr(args, "thinking", False)), + max_new_tokens=getattr(args, "max_new_tokens", None), + stop_sequences=tuple(getattr(args, "stop_sequence", []) or ()), + ) + + +def _run_transpiled_interactive_chat(args, run_transpiled_bundle) -> int: + bundle_dir = getattr(args, "bundle_dir", None) or getattr(args, "model_id", None) + print(f"Loading model from {bundle_dir}...") + try: + from cactus.transpile.component_bundle_runtime import _default_weights_dir_for_manifest + from cactus.transpile.component_bundle_runtime import load_component_bundle_manifest + from cactus.transpile.component_bundle_runtime import load_saved_component_graphs + from cactus.transpile.component_bundle_runtime import runtime_include_components_for_manifest + + _, manifest = load_component_bundle_manifest(bundle_dir) + resolved_weights_dir = _default_weights_dir_for_manifest( + manifest, + explicit=getattr(args, "weights_dir", None), + ) + include_components = runtime_include_components_for_manifest( + family=str(manifest.get("family", "") or ""), + task=str(manifest.get("task", "") or ""), + manifest=manifest, + ) + if ( + str(manifest.get("family", "") or "").strip().lower() == "gemma4" + and str(manifest.get("task", "") or "") == "multimodal_causal_lm_logits" + and not getattr(args, "image", None) + and not (getattr(args, "image_file", None) or []) + and not (getattr(args, "audio_file", None) or getattr(args, "audio", None)) + ): + manifest_components = { + str(component_entry.get("component", "")).strip() + for component_entry in manifest.get("components", []) + if isinstance(component_entry, dict) + } + if {"lm_encoder_step", "decoder_step"}.issubset(manifest_components): + include_components = {"lm_encoder_step", "decoder_step"} + load_saved_component_graphs( + bundle_dir, + weights_dir=resolved_weights_dir, + include_components=include_components, + ) + except Exception as exc: + print(f"Warning: deferred graph load until first turn ({exc})", file=sys.stderr) + print("Model loaded.") + print("Commands: /image [prompt], /audio [prompt], /clear, reset, exit\n") + + history: list[tuple[str, str]] = [] + current_image = getattr(args, "image", None) + image_files = list(getattr(args, "image_file", []) or []) + if current_image: + image_files.append(str(current_image)) + current_image = image_files[-1] if image_files else None + current_audio = getattr(args, "audio_file", None) or getattr(args, "audio", None) + initial_prompt = getattr(args, "prompt", None) + auto_send = bool(initial_prompt or current_image or current_audio) + + while True: + if auto_send: + auto_send = False + user_input = initial_prompt or "Describe the attached input." + print(f"You: {user_input}") + else: + try: + user_input = input("You: ") + except EOFError: + break + + user_input = user_input.rstrip(" \t") + if not user_input: + continue + if user_input in {"exit", "quit"}: + break + if user_input == "reset": + history.clear() + current_image = None + current_audio = None + print("Conversation reset.") + continue + if user_input == "/clear": + current_image = None + current_audio = None + print("Attachments cleared.") + continue + + parsed = _parse_transpiled_attachment_command(user_input, "/image ") + if parsed is not None: + path_value, remainder = parsed + if not Path(path_value).expanduser().exists(): + print(f"File not found: {path_value}", file=sys.stderr) + continue + current_image = str(Path(path_value).expanduser().resolve()) + if not remainder: + print(f"Image attached: {current_image}") + continue + user_input = remainder + + parsed = _parse_transpiled_attachment_command(user_input, "/audio ") + if parsed is not None: + path_value, remainder = parsed + if not Path(path_value).expanduser().exists(): + print(f"File not found: {path_value}", file=sys.stderr) + continue + current_audio = str(Path(path_value).expanduser().resolve()) + if not remainder: + print(f"Audio attached: {current_audio}") + continue + user_input = remainder + + if current_image: + print(f"[image: {current_image}]") + if current_audio: + print(f"[audio: {current_audio}]") + print("Assistant: ", end="", flush=True) + + # Text-only turns use the full chat history through the cached decode path. + # Multimodal graphs are statically shaped, so keep media turns focused on + # the current attachment instead of replaying a long text transcript. + prompt_history = _history_for_transpiled_turn( + history, + has_media=bool(current_image or current_audio), + ) + prompt = _build_transpiled_interactive_prompt( + system_prompt=getattr(args, "system", None), + history=prompt_history, + user_input=user_input, + ) + turn_args = _copy_namespace_for_transpiled_turn( + args, + prompt=prompt, + image=current_image, + audio=current_audio, + ) + + started = time.perf_counter() + try: + result = _run_transpiled_once(turn_args, run_transpiled_bundle, bundle_dir=bundle_dir) + except Exception as exc: + print() + print(f"Error: {exc}", file=sys.stderr) + continue + elapsed_ms = max(0.0, (time.perf_counter() - started) * 1000.0) + + response_text = _transpiled_response_text(result) + if response_text: + print(response_text) + else: + print(json.dumps(result, indent=2, sort_keys=True)) + + _print_transpiled_interactive_stats(result, elapsed_ms=elapsed_ms) + print() + + history.append(("user", user_input)) + if response_text: + history.append(("assistant", response_text)) + current_image = None + current_audio = None + + print("Goodbye.") + return 0 + + +def _copy_namespace_for_transpiled_turn(args, *, prompt: str, image: str | None, audio: str | None): + import argparse + + values = vars(args).copy() + values["prompt"] = prompt + values["image"] = image + values["image_file"] = [image] if image else [] + values["audio"] = audio + values["audio_file"] = audio + return argparse.Namespace(**values) + + +def _parse_transpiled_attachment_command(text: str, prefix: str) -> tuple[str, str] | None: + if not text.startswith(prefix): + return None + rest = text[len(prefix):].strip() + if not rest: + return "", "" + split = rest.find(" ") + if split < 0: + return rest, "" + return rest[:split], rest[split + 1 :].strip() + + +def _history_for_transpiled_turn( + history: list[tuple[str, str]], + *, + has_media: bool, +) -> list[tuple[str, str]]: + return [] if has_media else history + + +def _build_transpiled_interactive_prompt( + *, + system_prompt: str | None, + history: list[tuple[str, str]], + user_input: str, +) -> str: + if not system_prompt and not history: + return user_input + + turns: list[str] = [] + if system_prompt: + turns.append(f"System: {system_prompt}") + for role, content in history: + label = "User" if role == "user" else "Assistant" + turns.append(f"{label}: {content}") + turns.append(f"User: {user_input}") + turns.append("Assistant:") + return "\n".join(turns) + + +def _transpiled_response_text(result: dict[str, object]) -> str: + for key in ("response", "transcript"): + value = result.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + next_token = result.get("next_token") + if isinstance(next_token, str) and next_token: + return next_token + return "" + + +def _print_transpiled_interactive_stats(result: dict[str, object], *, elapsed_ms: float) -> None: + token_count = int( + result.get("decode_tokens", 0) + or result.get("decoder_steps", 0) + or len(result.get("generated_token_ids", []) or []) + or len(result.get("token_ids", []) or []) + ) + if token_count <= 0: + token_count = 1 if _transpiled_response_text(result) else 0 + ttft_s = float(result.get("time_to_first_token_ms", 0.0) or 0.0) / 1000.0 + total_s = float(result.get("total_ms", elapsed_ms) or elapsed_ms) / 1000.0 + tps = float(result.get("decode_tps", 0.0) or 0.0) + if tps <= 0.0 and total_s > 0.0: + tps = token_count / total_s + print( + f"[{token_count} tokens | latency: {ttft_s:.3f}s | " + f"total: {total_s:.3f}s | {tps:.1f} tok/s]" + ) + + +def _print_transpiled_run_result(result: dict[str, object]) -> None: + response_text = _transpiled_response_text(result) + if response_text: + print(response_text) + return + + print(json.dumps(result, indent=2, sort_keys=True)) diff --git a/python/cactus/convert/cli.py b/python/cactus/convert/cli.py index ee8375b8c..6b3d60971 100644 --- a/python/cactus/convert/cli.py +++ b/python/cactus/convert/cli.py @@ -31,6 +31,7 @@ from .model_adapters.detection import SUPPORTED_FAMILIES, detect_family from .model_adapters.adapters import adapter_for_family from .quantization.cq import quantize_hadamard, quantize_orthogonal, write_cq_tensor +from .compat import patch_transformers_import_compat ALPHA = 0.25 @@ -45,6 +46,8 @@ def _load_hf(model_id_or_path: str, device: str): import logging, warnings logging.getLogger("transformers").setLevel(logging.ERROR) warnings.filterwarnings("ignore", message=".*You are using a model of type.*") + for note in patch_transformers_import_compat(): + print(f"note={note}") try: from transformers import AutoConfig, AutoModel except Exception as exc: # pragma: no cover @@ -379,7 +382,7 @@ def convert(args: argparse.Namespace) -> None: model_config = adapter.runtime_config(cfg) model_config["model_type"] = adapter.runtime_model_type() write_config_txt({**_config_dict(cfg), **model_config}, out_dir) - copy_runtime_files(args.model, out_dir) + copy_runtime_files(args.model, out_dir, token=getattr(args, "token", None), cache_dir=_hf_cache_dir()) try: from .cactus_adapters.tokenizer import convert_hf_tokenizer @@ -541,7 +544,7 @@ def convert(args: argparse.Namespace) -> None: def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="python -m src.convert") + parser = argparse.ArgumentParser(prog="python -m cactus.convert") sub = parser.add_subparsers(dest="command", required=True) p = sub.add_parser("convert") p.add_argument("--model", required=True) diff --git a/python/cactus/convert/compat.py b/python/cactus/convert/compat.py new file mode 100644 index 000000000..99d4fb5ec --- /dev/null +++ b/python/cactus/convert/compat.py @@ -0,0 +1,34 @@ +from __future__ import annotations + + +def patch_transformers_import_compat() -> list[str]: + """Keep Transformers processor/model imports working in local dev envs. + + Some pyenv Python builds are missing the stdlib `_lzma` extension. Importing + `AutoProcessor` can still reach torchvision datasets, which import `lzma` + even when conversion only needs processor metadata. The transpiler already + has the battle-tested patch, so reuse it before converter-side HF imports. + """ + + notes: list[str] = [] + try: + from cactus.transpile.runtime_support import ( + ensure_transformers_supports_gemma4, + patch_torch_flex_attention_compat, + patch_transformers_torchvision_probe, + ) + except Exception: + return notes + + for patch in ( + patch_transformers_torchvision_probe, + patch_torch_flex_attention_compat, + ensure_transformers_supports_gemma4, + ): + try: + note = patch() + except Exception: + note = None + if note: + notes.append(str(note)) + return notes diff --git a/python/cactus/convert/export/files.py b/python/cactus/convert/export/files.py index ba389bce7..20b395def 100644 --- a/python/cactus/convert/export/files.py +++ b/python/cactus/convert/export/files.py @@ -24,10 +24,28 @@ ] -def copy_runtime_files(model_path: str | Path, out_dir: Path) -> None: +def copy_runtime_files( + model_path: str | Path, + out_dir: Path, + *, + token: str | None = None, + cache_dir: str | None = None, +) -> None: src_dir = Path(model_path) if not src_dir.exists() or not src_dir.is_dir(): - return + try: + from huggingface_hub import snapshot_download + + src_dir = Path( + snapshot_download( + repo_id=str(model_path), + allow_patterns=TOKENIZER_FILES, + token=token, + cache_dir=cache_dir, + ) + ) + except Exception: + return for name in TOKENIZER_FILES: src = src_dir / name if src.exists(): diff --git a/python/cactus/convert/export/reports.py b/python/cactus/convert/export/reports.py index a3fe4a806..9b759fd53 100644 --- a/python/cactus/convert/export/reports.py +++ b/python/cactus/convert/export/reports.py @@ -11,7 +11,8 @@ def write_reports(out_dir: Path, rows: list[dict[str, Any]]) -> dict[str, Any]: (out_dir / "conversion_manifest.json").write_text(json.dumps(rows, indent=2, sort_keys=True), encoding="utf-8") weights = [] for row in rows: - if row.get("output_file") and str(row.get("output_file")).endswith(".weights"): + output_file = str(row.get("output_file") or "") + if output_file.endswith((".weights", ".bias")): weights.append({ "source_name": row.get("source_name"), "hf_name": row.get("hf_name") or row.get("source_name"), diff --git a/python/cactus/convert/model_adapters/adapters.py b/python/cactus/convert/model_adapters/adapters.py index 487b5224d..bf51ec941 100644 --- a/python/cactus/convert/model_adapters/adapters.py +++ b/python/cactus/convert/model_adapters/adapters.py @@ -23,6 +23,7 @@ ) from .naming import NameMatch, cactus_name_for_tensor, gemma4_scale_factor, restore_hf_key_for_family from .policy import TensorPolicy, policy_for_tensor +from ..compat import patch_transformers_import_compat @dataclass(frozen=True) @@ -63,6 +64,7 @@ def runtime_model_type(self) -> str: return self.family def model_class(self, cfg: Any): + patch_transformers_import_compat() from transformers import AutoModel, AutoModelForCausalLM arch = " ".join(_cfg_get(cfg, "architectures", []) or []).lower() @@ -71,6 +73,7 @@ def model_class(self, cfg: Any): return AutoModelForCausalLM def load_processor(self, model_id_or_path: str): + patch_transformers_import_compat() from transformers import AutoProcessor, AutoTokenizer try: diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index 0fb0a022b..762b3460d 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -22,12 +22,24 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, out = match.output_name or "" if match.output_name is None: return TensorPolicy("ignored", "none", None, component, False, "none", "no output filename") + if "norm" in out.lower(): + return TensorPolicy("fallback", "FP16", None, component, False, "none", "norm tensor") if family in {"parakeet", "parakeet_tdt"} and out.endswith(".bias") and ( "conv_" in out or out.startswith("subsampling_") or out.startswith("ctc_head_") ): return TensorPolicy("fallback", "FP16", None, component, False, "none", "conv bias tensor") + if family in {"parakeet", "parakeet_tdt"} and "lstm" in out.lower(): + return TensorPolicy("fallback", "FP16", None, component, False, "none", "lstm recurrent tensor") + if family == "parakeet_tdt" and out.startswith("tdt_"): + return TensorPolicy("fallback", "FP16", None, component, False, "none", "tdt decoder tensor") + if family in {"parakeet", "parakeet_tdt"} and out.startswith("layer_") and ( + "conv_pointwise" in out or "conv_depthwise" in out + ): + return TensorPolicy("fallback", "FP16", None, component, False, "none", "conformer conv tensor") + if family == "whisper" and out.startswith("encoder.layer_"): + return TensorPolicy("fallback", "FP16", None, component, False, "none", "whisper encoder tensor") if name.endswith(".bias") or out.endswith(".bias") or ".bias." in out: - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "bias tensor") + return TensorPolicy("fallback", "FP16", None, component, False, "none", "bias tensor") if family in {"parakeet", "parakeet_tdt"} and "conv_pointwise" in out and len(shape) == 3 and shape[2] == 1: return TensorPolicy("fallback", "INT8", 8, component, False, "none", "pointwise conv tensor") if "conv_depthwise.weights" in out and len(shape) == 3 and shape[1] == 1: @@ -36,8 +48,6 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("fallback", "FP16", None, component, False, "none", "non-2d tensor") if "position_embedding" in out.lower() or "pos_embed" in out.lower() or "embed_positions" in out.lower(): return TensorPolicy("fallback", "FP16", None, component, False, "none", "position embedding tensor") - if "norm" in out.lower(): - return TensorPolicy("fallback", "FP16", None, component, False, "none", "norm tensor") if family == "gemma4" and name == "model.language_model.embed_tokens_per_layer.weight": return TensorPolicy("convert", "CQ2", 2, component, False, "hadamard") if component == "embedding" or out in {"token_embeddings.weights", "output_weight.weights"}: diff --git a/python/cactus/convert/tests/test_reports.py b/python/cactus/convert/tests/test_reports.py new file mode 100644 index 000000000..625170e4d --- /dev/null +++ b/python/cactus/convert/tests/test_reports.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from cactus.convert.export.reports import write_reports + + +def test_weights_manifest_includes_bias_tensors(tmp_path: Path) -> None: + write_reports( + tmp_path, + [ + { + "source_name": "decoder.layers.0.fc1.bias", + "output_file": "decoder.layer_0_mlp_fc1.bias", + "shape": [4], + "precision": "FP16", + "status": "fallback", + "component": "transcription", + }, + { + "source_name": "decoder.layers.0.fc1.weight", + "output_file": "decoder.layer_0_mlp_fc1.weights", + "shape": [4, 4], + "precision": "CQ4", + "status": "converted", + "component": "transcription", + }, + ], + ) + + manifest = json.loads((tmp_path / "weights_manifest.json").read_text()) + output_names = {row["output_name"] for row in manifest["weights"]} + + assert "decoder.layer_0_mlp_fc1.bias" in output_names + assert "decoder.layer_0_mlp_fc1.weights" in output_names diff --git a/python/cactus/transpile/README.md b/python/cactus/transpile/README.md new file mode 100644 index 000000000..a5a3ef4e2 --- /dev/null +++ b/python/cactus/transpile/README.md @@ -0,0 +1,1111 @@ +# Cactus Transpiler + +This folder contains the Python side of the Cactus transpiler: the code that takes +a Hugging Face or PyTorch model, captures it with `torch.export`, converts that +export into Cactus' intermediate representation (`IRGraph`), canonicalizes and +optimizes the IR, lowers it into a runtime `Graph`, and optionally saves or runs +the result. + +The transpiler has two main output shapes: + +- A single lowered graph for simple tasks such as text logits or encoder-only audio. +- A component bundle for models that are easier to split into staged graphs, most + notably Gemma4 multimodal and Parakeet TDT. + +This README is a code-first walkthrough of the actual flow in the repo today. + +## Related Entry Points + +These files are outside `python/cactus/transpile`, but they are the public top-level +entry points for the transpiler: + +- `python/cactus/cli/transpile.py` +- `python/cactus/transpile/hf_model.py` + +`cactus transpile` is a thin CLI wrapper around `python/cactus/transpile/hf_model.py`. +The packaged module is the real top-level transpile program. + +## CLI Entry Flow + +### `cactus transpile` + +`python/cactus/cli/transpile.py` resolves the transpiler module, ensures the Python runtime +library is available, and forwards most arguments to the transpiler: + +```python +def cmd_transpile(args): + transpile_lib = _ensure_python_runtime_library() + model_id = resolve_model_id_alias(args.model_id) + extra_args = list(getattr(args, "extra_args", []) or []) + command = [sys.executable, "-m", "cactus.transpile.hf_model", "--model-id", model_id] + if "--weights-dir" not in extra_args: + default_weights_dir = get_weights_dir(model_id) + if _weights_dir_looks_transpile_ready(default_weights_dir): + command.extend(["--weights-dir", str(default_weights_dir)]) + else: + return 1 + if not getattr(args, "execute_after_transpile", False) and "--skip-execute" not in extra_args: + command.append("--skip-execute") + command.extend(extra_args) +``` + +Two important details: + +- `cactus transpile` saves artifacts and skips execution by default. +- Normal transpilation requires converted Cactus CQ weights. Run `cactus convert` + first, or pass `--weights-dir` to an existing converted folder. +- The CLI only has an explicit `--execute-after-transpile` flag; most other + transpile options are forwarded as unknown args to `hf_model.py`. + +### `cactus run-transpiled` + +Saved bundles are executed through `cmd_run_transpiled()` in `python/cactus/cli/transpile.py`: + +```python +def cmd_run_transpiled(args): + transpile_lib = _ensure_python_runtime_library() + os.environ["CACTUS_LIB_PATH"] = str(transpile_lib) + from .transpile.component_bundle_runtime import run_transpiled_bundle + + result = run_transpiled_bundle( + bundle_dir, + audio_file=getattr(args, "audio_file", None) or getattr(args, "audio", None), + image_files=tuple(image_values), + prompt=getattr(args, "prompt", None), + input_ids=getattr(args, "input_ids", None), + weights_dir=getattr(args, "weights_dir", None), + system_prompt=getattr(args, "system", None), + enable_thinking=bool(getattr(args, "thinking", False)), + ) +``` + +`cactus run ` also auto-detects transpiled bundles by looking for a +`manifest.json` and dispatches to the same runtime path. + +## End-To-End Timeline + +## 1. Parse Args, Infer Task, Load Model Bundle + +The real transpile program lives in `python/cactus/transpile/hf_model.py`. +Its `main()` function decides what kind of model/task it is dealing with, validates +optional converted weights, and loads the HF model plus tokenizer/processor: + +```python +if args.task == "auto": + inferred_task = _infer_task_from_config(args.model_id) + ... + if image_files or (args.audio_file and has_multimodal_config and inferred_task == "causal_lm_logits"): + task = "multimodal_causal_lm_logits" + else: + task = inferred_task +else: + task = args.task + +validated_weights_dir = _validate_weights_dir(args.weights_dir.strip() or None, model_id=args.model_id) +weights_dir = str(validated_weights_dir) if validated_weights_dir is not None else None + +model_source, processor_or_tokenizer, model, model_config = _load_transformers_bundle( + model_id=args.model_id, + task=task, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, +) +``` + +### What decides the task? + +`_infer_task_from_config()` looks at: + +- `config.json` architectures +- model type +- known special cases like Parakeet TDT and Whisper +- model id substrings as a last fallback + +Today the main task shapes are: + +- `causal_lm_logits` +- `multimodal_causal_lm_logits` +- `ctc_logits` +- `encoder_hidden_states` +- `tdt_transcription` + +## 2. Prepare Example Inputs + +Before capture, the transpiler builds one concrete example input batch. That +example batch drives both `torch.export` capture and later optional execution. + +The high-level branches are: + +- `_prepare_text_inputs()` for text-only causal LM capture +- `_prepare_audio_inputs()` for speech/audio tasks +- `_prepare_gemma4_multimodal_inputs()` for Gemma4 multimodal + +Example: + +```python +if task == "causal_lm_logits": + prepared = _prepare_text_inputs(...) +elif task == "multimodal_causal_lm_logits": + prepared = _prepare_gemma4_multimodal_inputs(...) +elif task == "tdt_transcription": + prepared = _prepare_audio_inputs(...) +else: + prepared = _prepare_audio_inputs(...) +``` + +The `PreparedInputs` object carries: + +- `names`: logical input names like `input_ids` or `input_features` +- `tensors`: the actual example tensors +- `metadata`: prompt/audio/image/sample-rate info that later gets written into + manifests + +## 3. Canonicalize The Model Interface + +The transpiler does not try to capture every model's native HF forward signature +as-is. It first wraps the model in an adapter that exposes a smaller, more stable +task-specific interface. + +That happens in `model_adapters.py`: + +```python +def canonicalize_model_interface(model, task="causal_lm_logits", *, input_names=None, weights_dir=None): + family = _family_key(model) + ... + if task == "causal_lm_logits": + ... + elif task == "multimodal_causal_lm_logits": + adapter_factory = lambda inner_model: Gemma4MultimodalCausalLMLogitsAdapter( + inner_model, + input_names=resolved_input_names, + weights_dir=weights_dir, + ) + elif task == "ctc_logits": + adapter_factory = lambda inner_model: CTCLogitsAdapter(...) + elif task == "encoder_hidden_states": + adapter_factory = lambda inner_model: EncoderHiddenStatesAdapter(...) +``` + +This step is where family-specific knowledge lives: + +- Gemma/Gemma3/Gemma4/Qwen causal LM wrappers +- Gemma4 multimodal wrappers +- generic CTC and encoder wrappers + +The adapter is also where graph metadata and import hints get attached. + +## 3b. Import PyTorch Ops, Not Model Layer Names + +The compiler boundary is the `torch.export` graph. After capture, Cactus imports +`call_function` nodes by their canonical ATen operation name, not by Hugging Face +module names. For example, `torch.ops.aten.linear.default`, `aten.linear.default`, +and equivalent exported spellings normalize through `aten_ops.py` to the same +canonical IR op: + +```python +aten.linear.default -> linear +aten.silu.default -> silu +aten.conv1d.default -> conv1d +aten.rms_norm.default -> rms_norm +``` + +This is the main generality boundary: model wrappers may still prepare inputs or +split components, but actual compute lowering is ATen op / pattern based. Layer +names are allowed as structural hints for component boundaries and exact +converted-weight manifest aliases; they should not be required to decide how a +PyTorch op maps to a Cactus op. + +## 4. Decide Between Monolithic Capture And Component Capture + +After input prep, the script asks `build_component_module_specs()` whether the +current model supports a split component pipeline: + +```python +component_specs = build_component_module_specs( + model, + task=task, + named_tensors=_named_tensor_store(prepared), + weights_dir=weights_dir, +) +``` + +Today that returns component specs for: + +- Gemma4 multimodal: `vision_encoder`, `audio_encoder`, `lm_encoder`, `decoder` +- Parakeet TDT: `audio_encoder`, `decoder` + +If component specs exist and `--component-pipeline` is `auto` or `on`, the script +takes the component path. Otherwise it captures a single wrapped model graph. + +## 5A. Component Pipeline Path + +The component path is driven by `_run_component_pipeline_transpile()` in +`python/cactus/transpile/hf_model.py`: + +```python +for spec in component_specs: + captured_components[spec.component] = capture_component_spec(spec, fusion_config=fusion_config) +``` + +Each component spec is captured by `component_pipeline.capture_component_spec()`: + +```python +captured = capture_model(wrapped, spec.example_inputs) +raw_ir_graph = copy.deepcopy(captured.ir_graph) +optimized_ir_graph = copy.deepcopy(captured.ir_graph) +canonicalize_exported_graph(optimized_ir_graph) +optimize_graph(optimized_ir_graph, config=fusion_config) +transpiled_graph = transpile_preoptimized_ir(copy.deepcopy(optimized_ir_graph)) +``` + +So each component goes through the same capture/import/canonicalize/optimize/lower +pipeline independently. + +For Gemma4 multimodal, `model_adapters.py` builds the specs by first computing +example image/audio features, then generating example inputs for the downstream +`lm_encoder` and `decoder` stages. + +## 5B. Monolithic Capture Path + +The non-component path wraps the canonical adapter in `TranspileWrapper` and runs +the full graph through capture: + +```python +wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + +captured = capture_model(wrapper, prepared.tensors) +raw_ir_graph = copy.deepcopy(captured.ir_graph) + +canonicalize_exported_graph(captured.ir_graph) +optimize_graph(captured.ir_graph, config=fusion_config) +tg = _lower_preoptimized_ir(captured.ir_graph) +``` + +`TranspileWrapper` mainly exists to pass `weights_dir` into the graph metadata: + +```python +def get_transpile_metadata(self) -> dict[str, object]: + ... + if self.weights_dir: + graph_meta["weights_dir"] = self.weights_dir +``` + +## 6. Capture With `torch.export` + +`capture_pytorch.py` owns the `torch.export` boundary: + +```python +normalized_kwargs = _inject_export_safe_kwargs(model, _normalize_kwargs(kwargs)) +... +transpile_metadata = _collect_transpile_metadata(model, example_args, example_kwargs) + +ep = export(model, args=example_args, kwargs=example_kwargs, strict=strict) +... +ir_graph = import_captured_to_ir(raw_captured, strict=strict) +``` + +Important behavior here: + +- HF-style `use_cache=False` and `return_dict=False` are injected automatically + when the model signature supports them. +- Metadata providers are collected from any module that defines + `get_transpile_metadata()` or `transpile_metadata()`. +- The captured export is immediately imported into the repo's own IR. + +The capture output is a `CapturedModel` dataclass holding: + +- the exported program +- the FX graph +- the lifted state dict +- the imported `IRGraph` +- the example inputs +- the collected metadata + +## 7. Import Exported FX Into `IRGraph` + +`import_ir.py` converts the exported FX/Export graph into the repo's intermediate +representation: + +```python +ir = IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[], constants={}, meta=dict(graph_meta)) +ctx = ImportContext(strict=strict, transpile_metadata=transpile_metadata) +weights_dir = resolve_transpile_weights_dir(ir.meta) + +for node in captured.graph.nodes: + if node.op == "placeholder": + ... + if node.op == "get_attr": + ... + if node.op == "call_function": + import_call_function(...) + if node.op == "output": + import_output(ir, node, ctx) + +apply_import_semantics(ir) +annotate_ir_components(ir) +verify_ir(ir) +``` + +### What gets imported? + +- Real graph inputs become `IRGraph.inputs` +- lifted parameters/buffers/constants become `IRGraph.constants` +- FX `call_function` nodes become `IRNode`s via `importers.py` + +### How weight binding metadata enters the IR + +When a captured constant corresponds to a known converted weight file, +`import_captured_to_ir()` resolves that mapping early: + +```python +binding = resolve_weight_binding(weights_dir=weights_dir, source_name=target) +... +ir.values[value_id_str].meta.update( + { + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } +) +``` + +That metadata is what later lets lowering use `g.mmap_weights()` instead of +embedding the constant payload into the graph. Embedding lookups still lower to +`embedding_from_tensor()`, but the backing table is represented as a normal mmap +weight tensor so tied output heads can reuse the same constant safely. +By default this resolver only trusts explicit `weights_manifest.json` entries +written by `cactus convert` plus wrapper-local aliases derived from those entries. +It does not guess old model-specific filenames from layer names. + +## 8. Importers: FX Target To Canonical IR Node + +`importers.py` is the large target-dispatch layer. It normalizes FX/PyTorch targets +to canonical op names and creates `IRNode`s with stable attributes. + +The dispatch point is: + +```python +def import_call_function(ir, node, ctx, *, shape, dtype, torch_op): + op = normalize_target(torch_op) + importer = OP_IMPORTERS.get(op) + if importer is None: + import_opaque_call_function(...) + else: + importer(...) +``` + +Import is intentionally ATen/op based. It does not apply module-path hints such +as `model.layers.N.self_attn`; model-specific behavior should live in explicit +preprocessing/component boundaries or in topology-based fusion passes. + +Some notable importer behaviors: + +- `normalize.py` reduces many `aten.*` spellings into one canonical name. +- Unknown ops are imported as `kind="opaque"` IR nodes rather than being silently + dropped. +- Import hints can inject attrs/meta into matching nodes after import. +- Some nested exported subgraphs such as `wrap_with_set_grad_enabled` are inlined + explicitly in `import_ir.py`. + +## 9. Import-Time Semantic Rewrites + +Immediately after raw import, `import_semantics.py` performs a first semantic pass: + +```python +def apply_import_semantics(graph: IRGraph) -> IRGraph: + _rewrite_explicit_attention(graph) + _tag_explicit_linear(graph) + _rewrite_early_rope(graph) + rebuild_graph(graph) + verify_ir(graph) + return graph +``` + +This step is intentionally early. It rewrites export patterns into higher-level +meaning before the main optimizer sees them. + +Examples: + +- `scaled_dot_product_attention` becomes semantic `attention` +- explicit linears get tagged +- RoPE patterns can become semantic `rope` before later fusion passes + +## 10. Canonical Cleanup + +`canonicalize/cleanup.py` turns imported IR into a stricter canonical form: + +```python +def canonicalize_exported_graph(graph: IRGraph, *, max_passes: int = 8) -> IRGraph: + for _ in range(max_passes): + for node_id in list(graph.order): + _canonicalize_node(graph, node) + rebuild_graph(graph) + + for node_id in list(graph.order): + _simplify_node(graph, node) + rebuild_graph(graph) + + if _legalize_precisions(graph): + rebuild_graph(graph) + + dce(graph) +``` + +What this pass is responsible for: + +- renaming ops into canonical spellings +- turning shape-only ops into a stable `view` +- rewriting `transpose` and `movedim` into `permute` +- constant-folding trivial expressions +- materializing `ones`, `zeros`, and `arange` constants +- removing no-op casts/views/slices +- inserting FP16 legalizing casts where the runtime requires them +- dead-code elimination + +It also records unsupported canonical ops in `graph.meta["canonical_unsupported_op_counts"]`. + +## 11. Fusion And Graph Optimization + +`optimize_graph.py` runs the semantic fusion passes: + +```python +def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None) -> IRGraph: + canonicalize_exported_graph(graph) + ... + if config.enable_rms_norm: + fuse_rms_norm(graph) + fuse_rms_norm_scale_multiply(graph) + if config.enable_rope: + fuse_rope(graph) + if config.enable_attention: + fuse_attention(graph) + ... + normalize_gemma4_decoder_attention_semantics(graph) + fuse_dense_mlp_tq(graph) + ... + annotate_gold_patterns(graph) + _prune_unused_inputs(graph) +``` + +This is where recognizable subgraphs collapse into semantic or fused nodes such as: + +- `rms_norm` +- `rope` +- `attention` +- `attention_block` +- `self_attention_block` +- `conv_module` +- `lstm_cell` +- `gated_deltanet_prefill` +- `gated_deltanet_decode` + +### Gemma4-specific optimization logic + +Gemma4 gets extra handling in this pass: + +- import hints and graph metadata carry layer types and sliding window info +- exported masks may be elided and rewritten into `window_size` +- some decoder attention behavior is normalized specifically for the saved runtime + +## 12. Lower IR Into A Cactus `Graph` + +`lower.py` is the last compiler stage. It maps `IRValue`s and `IRNode`s into the +runtime `Graph` object from `cactus.bindings.graph`. + +The high-level structure is: + +```python +def transpile_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + g = Graph() + env = {} + + for value_id in ir.inputs: + tensor = _lower_input_value(g, ir.values[value_id]) + env[value_id] = tensor + + for value_id, const in ir.constants.items(): + binding = _lookup_weight_binding(value) + if binding is not None: + binding = ensure_binding_compatible(binding, source_tensor=const) + lowered_const = _lower_constant_value(g, value, const, binding=binding) + env[value_id] = lowered_const + + for node_id in ir.order: + outputs = _lower_ir_node(g, ir.nodes[node_id], env, ir) +``` + +### Constant lowering + +This is the critical constant binding behavior: + +```python +def _lower_constant_value(g, value, const, *, binding=None): + if binding is not None: + return g.mmap_weights(binding.path) + + if tensor_value.numel() == 1: + return tensor_value.item() + + return _materialize_constant_tensor(g, tensor_value) +``` + +So constants can become: + +- `g.mmap_weights(path)` +- an inline scalar +- a materialized tensor node baked into the graph + +### Where weight compatibility is repaired + +`weight_compat.py` sits between IR metadata and actual lowering. It can rewrite a +binding to a more executable companion file, for example: + +- convert INT8 embeddings to a cached CQ embedding format +- materialize CQ4 companion files for legacy packed INT4 tensors + +### What lowering implements + +`_lower_ir_node()` is the actual op dispatcher. It covers: + +- elementwise math and comparisons +- views, reshape, expand, permute, transpose +- matmul and linear +- embedding and gather/index paths +- reductions and softmax +- conv1d/conv2d +- norms +- fused attention-family nodes +- fused LSTM and Conformer-style blocks + +If an IR op survives all prior passes and still has no lowering, `lower.py` raises: + +```python +raise NotImplementedError(f"unsupported IR op in lowering: {op}") +``` + +### Gemma4 decoder attention fallback + +One especially important special case is that Gemma4 decoder attention can bypass +the runtime attention kernel and be lowered manually as matmul + mask + softmax: + +```python +def _should_lower_gemma4_decoder_attention_without_kernel(ir, node): + family = str(ir.meta.get("adapter_family") or ir.meta.get("family") or "").strip().lower() + component = str(ir.meta.get("component", "") or "").strip().lower() + if family != "gemma4" or component != "decoder": + return False + return node.op in {"attention", "scaled_dot_product_attention"} +``` + +That path exists because the transpiled Gemma4 decoder has some runtime/kernel +compatibility constraints that are easier to control in lowered graph code. + +## 13. Save Artifacts + +After lowering, `hf_model.py` writes the IR and bundle artifacts. + +For the monolithic path it saves: + +- `raw_ir.json` +- `optimized_ir.json` +- `graph.cactus` +- `graph_bindings.json` +- component subgraphs under `components/` + +For the component path it saves: + +- `raw_ir.json` +- `optimized_ir.json` +- per-component IRs +- `components/manifest.json` +- one lowered graph per component + +The component bundle writer is `_write_component_bundle()`: + +```python +manifest_path = bundle_dir / "manifest.json" +_write_json( + manifest_path, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component_order": component_order, + "inputs": _serialize_json_compatible(inputs_metadata), + "components": manifest_components, + }, +) +``` + +Why both a `.cactus` graph and a binding manifest? + +Because `Graph.save()` stores the graph structure, but the file-backed weight +bindings need to be reattached later. The manifests carry: + +- logical input/output names +- runtime node ids +- paths to bound weights or saved Cactus tensor constants + +## 14. Optional Execution And Reference Compare + +If execution is enabled, the transpiler module will run the lowered graph immediately. + +By task: + +- text LM: run graph, report next token +- Gemma4 multimodal: run pipeline, report next token +- Parakeet TDT: run component encoder + custom decode loop, report transcript +- generic audio: run encoder graph and report output shapes + +Many paths also compare the transpiled output against a PyTorch reference forward +unless `--skip-reference-compare` is set. + +## 15. Run A Saved Bundle Later + +`component_bundle_runtime.py` is the runtime-side loader for saved bundles. + +The public entry is: + +```python +def run_transpiled_bundle(bundle_dir_or_manifest, *, audio_file=None, image_files=(), prompt=None, input_ids=None, weights_dir=None, ...): + bundle_root, manifest = load_component_bundle_manifest(bundle_dir_or_manifest) + component_graphs, manifest = load_saved_component_graphs( + bundle_dir_or_manifest, + weights_dir=resolved_weights_dir, + ) + ... +``` + +There are two ways a saved component can be reloaded: + +1. Load a serialized `.cactus` graph and rebind its constants. +2. Rebuild from saved raw/optimized IR JSON and re-run lowering. + +That second path is handled by `_load_component_graph_from_ir()`: + +```python +ir_graph = _deserialize_saved_ir_graph(...) +if use_raw_ir: + canonicalize_exported_graph(ir_graph) + optimize_graph(ir_graph) +else: + if family == "gemma4" and component in {"decoder", "decoder_step"}: + optimize_graph(ir_graph) +transpiled = transpile_preoptimized_ir(ir_graph) +``` + +So saved IR artifacts are not just for inspection; they are also a fallback loading +format that can be re-lowered on demand. + +### Bound constant rebinding at runtime + +The runtime loader reattaches Cactus `.weights` tensor files with +`graph.set_external_input(...)`: + +```python +tensor = graph._tensor_from_node(node_id) +graph.set_external_input(tensor, int(tensor_file.data.ctypes.data), dtype=tensor_file.precision) +... +if tensor_file.scales is not None and tensor_file.group_size > 0: + _lib.cactus_graph_set_grouped_scales(...) +``` + +That is why the binding manifests are essential. + +## Major Situations And How They Are Handled + +### Hugging Face models that normally return caches or `ModelOutput` + +`capture_pytorch.py` forces export-safe kwargs: + +- `use_cache=False` +- `return_dict=False` + +This keeps the capture boundary tensor-only. + +### Weights directory exists but has no manifest + +`hf_model.py` fails fast: + +```python +if not manifest_path.exists(): + raise RuntimeError("weights_dir is missing weights_manifest.json") +``` + +This is intentional. Converted CQ/Cactus weights are the source of truth for +transpiled bundles; raw filename guessing is not part of the runtime contract. + +### Weights directory resolves, but no bindings match + +The transpile script fails hard and suggests re-running conversion: + +- component path: if component graphs resolve `0` bindings +- monolithic path: if full optimized IR resolves `0` bindings + +### Local snapshot exists but is incomplete + +`_load_model_source()` will reject incomplete local snapshots when +`--local-files-only` is set. + +### Gemma4 transformers support is missing locally + +`runtime_support.py` and `hf_model.py` search alternate site-packages and +patch import compatibility for: + +- missing `transformers.models.gemma4` +- missing `_lzma` +- torchvision import probes +- missing `torchvision::nms` +- missing `flex_attention.AuxRequest` + +### Gemma4 local checkpoint key mismatches + +`_repair_gemma4_checkpoint_weights()` remaps local checkpoint keys before loading +the model. + +### Gemma4 multimodal prompt drift + +The transpiler normalizes prompt and tokenization details to match native Cactus: + +- inserts missing image/audio placeholder tokens +- builds a Cactus-style chat prompt when requested +- expands HF newline-merge tokens `108` and `109` back into repeated newline token + `107` +- can replace processor-generated image/audio tensors with native-like versions + +### Parakeet TDT does not run as a single generic forward + +`tdt_runtime.py` provides: + +- a local PyTorch reconstruction of the model +- audio preprocessing +- a greedy TDT decode loop +- component specs for an encoder graph and a recurrent decoder-step graph + +### Some runtime symbols are not present in the current shared library + +`runtime_compat.py` monkey-patches the Python `Graph` wrapper so the transpiler can +still build graphs against the current runtime. It provides compatibility behavior +for items like: + +- generic `matmul` +- `gather(axis=0)` +- limited `conv2d` +- `not_equal` +- scalar math legalization + +### Opaque import is allowed, but execution may still fail later + +If `importers.py` does not know a target, it creates an opaque IR node instead of +dropping it. That preserves information, but lowering still needs a real lowering +case. Opaque nodes are therefore a sign that later lowering may stop with +`unsupported IR op`. + +## Artifact Layout + +A typical transpile artifact directory looks like this: + +```text +transpiled// + raw_ir.json + optimized_ir.json + graph.cactus + graph_bindings.json + result.json + components/ + manifest.json + vision_encoder/ + graph.cactus + bound_constants/ + ... + audio_encoder/ + graph.cactus + lm_encoder/ + graph.cactus + decoder/ + graph.cactus +``` + +Notes: + +- `graph.cactus` is the monolithic lowered graph, when one exists. +- `components/manifest.json` is the portable bundle manifest. +- `graph_bindings.json` and per-component `bound_constant_bindings` tell the loader + how to reattach mmap-backed weights and saved Cactus tensor constants. + +## File-By-File Map + +### Core transpiler files + +| File | Responsibility | +| --- | --- | +| `audio_preprocess.py` | WAV loading, resampling, generic log-mel extraction, Cactus audio frontend wrappers, Parakeet-native features, Gemma4-native audio features. | +| `capture_pytorch.py` | `torch.export` capture wrapper, export-safe kwargs injection, metadata collection, shape propagation, and `CapturedModel` creation. | +| `component_bundle_runtime.py` | Loader and executor for saved transpiled bundles; reloads `.cactus` graphs or re-lowers saved IR, then runs task-specific bundle executors. | +| `component_partition.py` | Heuristic component labeling of IR nodes/values and subgraph extraction into `audio_encoder` / `vision_encoder` / `lm_encoder` / `decoder`. | +| `component_pipeline.py` | Capture/optimize/lower loop for one component spec, plus runtime execution of a staged pipeline of components. | +| `graph_ir.py` | Definitions for `IRValue`, `IRNode`, `IRGraph`, deep-copy behavior, and IR consistency verification. | +| `import_ir.py` | Top-level importer from exported FX graph into `IRGraph`; resolves lifted constants, weight binding metadata, and inlined subgraphs. | +| `import_semantics.py` | Early semantic rewrites done immediately after import, especially attention and RoPE recognition. | +| `importers.py` | Large target-to-IR importer registry; converts normalized PyTorch ops into canonical `IRNode`s. | +| `lower.py` | Final lowering from optimized IR into runtime `Graph`, including op lowering, constant binding, fused op handling, and execution wrapper. | +| `model_adapters.py` | Family- and task-specific forward adapters, Gemma4 multimodal pipeline adapters, task canonicalization, and component spec builders. | +| `model_patterns.py` | Topology/op catalog used to annotate recognizable transformer structures after optimization. | +| `model_profiles.py` | Compact model-family profile constants, including compatibility modules, graph context defaults, and weight-name aliases used by fallback runtimes. | +| `multimodal_runtime.py` | Shared multimodal runtime helpers used by bundle execution: prompt normalization, image/audio limiting, and native-like multimodal preprocessing. | +| `normalize.py` | Target-name normalization (`aten.*` to canonical names) and dtype-name normalization into IR spellings. | +| `ops.py` | Canonical op registry and alias table for IR ops, including backend op names and attr schemas. | +| `optimize_graph.py` | Main fusion/optimization pass driver, pattern fusion, Gemma4 attention normalization, and gold-pattern annotation. | +| `runtime_support.py` | Shared import/runtime compatibility helpers for optional Transformers modules, torchvision probes, and flex-attention shims. | +| `tdt_runtime.py` | TDT fallback runtime: local recurrent decoder loop, config loader, and encoder/decoder component-spec generation. | +| `runtime_compat.py` | Runtime wrapper patches that make the current Python `Graph` API usable for transpiled graphs even when some C symbols are absent. | +| `weight_binding.py` | Resolves IR constants to converted `.weights` files through `weights_manifest.json`; also resolves default weights dirs. | +| `weight_compat.py` | Makes bound weights executable with the current runtime by materializing compatibility companion files when needed. | + +### Canonicalization subdirectory + +| File | Responsibility | +| --- | --- | +| `canonicalize/cleanup.py` | Multi-pass canonical cleanup, constant folding, no-op removal, attr normalization, and precision legalization. | +| `canonicalize/utils.py` | Shared graph rebuild, DCE, node removal, dtype helpers, and constant materialization utilities. | + +### Fusion subdirectory + +| File | Responsibility | +| --- | --- | +| `fusion/__init__.py` | Re-exports the fusion matchers used by `optimize_graph.py`. | +| `fusion/common.py` | Shared fusion helper types and graph traversal helpers. | +| `fusion/linear.py` | Matcher for linear projection patterns. | +| `fusion/rms_norm.py` | Matcher for RMSNorm computational subgraphs. | +| `fusion/rope.py` | Matcher for rotary embedding patterns. | +| `fusion/attention.py` | Matchers for attention, attention blocks, self-attention blocks, and GQA-style projection patterns. | +| `fusion/conv.py` | Matcher for Conformer-like convolution modules. | +| `fusion/lstm.py` | Matcher for LSTM cell subgraphs. | +| `fusion/mlp.py` | Matcher for gated MLP branches such as GELU/SiLU gate-up-down patterns. | +| `fusion/rel_pos_bias.py` | Matcher for relative-position-bias subgraphs. | +| `fusion/deltanet.py` | Matcher for fused gated DeltaNet patterns. | + +## CLI Usage + +## 1. Basic Transpile + +The simplest public command is: + +```bash +cactus transpile +``` + +By default this: + +- requires a converted Cactus CQ weights directory, either passed with + `--weights-dir` or found in the default `weights/` location +- resolves the runtime library +- launches `python/cactus/transpile/hf_model.py` +- saves artifacts under `./transpiled//` +- adds `--skip-execute`, so it does not run the graph unless you ask + +To also execute right after transpiling: + +```bash +cactus transpile --execute-after-transpile +``` + +## 2. Forwarded Transpile Options + +Most useful transpile options are parsed by `python/cactus/transpile/hf_model.py` +and simply forwarded through `cactus transpile`. + +Common ones: + +- `--task` +- `--prompt` +- `--input-ids` +- `--audio-file` +- `--image-file` +- `--weights-dir` +- `--artifact-dir` +- `--component-pipeline` +- `--skip-reference-compare` +- `--graph-filename` +- `--no-fuse-rms-norm` +- `--no-fuse-rope` +- `--no-fuse-attention` +- `--allow-unconverted-weights` for compiler-only debugging + +Examples: + +### Text causal LM + +```bash +cactus convert /path/to/converted_weights --bits 4 +cactus transpile \ + --weights-dir /path/to/converted_weights \ + --prompt "The capital of France is" \ + --artifact-dir ./transpiled/text_demo +``` + +### Text causal LM with explicit converted weights + +```bash +cactus transpile \ + --weights-dir /path/to/converted_weights \ + --prompt "Write a haiku about compilers" +``` + +### Gemma4 multimodal + +```bash +cactus transpile \ + --task multimodal_causal_lm_logits \ + --image-file /path/to/image.jpg \ + --audio-file /path/to/audio.wav \ + --prompt "Describe what is happening in the image and audio." \ + --weights-dir /path/to/converted_weights +``` + +### Whisper-style encoder capture + +```bash +cactus convert /path/to/converted_weights --bits 4 +cactus transpile \ + --audio-file /path/to/sample.wav \ + --task encoder_hidden_states \ + --weights-dir /path/to/converted_weights +``` + +### Parakeet TDT + +```bash +cactus transpile \ + --audio-file /path/to/sample.wav \ + --task tdt_transcription \ + --weights-dir /path/to/converted_weights +``` + +If you want the full direct script help: + +```bash +python -m cactus.transpile.hf_model --help +``` + +## 3. Run A Saved Transpiled Bundle + +Use the bundle root directory or a `manifest.json` path: + +```bash +cactus run-transpiled /path/to/transpiled/model +``` + +For direct bundle execution, audio input is passed with `--file`. The `--audio` +flag belongs to `cactus run`, which may forward into the same runtime after +bundle auto-detection. + +### Text causal LM bundle + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --prompt "The capital of France is" +``` + +Or pass token ids directly: + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --input-ids 2,13,42 +``` + +### Multimodal Gemma4 bundle + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --image-file /path/to/image.jpg \ + --file /path/to/audio.wav \ + --prompt "What do you observe?" \ + --system "Answer concisely." +``` + +### Audio encoder / transcription bundle + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --file /path/to/sample.wav +``` + +### Bundles with bound weights + +If the bundle expects converted mmap weights, pass the weights directory when the +saved paths are not directly reachable: + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --weights-dir /path/to/converted_weights \ + --prompt "Hello" +``` + +### Save result payload + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --prompt "Hello" \ + --result-json ./result.json +``` + +## 4. `cactus run` Also Works + +`cactus run` detects transpiled bundles automatically: + +```bash +cactus run /path/to/transpiled/model --prompt "Hello" +``` + +If you are passing media through `cactus run`, use `--image` and `--audio` +rather than `run-transpiled`'s direct `--file` audio flag. + +Internally, `cli.py` checks whether the path contains a transpiled manifest and +redirects to `cmd_run_transpiled()`. + +## Current Runtime Limitations + +As of the current code: + +- Saved bundle execution is implemented for `causal_lm_logits`, + `multimodal_causal_lm_logits`, `encoder_hidden_states`, and + `tdt_transcription`. +- `ctc_logits` can be transpiled, but `run_transpiled_bundle()` does not currently + implement a saved-bundle executor for that task. +- The component pipeline is model-family-specific; it is not a universal split-IR + path. + +## Useful Environment Variables + +- `CACTUS_TRANSPILER_WEIGHTS_DIR`: + fallback weights directory used by `weight_binding.py` +- `CACTUS_TRANSPILER_WEIGHTS_DIR_`: + family-specific weights directory override +- `CACTUS_TRANSPILER_DEBUG_MMAP=1`: + log constant binding behavior during lowering +- `CACTUS_GEMMA4_CAPTURE_FP32=1`: + temporarily promote some Gemma4 text modules to FP32 during capture on CPU + +## Practical Summary + +If you only want the high-level lifecycle, it is: + +1. `cactus transpile` launches `python/cactus/transpile/hf_model.py`. +2. The script infers a task, loads the HF model, and creates one example input batch. +3. `model_adapters.py` wraps the model into a smaller canonical task interface. +4. `capture_pytorch.py` runs `torch.export`. +5. `import_ir.py` + `importers.py` convert the export to `IRGraph`. +6. `import_semantics.py` adds early semantic rewrites. +7. `canonicalize/cleanup.py` normalizes and simplifies the IR. +8. `optimize_graph.py` fuses patterns into higher-level semantic ops. +9. `lower.py` builds a runtime `Graph`, mmaps bound weights when possible, and + materializes everything else. +10. The script writes `raw_ir.json`, `optimized_ir.json`, `graph.cactus`, and + `components/manifest.json`. +11. `cactus run-transpiled` later reloads those artifacts with + `component_bundle_runtime.py`, rebinds external tensors, and executes the + task-specific bundle runner. diff --git a/python/cactus/transpile/__init__.py b/python/cactus/transpile/__init__.py new file mode 100644 index 000000000..47c8ee364 --- /dev/null +++ b/python/cactus/transpile/__init__.py @@ -0,0 +1 @@ +"""Cactus graph transpiler package.""" diff --git a/python/cactus/transpile/aten_ops.py b/python/cactus/transpile/aten_ops.py new file mode 100644 index 000000000..4cb8967f8 --- /dev/null +++ b/python/cactus/transpile/aten_ops.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class AtenOpRule: + torch_prefix: str + ir_op: str + + +def canonical_torch_op(target: Any) -> str: + """Return a stable torch/export op spelling independent of Python repr noise.""" + + if isinstance(target, str): + value = target + else: + rendered = str(target).strip() + if rendered.startswith("aten.") or rendered.startswith("": "getitem", + "aten.view.default": "reshape", + "aten.reshape.default": "reshape", + "aten._unsafe_view.default": "reshape", + "aten.flatten.using_ints": "flatten", + "aten.t.default": "transpose", + "aten.gelu.erf": "gelu_erf", + # Keep the legacy importer key until the importer itself is renamed. + "aten.diff.default": "aten.diff.default", +} + + +_PREFIX_RULES: tuple[AtenOpRule, ...] = ( + AtenOpRule("aten.mul_", "multiply_inplace"), + AtenOpRule("aten._assert_tensor_metadata", "identity"), + AtenOpRule("aten.lift_fresh_copy", "identity"), + AtenOpRule("aten.clone", "identity"), + AtenOpRule("aten.alias", "identity"), + AtenOpRule("aten.detach", "identity"), + AtenOpRule("aten.dropout", "identity"), + AtenOpRule("aten.native_dropout", "identity"), + AtenOpRule("aten.arange.start", "arange"), + AtenOpRule("aten.arange", "arange"), + AtenOpRule("aten.add", "add"), + AtenOpRule("aten.sub", "subtract"), + AtenOpRule("aten.mul", "multiply"), + AtenOpRule("aten.floor_divide", "floor_divide"), + AtenOpRule("aten.div", "divide"), + AtenOpRule("aten.to.dtype", "precision_cast"), + AtenOpRule("aten.to.device", "identity"), + AtenOpRule("aten.type_as", "type_as"), + AtenOpRule("aten.neg", "negate"), + AtenOpRule("aten.ne.", "not_equal"), + AtenOpRule("aten.eq.", "equal"), + AtenOpRule("aten.gt.", "greater"), + AtenOpRule("aten.ge.", "greater_equal"), + AtenOpRule("aten.lt.", "less"), + AtenOpRule("aten.le.", "less_equal"), + AtenOpRule("aten.logical_or", "logical_or"), + AtenOpRule("aten.__or__", "logical_or"), + AtenOpRule("aten.logical_and", "logical_and"), + AtenOpRule("aten.__and__", "logical_and"), + AtenOpRule("aten.logical_not", "logical_not"), + AtenOpRule("aten.bitwise_not", "logical_not"), + AtenOpRule("aten.abs", "abs"), + AtenOpRule("aten.clamp", "clamp"), + AtenOpRule("aten.where", "where"), + AtenOpRule("aten.masked_scatter", "masked_scatter"), + AtenOpRule("aten.masked_fill", "masked_fill"), + AtenOpRule("aten.expand", "expand"), + AtenOpRule("aten.repeat", "repeat"), + AtenOpRule("aten.exp", "scalar_exp"), + AtenOpRule("aten.sqrt", "scalar_sqrt"), + AtenOpRule("aten.reciprocal", "reciprocal"), + AtenOpRule("aten.log", "scalar_log"), + AtenOpRule("aten.rsqrt", "rsqrt"), + AtenOpRule("aten.cos", "cos"), + AtenOpRule("aten.sin", "sin"), + AtenOpRule("aten.squeeze", "squeeze"), + AtenOpRule("aten.unsqueeze", "unsqueeze"), + AtenOpRule("aten.transpose", "transpose"), + AtenOpRule("aten.permute", "permute"), + AtenOpRule("aten.numpy_T", "numpy_T"), + AtenOpRule("aten.movedim", "movedim"), + AtenOpRule("aten.contiguous", "contiguous"), + AtenOpRule("aten.mm", "matmul"), + AtenOpRule("aten.matmul", "matmul"), + AtenOpRule("aten.bmm", "matmul"), + AtenOpRule("aten.linear", "linear"), + AtenOpRule("aten.addmm", "addmm"), + AtenOpRule("aten.relu", "relu"), + AtenOpRule("aten.silu", "silu"), + AtenOpRule("aten.gelu", "gelu"), + AtenOpRule("aten.sigmoid", "sigmoid"), + AtenOpRule("aten.glu", "glu"), + AtenOpRule("aten.softplus", "softplus"), + AtenOpRule("aten.tanh", "tanh"), + AtenOpRule("aten.softmax", "softmax"), + AtenOpRule("aten.scaled_dot_product_attention", "scaled_dot_product_attention"), + AtenOpRule("aten.sum", "sum"), + AtenOpRule("aten.mean", "mean"), + AtenOpRule("aten.all", "min"), + AtenOpRule("aten.var", "variance"), + AtenOpRule("aten.min", "min"), + AtenOpRule("aten.amin", "min"), + AtenOpRule("aten.max", "max"), + AtenOpRule("aten.amax", "max"), + AtenOpRule("aten.cat", "cat"), + AtenOpRule("aten.split_with_sizes", "split_with_sizes"), + AtenOpRule("aten.chunk", "chunk"), + AtenOpRule("aten.ones", "ones"), + AtenOpRule("aten.slice", "slice"), + AtenOpRule("aten.select", "index"), + AtenOpRule("aten.gather", "gather"), + AtenOpRule("aten.pad", "pad"), + AtenOpRule("aten.one_hot", "one_hot"), + AtenOpRule("aten.tril", "tril"), + AtenOpRule("aten.unfold", "unfold"), + AtenOpRule("aten.embedding", "embedding"), + AtenOpRule("aten.conv1d", "conv1d"), + AtenOpRule("aten.conv2d", "conv2d"), + AtenOpRule("aten.pow", "pow"), + AtenOpRule("aten.layer_norm", "layer_norm"), + AtenOpRule("aten.native_layer_norm", "layer_norm"), + AtenOpRule("aten.group_norm", "group_norm"), + AtenOpRule("aten.batch_norm", "batch_norm"), + AtenOpRule("aten._native_batch_norm_legit_no_training", "batch_norm"), + AtenOpRule("aten.rms_norm", "rms_norm"), +) + + +def canonical_ir_op(target: Any) -> str: + torch_op = canonical_torch_op(target) + exact = _EXACT_RULES.get(torch_op) + if exact is not None: + return exact + if torch_op in {"aten.ne", "aten.eq", "aten.gt", "aten.ge", "aten.lt", "aten.le"}: + return { + "aten.ne": "not_equal", + "aten.eq": "equal", + "aten.gt": "greater", + "aten.ge": "greater_equal", + "aten.lt": "less", + "aten.le": "less_equal", + }[torch_op] + for rule in _PREFIX_RULES: + if torch_op.startswith(rule.torch_prefix): + return rule.ir_op + return torch_op + + +def is_aten_op(target: Any) -> bool: + return canonical_torch_op(target).startswith("aten.") diff --git a/python/cactus/transpile/audio_preprocess.py b/python/cactus/transpile/audio_preprocess.py new file mode 100644 index 000000000..5e6d0aeea --- /dev/null +++ b/python/cactus/transpile/audio_preprocess.py @@ -0,0 +1,400 @@ +from __future__ import annotations + +import os +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from functools import lru_cache +from pathlib import Path + +import numpy as np +import torch +from scipy.io import wavfile +from scipy.signal import resample_poly + + +_PARAKEET_SAMPLE_RATE = 16000 +_PARAKEET_NUM_MELS = 128 +_PARAKEET_N_FFT = 512 +_PARAKEET_FRAME_LENGTH = 400 +_PARAKEET_HOP_LENGTH = 160 +_PARAKEET_PREEMPHASIS = 0.97 +_PARAKEET_LOG_FLOOR = np.float32(2**-24) +_DEFAULT_MAX_AUDIO_SECONDS = 30.0 + + +def audio_duration_limit_seconds() -> float: + raw = os.environ.get("CACTUS_TRANSPILER_MAX_AUDIO_SECONDS", str(_DEFAULT_MAX_AUDIO_SECONDS)) + try: + value = float(raw) + except (TypeError, ValueError): + return _DEFAULT_MAX_AUDIO_SECONDS + return max(0.0, value) + + +def normalize_audio_samples(samples: np.ndarray) -> np.ndarray: + if samples.dtype == np.uint8: + return ((samples.astype(np.float32) - 128.0) / 128.0).astype(np.float32) + if np.issubdtype(samples.dtype, np.integer): + info = np.iinfo(samples.dtype) + scale = float(max(abs(info.min), info.max)) + if scale <= 0: + scale = 1.0 + return (samples.astype(np.float32) / scale).astype(np.float32) + return samples.astype(np.float32) + + +def _limit_waveform_duration( + waveform: np.ndarray, + *, + sample_rate: int, + max_seconds: float | None, +) -> np.ndarray: + if max_seconds is None or max_seconds <= 0.0: + return waveform + max_samples = int(round(float(sample_rate) * float(max_seconds))) + if max_samples <= 0 or int(waveform.shape[0]) <= max_samples: + return waveform + return np.ascontiguousarray(waveform[:max_samples]) + + +def load_audio_waveform( + audio_file: str | Path, + *, + target_sample_rate: int, + max_seconds: float | None = None, +) -> np.ndarray: + if max_seconds is None: + max_seconds = audio_duration_limit_seconds() + sample_rate, waveform = wavfile.read(str(audio_file)) + if waveform.ndim > 1: + waveform = waveform.mean(axis=1) + waveform = normalize_audio_samples(np.asarray(waveform)) + if int(sample_rate) != int(target_sample_rate): + gcd = int(np.gcd(int(sample_rate), int(target_sample_rate))) + waveform = resample_poly( + waveform, + int(target_sample_rate) // gcd, + int(sample_rate) // gcd, + ).astype(np.float32) + sample_rate = int(target_sample_rate) + waveform = _limit_waveform_duration( + waveform.astype(np.float32), + sample_rate=int(target_sample_rate), + max_seconds=max_seconds, + ) + return waveform.astype(np.float32) + + +@contextmanager +def limited_audio_file( + audio_file: str | Path, + *, + target_sample_rate: int = _PARAKEET_SAMPLE_RATE, + max_seconds: float | None = None, +) -> Iterator[Path]: + if max_seconds is None: + max_seconds = audio_duration_limit_seconds() + if max_seconds <= 0.0: + yield Path(audio_file).expanduser().resolve() + return + + waveform = load_audio_waveform( + audio_file, + target_sample_rate=target_sample_rate, + max_seconds=max_seconds, + ) + temp_audio_path: Path | None = None + try: + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file: + temp_audio_path = Path(temp_file.name) + wavfile.write( + str(temp_audio_path), + int(target_sample_rate), + (np.clip(waveform, -1.0, 1.0) * np.float32(32767.0)).astype(np.int16), + ) + yield temp_audio_path + finally: + if temp_audio_path is not None: + try: + temp_audio_path.unlink() + except OSError: + pass + + +def _hz_to_mel(hz: np.ndarray) -> np.ndarray: + return 2595.0 * np.log10(1.0 + hz / 700.0) + + +def _mel_to_hz(mel: np.ndarray) -> np.ndarray: + return 700.0 * (np.power(10.0, mel / 2595.0) - 1.0) + + +@lru_cache(maxsize=16) +def _mel_filter_bank(num_mels: int, sample_rate: int, n_fft: int) -> np.ndarray: + try: + from transformers.audio_utils import mel_filter_bank as transformers_mel_filter_bank # type: ignore + + return transformers_mel_filter_bank( + num_frequency_bins=n_fft // 2 + 1, + num_mel_filters=num_mels, + min_frequency=0.0, + max_frequency=sample_rate / 2.0, + sampling_rate=sample_rate, + norm="slaney", + mel_scale="slaney", + ).astype(np.float32) + except Exception: + pass + + num_freq_bins = n_fft // 2 + 1 + mel_points = np.linspace( + _hz_to_mel(np.array([0.0], dtype=np.float32))[0], + _hz_to_mel(np.array([sample_rate / 2.0], dtype=np.float32))[0], + num_mels + 2, + ) + hz_points = _mel_to_hz(mel_points) + bins = np.floor((n_fft + 1) * hz_points / sample_rate).astype(np.int64) + + filters = np.zeros((num_mels, num_freq_bins), dtype=np.float32) + for index in range(1, num_mels + 1): + left = max(0, min(num_freq_bins - 1, int(bins[index - 1]))) + center = max(left + 1, min(num_freq_bins - 1, int(bins[index]))) + right = max(center + 1, min(num_freq_bins, int(bins[index + 1]))) + for freq in range(left, center): + filters[index - 1, freq] = float(freq - left) / float(max(center - left, 1)) + for freq in range(center, right): + filters[index - 1, freq] = float(right - freq) / float(max(right - center, 1)) + return filters + + +def generic_log_mel_features( + waveform: np.ndarray, + *, + sample_rate: int, + num_mels: int, + n_fft: int, + hop_length: int, + frame_length: int, + preemphasis: float | None = None, + mel_floor: np.float32 = _PARAKEET_LOG_FLOOR, + normalize_active_frames_only: bool = True, +) -> tuple[np.ndarray, int]: + waveform_tensor = torch.from_numpy(waveform).to(torch.float32).unsqueeze(0) + if preemphasis is not None and abs(preemphasis) > 0.0: + waveform_tensor = torch.cat( + [ + waveform_tensor[:, :1], + waveform_tensor[:, 1:] - float(preemphasis) * waveform_tensor[:, :-1], + ], + dim=1, + ) + + window = torch.hann_window(frame_length, periodic=False, dtype=torch.float32) + stft = torch.stft( + waveform_tensor, + n_fft=n_fft, + hop_length=hop_length, + win_length=frame_length, + window=window, + return_complex=True, + center=True, + pad_mode="constant", + ) + magnitudes = torch.view_as_real(stft) + magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1)) + power = magnitudes.pow(2) + + feature_length = max(1, int(waveform.shape[0] // hop_length)) + mel_filters = torch.from_numpy(_mel_filter_bank(num_mels, sample_rate, n_fft)).to(torch.float32) + mel_spec = power.transpose(1, 2) @ mel_filters + mel_spec = torch.log(torch.clamp(mel_spec, min=float(mel_floor))) + + if normalize_active_frames_only: + attention_mask = ( + torch.arange(mel_spec.shape[1], dtype=torch.long, device=mel_spec.device).unsqueeze(0) + < feature_length + ) + mask = attention_mask.unsqueeze(-1).to(torch.float32) + masked = mel_spec * mask + mean = masked.sum(dim=1) / float(feature_length) + centered = masked - mean.unsqueeze(1) + denom = float(max(1, feature_length - 1)) + variance = (centered.pow(2) * mask).sum(dim=1) / denom + mel_spec = (mel_spec - mean.unsqueeze(1)) / torch.sqrt(variance.unsqueeze(1) + np.float32(1e-5)) + mel_spec = mel_spec * mask + else: + # Native Parakeet normalizes across the full centered-STFT frame count, + # then trims to waveform_samples // hop_length. Keep that order for parity. + frame_count = int(mel_spec.shape[1]) + mean = mel_spec.mean(dim=1) + centered = mel_spec - mean.unsqueeze(1) + denom = float(max(1, frame_count - 1)) + variance = centered.pow(2).sum(dim=1) / denom + mel_spec = centered / torch.sqrt(variance.unsqueeze(1) + np.float32(1e-5)) + return mel_spec[0].cpu().numpy().astype(np.float32), feature_length + + +def prepare_cactus_audio_features( + audio_file: str | Path, + *, + model_type: str, + expected_frames: int | None, + expected_mels: int, + torch_dtype: torch.dtype, + layout: str = "frames_mels", +) -> tuple[torch.Tensor, int]: + from cactus.bindings.cactus import cactus_preprocess_audio_features + + capacity_frames = int(expected_frames) if isinstance(expected_frames, int) and expected_frames > 0 else 0 + if capacity_frames <= 0: + lowered_model_type = model_type.lower() + if "whisper" in lowered_model_type: + capacity_frames = 3000 + else: + sample_rate, waveform = wavfile.read(str(audio_file)) + samples = int(waveform.shape[0]) if hasattr(waveform, "shape") else len(waveform) + if sample_rate <= 0: + sample_rate = _PARAKEET_SAMPLE_RATE + capacity_frames = max( + 1, + int(np.ceil(samples * (_PARAKEET_SAMPLE_RATE / float(sample_rate)) / _PARAKEET_HOP_LENGTH)) + 8, + ) + if "whisper" in model_type.lower(): + capacity_frames = max(capacity_frames, 3000) + capacity = max(1, capacity_frames * int(expected_mels)) + with limited_audio_file(audio_file, target_sample_rate=_PARAKEET_SAMPLE_RATE) as frontend_audio_file: + values, mel_bins, frames = cactus_preprocess_audio_features( + str(frontend_audio_file), + model_type, + int(expected_mels), + capacity, + ) + if int(mel_bins) != int(expected_mels): + raise ValueError(f"Cactus audio frontend returned {mel_bins} mel bins, expected {expected_mels}") + feature_array = np.asarray(values, dtype=np.float32).reshape(int(mel_bins), int(frames)) + active_frames = int(frames) + if isinstance(expected_frames, int) and expected_frames > 0: + active_frames = min(active_frames, expected_frames) + if layout == "frames_mels": + feature_array = feature_array[:, :active_frames].T + if isinstance(expected_frames, int) and expected_frames > active_frames: + feature_array = np.pad(feature_array, ((0, expected_frames - active_frames), (0, 0)), mode="constant") + elif layout == "mels_frames": + feature_array = feature_array[:, :active_frames] + if isinstance(expected_frames, int) and expected_frames > active_frames: + feature_array = np.pad(feature_array, ((0, 0), (0, expected_frames - active_frames)), mode="constant") + else: + raise ValueError(f"unsupported Cactus audio feature layout: {layout!r}") + tensor = torch.from_numpy(np.ascontiguousarray(feature_array)).unsqueeze(0).to(dtype=torch_dtype) + return tensor, active_frames + + +def prepare_native_parakeet_audio_features( + audio_file: str | Path, + *, + expected_frames: int | None, + expected_mels: int, + torch_dtype: torch.dtype, +) -> tuple[torch.Tensor, int]: + try: + return prepare_cactus_audio_features( + audio_file, + model_type="parakeet", + expected_frames=expected_frames, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + except Exception: + pass + + waveform = load_audio_waveform(audio_file, target_sample_rate=_PARAKEET_SAMPLE_RATE) + features, feature_length = generic_log_mel_features( + waveform, + sample_rate=_PARAKEET_SAMPLE_RATE, + num_mels=expected_mels or _PARAKEET_NUM_MELS, + n_fft=_PARAKEET_N_FFT, + hop_length=_PARAKEET_HOP_LENGTH, + frame_length=_PARAKEET_FRAME_LENGTH, + preemphasis=_PARAKEET_PREEMPHASIS, + mel_floor=_PARAKEET_LOG_FLOOR, + normalize_active_frames_only=False, + ) + active_frames = feature_length + if isinstance(expected_frames, int) and expected_frames > 0: + active_frames = min(active_frames, expected_frames) + features = features[:active_frames, :] + if features.shape[1] != expected_mels: + raise ValueError( + f"feature mel dimension mismatch: expected {expected_mels}, got {features.shape[1]}" + ) + if isinstance(expected_frames, int) and expected_frames > active_frames: + features = np.pad(features, ((0, expected_frames - active_frames), (0, 0)), mode="constant") + tensor = torch.from_numpy(np.ascontiguousarray(features)).unsqueeze(0).to(dtype=torch_dtype) + return tensor, active_frames + + +def prepare_native_gemma4_audio_features( + audio_file: str | Path, + *, + expected_mels: int, + torch_dtype: torch.dtype, + max_seconds: float | None = None, + pad_to_max_seconds: bool = True, +) -> tuple[torch.Tensor, torch.Tensor, int]: + if max_seconds is None: + max_seconds = audio_duration_limit_seconds() + waveform = load_audio_waveform( + audio_file, + target_sample_rate=_PARAKEET_SAMPLE_RATE, + max_seconds=max_seconds, + ) + # Native Gemma4 pads to the nearest 320 samples, adds 160 samples of + # semicausal left padding, and then runs a 321-point analysis frame. + estimated_frames = int(np.ceil((len(waveform) + 640) / 160.0)) + 8 + capacity = max(1, estimated_frames * int(expected_mels)) + values, mel_bins, frames = prepare_cactus_audio_features_raw( + audio_file, + model_type="gemma4", + mel_bins=int(expected_mels), + capacity=capacity, + max_seconds=max_seconds, + ) + if int(mel_bins) != int(expected_mels): + raise ValueError(f"Cactus Gemma4 audio frontend returned {mel_bins} mel bins, expected {expected_mels}") + max_frames = estimated_frames + if max_seconds is not None and max_seconds > 0.0: + max_samples = int(round(float(_PARAKEET_SAMPLE_RATE) * float(max_seconds))) + max_frames = int(np.ceil((max_samples + 640) / 160.0)) + 8 + active_frames = min(int(frames), int(max_frames)) + features = np.asarray(values, dtype=np.float32).reshape(int(frames), int(mel_bins))[:active_frames, :] + output_frames = max(active_frames, int(max_frames)) if pad_to_max_seconds else active_frames + if output_frames > active_frames: + features = np.pad(features, ((0, output_frames - active_frames), (0, 0)), mode="constant") + feature_tensor = torch.from_numpy(np.ascontiguousarray(features)).unsqueeze(0).to(dtype=torch_dtype) + mask = torch.zeros((1, int(output_frames)), dtype=torch.bool) + mask[:, :active_frames] = True + return feature_tensor, mask, int(active_frames) + + +def prepare_cactus_audio_features_raw( + audio_file: str | Path, + *, + model_type: str, + mel_bins: int, + capacity: int, + max_seconds: float | None = None, +) -> tuple[list[float], int, int]: + from cactus.bindings.cactus import cactus_preprocess_audio_features + + with limited_audio_file( + audio_file, + target_sample_rate=_PARAKEET_SAMPLE_RATE, + max_seconds=max_seconds, + ) as frontend_audio_file: + return cactus_preprocess_audio_features( + str(frontend_audio_file), + model_type, + int(mel_bins), + int(capacity), + ) diff --git a/python/cactus/transpile/canonicalize/cleanup.py b/python/cactus/transpile/canonicalize/cleanup.py new file mode 100644 index 000000000..2d7ec7f2e --- /dev/null +++ b/python/cactus/transpile/canonicalize/cleanup.py @@ -0,0 +1,686 @@ +from __future__ import annotations + +from collections import Counter + +import torch + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.graph_ir import verify_ir +from cactus.transpile.ops import canonicalize_op +from cactus.transpile.ops import get_op +from cactus.transpile.ops import has_op +from cactus.transpile.canonicalize.utils import bypass_unary_node +from cactus.transpile.canonicalize.utils import dce +from cactus.transpile.canonicalize.utils import ir_dtype_to_torch +from cactus.transpile.canonicalize.utils import materialize_constant_output +from cactus.transpile.canonicalize.utils import normalize_dim +from cactus.transpile.canonicalize.utils import normalize_dtype_name +from cactus.transpile.canonicalize.utils import numel +from cactus.transpile.canonicalize.utils import rebuild_graph + + +COMPILER_SUPPORTED_OPS = { + "arange", + "ones", +} + +CANONICAL_UNSUPPORTED_RENAMES = { + "aten.__and__.Tensor": "logical_and", + "aten.__or__.Tensor": "logical_or", + "aten.cumsum.default": "cumsum", + "aten.eq.Scalar": "equal", + "aten.eq.Tensor": "equal", + "aten.ge.Scalar": "greater_equal", + "aten.ge.Tensor": "greater_equal", + "aten.gt.Scalar": "greater", + "aten.gt.Tensor": "greater", + "aten.index.Tensor": "advanced_index", + "aten.le.Scalar": "less_equal", + "aten.le.Tensor": "less_equal", + "aten.lt.Scalar": "less", + "aten.lt.Tensor": "less", +} + +FP32_SUPPORTED_ALL_INPUT_OPS = { + "add", + "subtract", + "multiply", + "divide", + "precision_cast", + "view", + "permute", + "transpose", + "slice", + "index", + "advanced_index", + "gather", + "matmul", + "linear", + "addmm", + "layer_norm", + "batch_norm", + "glu", + "sum", + "mean", + "variance", + "min", + "max", + "scalar_add", + "scalar_subtract", + "scalar_subtract_reverse", + "scalar_multiply", + "scalar_divide", + "scalar_floor_divide", + "scalar_not_equal", + "scalar_equal", + "scalar_greater", + "scalar_greater_equal", + "scalar_less", + "scalar_less_equal", + "abs", + "negate", + "pow", + "scalar_exp", + "scalar_sqrt", + "scalar_log", + "scalar_cos", + "scalar_sin", +} + +FP32_SUPPORTED_INPUT_INDICES = { + "embedding": {1}, +} + +FP16_ONLY_OUTPUT_OPS = { + "scalar_divide_reverse", + "not_equal", + "equal", + "greater", + "greater_equal", + "less", + "less_equal", + "logical_and", + "logical_or", + "logical_not", + "one_hot", + "scalar_not_equal", + "scalar_equal", + "scalar_greater", + "scalar_greater_equal", + "scalar_less", + "scalar_less_equal", + "cat", + "softmax", + "rms_norm", + "embedding", + "group_norm", + "lstm_cell", + "relu", + "silu", + "gelu", + "gelu_erf", + "sigmoid", + "tanh", + "attention", + "attention_block", + "conv_module", + "dense_mlp_tq_fused", + "rope", +} + + +def canonicalize_exported_graph(graph: IRGraph, *, max_passes: int = 8) -> IRGraph: + verify_ir(graph) + + for _ in range(max_passes): + changed = False + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if _canonicalize_node(graph, node): + changed = True + + rebuild_graph(graph) + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if _simplify_node(graph, node): + changed = True + + rebuild_graph(graph) + + if _legalize_precisions(graph): + changed = True + rebuild_graph(graph) + + dce(graph) + + if not changed: + break + + unsupported_counts = summarize_unsupported_ops(graph) + graph.meta = dict(graph.meta) + graph.meta["canonical_unsupported_ops"] = tuple(sorted(unsupported_counts)) + graph.meta["canonical_unsupported_op_counts"] = unsupported_counts + verify_ir(graph) + return graph + + +def summarize_unsupported_ops(graph: IRGraph) -> dict[str, int]: + counts: Counter[str] = Counter() + for node_id in graph.order: + node = graph.nodes[node_id] + if not _is_supported_without_backend_kernel(node.op): + counts[node.op] += 1 + return dict(sorted(counts.items())) + + +def _canonicalize_node(graph: IRGraph, node: IRNode) -> bool: + original_op = node.op + + if original_op in {"aten.new_ones.default", "aten.ones.default"}: + return _materialize_new_ones_constant(graph, node) + + if original_op in {"aten.new_zeros.default", "aten.zeros.default", "new_zeros"}: + return _materialize_new_zeros_constant(graph, node) + + if original_op == "arange": + return _materialize_arange_constant(graph, node) + + if original_op in {"identity", "contiguous"}: + return bypass_unary_node(graph, node) + + if original_op in CANONICAL_UNSUPPORTED_RENAMES: + node.op = CANONICAL_UNSUPPORTED_RENAMES[original_op] + return True + + if original_op == "multiply_inplace": + node.op = "multiply" + return True + + if original_op == "view" and "shape" not in node.attrs: + return _rewrite_view_from_output_shape(graph, node) + + if original_op in {"reshape", "flatten", "unsqueeze", "squeeze"}: + return _rewrite_view_from_output_shape(graph, node) + + if original_op == "expand": + return _rewrite_expand_if_view_equivalent(graph, node) + + if original_op == "transpose": + return _rewrite_transpose_as_permute(graph, node) + + if original_op == "movedim": + return _rewrite_movedim_as_permute(graph, node) + + if original_op == "type_as": + return _rewrite_type_as(graph, node) + + if original_op == "concat": + node.op = "cat" + if "dim" in node.attrs and "axis" not in node.attrs: + node.attrs = {**node.attrs, "axis": int(node.attrs["dim"])} + return True + + if original_op == "scaled_dot_product_attention": + node.op = "attention" + node.attrs = { + "scale": float(node.attrs.get("scale", 0.0)), + "is_causal": bool(node.attrs.get("is_causal", False)), + "window_size": int(node.attrs.get("window_size", 0)), + **({"mask": node.attrs["mask"]} if "mask" in node.attrs else {}), + **({"additive_mask": bool(node.attrs.get("additive_mask", False))} if "additive_mask" in node.attrs else {}), + } + return True + + canonical_op = canonicalize_op(original_op) + if canonical_op != original_op: + node.op = canonical_op + _normalize_attr_spellings(node) + if has_op(node.op): + _filter_attrs_to_schema(node) + return True + + if has_op(node.op): + _normalize_attr_spellings(node) + _filter_attrs_to_schema(node) + return True + + return False + + +def _simplify_node(graph: IRGraph, node: IRNode) -> bool: + if _fold_constant_node(graph, node): + return True + + if node.op == "precision_cast": + return _remove_noop_precision_cast(graph, node) + + if node.op == "view": + if _remove_noop_view(graph, node): + return True + return _collapse_view_chain(graph, node) + + if node.op == "slice": + return _remove_noop_slice(graph, node) + + if node.op == "scalar_add": + return _remove_noop_scalar(graph, node, neutral=0.0) + + if node.op == "scalar_subtract": + return _remove_noop_scalar(graph, node, neutral=0.0) + + if node.op == "scalar_multiply": + return _remove_noop_scalar(graph, node, neutral=1.0) + + if node.op == "scalar_divide": + return _remove_noop_scalar(graph, node, neutral=1.0) + + return False + + +def _fold_constant_node(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + + output_value = graph.values.get(node.outputs[0]) + if output_value is None: + return False + + if node.op in FP16_ONLY_OUTPUT_OPS: + output_value.dtype = "fp16" + node.meta["dtype"] = "fp16" + + output_torch_dtype = ir_dtype_to_torch(output_value.dtype) if output_value.dtype is not None else None + + constant_inputs = [graph.constants.get(value_id) for value_id in node.inputs] + if node.op == "view" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + if output_value.shape is None: + return False + return materialize_constant_output( + graph, + node, + constant_inputs[0].reshape(tuple(int(dim) for dim in output_value.shape)).to(dtype=output_torch_dtype), + ) + + if node.op == "expand" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + if output_value.shape is None: + return False + return materialize_constant_output( + graph, + node, + constant_inputs[0].expand(tuple(int(dim) for dim in output_value.shape)).clone().to(dtype=output_torch_dtype), + ) + + if node.op == "precision_cast" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + dtype = normalize_dtype_name(node.attrs.get("dtype")) + if dtype is None: + return False + return materialize_constant_output(graph, node, constant_inputs[0].to(dtype=ir_dtype_to_torch(dtype))) + + if node.op in {"cos", "scalar_cos"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + return materialize_constant_output(graph, node, torch.cos(constant_inputs[0])) + + if node.op in {"sin", "scalar_sin"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + return materialize_constant_output(graph, node, torch.sin(constant_inputs[0])) + + if node.op in {"scalar_add", "scalar_subtract", "scalar_multiply", "scalar_divide"}: + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + scalar = float(node.attrs["value"]) + if node.op == "scalar_add": + result = constant_inputs[0] + scalar + elif node.op == "scalar_subtract": + result = constant_inputs[0] - scalar + elif node.op == "scalar_multiply": + result = constant_inputs[0] * scalar + else: + result = constant_inputs[0] / scalar + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op in {"scalar_not_equal", "scalar_equal", "scalar_greater", "scalar_greater_equal", "scalar_less", "scalar_less_equal"}: + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + scalar = float(node.attrs["value"]) + if node.op == "scalar_not_equal": + result = constant_inputs[0] != scalar + elif node.op == "scalar_equal": + result = constant_inputs[0] == scalar + elif node.op == "scalar_greater": + result = constant_inputs[0] > scalar + elif node.op == "scalar_greater_equal": + result = constant_inputs[0] >= scalar + elif node.op == "scalar_less": + result = constant_inputs[0] < scalar + else: + result = constant_inputs[0] <= scalar + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op in {"not_equal", "equal", "greater", "greater_equal", "less", "less_equal", "logical_and", "logical_or"}: + if len(constant_inputs) != 2 or not all(isinstance(value, torch.Tensor) for value in constant_inputs): + return False + lhs, rhs = constant_inputs + if node.op == "not_equal": + result = lhs != rhs + elif node.op == "equal": + result = lhs == rhs + elif node.op == "greater": + result = lhs > rhs + elif node.op == "greater_equal": + result = lhs >= rhs + elif node.op == "less": + result = lhs < rhs + elif node.op == "less_equal": + result = lhs <= rhs + elif node.op == "logical_and": + result = torch.logical_and(lhs != 0, rhs != 0) + else: + result = torch.logical_or(lhs != 0, rhs != 0) + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op == "logical_not": + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + result = torch.logical_not(constant_inputs[0] != 0) + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + return False + + +def _legalize_precisions(graph: IRGraph) -> bool: + changed = False + counter = 0 + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + if node.op in FP32_SUPPORTED_ALL_INPUT_OPS: + continue + + inserted: list[IRNode] = [] + allowed_indices = FP32_SUPPORTED_INPUT_INDICES.get(node.op, set()) + for input_index, value_id in enumerate(list(node.inputs)): + if input_index in allowed_indices: + continue + value = graph.values.get(value_id) + if value is None or normalize_dtype_name(value.dtype) != "fp32": + continue + + counter += 1 + cast_node_id = _unique_ir_id( + f"{node.id}_legalize_fp16_{counter}", + graph.nodes.keys(), + graph.order, + ) + cast_value_id = _unique_ir_id( + f"{value_id}_legalize_fp16_{counter}", + graph.values.keys(), + graph.constants.keys(), + ) + cast_node = IRNode( + id=cast_node_id, + op="precision_cast", + inputs=[value_id], + outputs=[cast_value_id], + attrs={"dtype": "fp16"}, + meta={"inserted_by": "canonicalize_exported_graph"}, + ) + graph.values[cast_value_id] = IRValue( + id=cast_value_id, + shape=value.shape, + dtype="fp16", + producer=cast_node_id, + users=[], + ) + node.inputs[input_index] = cast_value_id + inserted.append(cast_node) + changed = True + + if inserted: + insert_at = graph.order.index(node_id) + for offset, cast_node in enumerate(inserted): + graph.nodes[cast_node.id] = cast_node + graph.order.insert(insert_at + offset, cast_node.id) + + if node.op in FP16_ONLY_OUTPUT_OPS: + for output_id in node.outputs: + output_value = graph.values.get(output_id) + if output_value is not None: + output_value.dtype = "fp16" + node.meta["dtype"] = "fp16" + + return changed + + +def _unique_ir_id(base: str, *existing_groups) -> str: + existing: set[str] = set() + for group in existing_groups: + existing.update(str(item) for item in group) + if base not in existing: + return base + suffix = 1 + while True: + candidate = f"{base}_{suffix}" + if candidate not in existing: + return candidate + suffix += 1 + + +def _rewrite_view_from_output_shape(graph: IRGraph, node: IRNode) -> bool: + if not node.outputs: + return False + output_value = graph.values.get(node.outputs[0]) + if output_value is None or output_value.shape is None: + return False + node.op = "view" + node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} + return True + + +def _rewrite_expand_if_view_equivalent(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs or not node.outputs: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if numel(input_value.shape) != numel(output_value.shape): + node.meta["canonicalization_blocked"] = "expand_requires_broadcast" + return False + node.op = "view" + node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} + return True + + +def _rewrite_transpose_as_permute(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.shape is None: + return False + rank = len(input_value.shape) + dim0 = normalize_dim(int(node.attrs["dim0"]), rank) + dim1 = normalize_dim(int(node.attrs["dim1"]), rank) + permutation = list(range(rank)) + permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] + node.op = "permute" + node.attrs = {"permutation": tuple(permutation)} + return True + + +def _rewrite_movedim_as_permute(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.shape is None: + return False + rank = len(input_value.shape) + source = [normalize_dim(int(dim), rank) for dim in node.attrs["source"]] + destination = [normalize_dim(int(dim), rank) for dim in node.attrs["destination"]] + if len(source) != len(destination): + return False + + remaining = [dim for dim in range(rank) if dim not in source] + permutation = list(remaining) + for dst, src in sorted(zip(destination, source)): + permutation.insert(dst, src) + + node.op = "permute" + node.attrs = {"permutation": tuple(permutation)} + return True + + +def _rewrite_type_as(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + target_dtype = None + if node.outputs: + output_value = graph.values.get(node.outputs[0]) + if output_value is not None: + target_dtype = output_value.dtype + if target_dtype is None and len(node.inputs) > 1: + target_value = graph.values.get(node.inputs[1]) + if target_value is not None: + target_dtype = target_value.dtype + if target_dtype is None: + return False + node.op = "precision_cast" + node.inputs = [node.inputs[0]] + node.attrs = {"dtype": normalize_dtype_name(target_dtype)} + return True + + +def _normalize_attr_spellings(node: IRNode) -> None: + attrs = dict(node.attrs) + if "dim" in attrs and "axis" not in attrs and node.op in {"softmax", "sum", "mean", "variance", "min", "max", "cat", "slice", "index"}: + attrs["axis"] = attrs.pop("dim") + if node.op == "permute" and "dims" in attrs and "permutation" not in attrs: + attrs["permutation"] = tuple(int(dim) for dim in attrs.pop("dims")) + if node.op == "slice": + attrs.setdefault("step", 1) + if node.op == "precision_cast" and "dtype" in attrs: + attrs["dtype"] = normalize_dtype_name(attrs["dtype"]) + node.attrs = attrs + + +def _filter_attrs_to_schema(node: IRNode) -> None: + schema = get_op(node.op) + if not schema.attrs: + node.attrs = {} + return + node.attrs = {name: node.attrs[name] for name in schema.attrs if name in node.attrs} + + +def _remove_noop_precision_cast(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + target_dtype = normalize_dtype_name(node.attrs.get("dtype")) + input_dtype = None if input_value is None else normalize_dtype_name(input_value.dtype) + output_dtype = None if output_value is None else normalize_dtype_name(output_value.dtype) + if target_dtype is None or target_dtype == input_dtype or (input_dtype is not None and output_dtype == input_dtype): + return bypass_unary_node(graph, node) + return False + + +def _remove_noop_view(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if tuple(input_value.shape) == tuple(output_value.shape): + return bypass_unary_node(graph, node) + return False + + +def _collapse_view_chain(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.producer is None: + return False + producer = graph.nodes.get(input_value.producer) + if producer is None or producer.op != "view" or len(producer.inputs) != 1: + return False + node.inputs = [producer.inputs[0]] + return True + + +def _remove_noop_slice(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if tuple(input_value.shape) == tuple(output_value.shape): + return bypass_unary_node(graph, node) + return False + + +def _remove_noop_scalar(graph: IRGraph, node: IRNode, *, neutral: float) -> bool: + value = node.attrs.get("value") + if value is None: + return False + if float(value) != float(neutral): + return False + return bypass_unary_node(graph, node) + + +def _materialize_new_ones_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None or output_value.shape is None or output_value.dtype is None: + return False + torch_dtype = ir_dtype_to_torch(output_value.dtype) + const = torch.ones(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _materialize_new_zeros_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None or output_value.shape is None or output_value.dtype is None: + return False + torch_dtype = ir_dtype_to_torch(output_value.dtype) + const = torch.zeros(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _materialize_arange_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + start = int(node.attrs.get("start", 0)) + end = node.attrs.get("end") + if end is None: + return False + step = int(node.attrs.get("step", 1)) + dtype = normalize_dtype_name(node.attrs.get("dtype", "fp32")) + torch_dtype = ir_dtype_to_torch(dtype) + const = torch.arange(start, int(end), step=step, dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _is_supported_without_backend_kernel(op_name: str) -> bool: + return has_op(op_name) or op_name in COMPILER_SUPPORTED_OPS diff --git a/python/cactus/transpile/canonicalize/utils.py b/python/cactus/transpile/canonicalize/utils.py new file mode 100644 index 000000000..ba956328a --- /dev/null +++ b/python/cactus/transpile/canonicalize/utils.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +from typing import Any + +import torch + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.graph_ir import verify_ir + + +def normalize_dtype_name(dtype: Any) -> str | None: + if dtype is None: + return None + if not isinstance(dtype, str): + return str(dtype) + if dtype.startswith("torch."): + torch_name = dtype.split(".", 1)[1] + mapping = { + "bool": "bool", + "int8": "int8", + "int16": "int16", + "int32": "int32", + "int64": "int64", + "float16": "fp16", + "bfloat16": "bf16", + "float32": "fp32", + "float64": "fp64", + } + return mapping.get(torch_name, dtype) + return dtype + + +def ir_dtype_to_torch(dtype: str) -> torch.dtype: + normalized = normalize_dtype_name(dtype) + if normalized == "bool": + return torch.bool + if normalized == "int8": + return torch.int8 + if normalized == "int16": + return torch.int16 + if normalized == "int32": + return torch.int32 + if normalized == "int64": + return torch.int64 + if normalized == "fp16": + return torch.float16 + if normalized == "bf16": + return torch.bfloat16 + if normalized == "fp32": + return torch.float32 + if normalized == "fp64": + return torch.float64 + raise NotImplementedError(f"unsupported IR dtype for constant materialization: {dtype}") + + +def normalize_dim(dim: int, rank: int) -> int: + if dim < 0: + dim += rank + return dim + + +def numel(shape: tuple[int, ...]) -> int: + total = 1 + for dim in shape: + total *= int(dim) + return total + + +def refresh_value_users(graph: IRGraph) -> None: + for value in graph.values.values(): + value.users = [] + for node_id in graph.order: + node = graph.nodes[node_id] + for input_id in node.inputs: + value = graph.values.get(input_id) + if value is not None: + value.users.append(node_id) + + +def rebuild_graph(graph: IRGraph) -> None: + new_values: dict[str, IRValue] = {} + + for value_id in graph.inputs: + old = graph.values.get(value_id, IRValue(id=value_id)) + new_values[value_id] = IRValue( + id=value_id, + shape=old.shape, + dtype=old.dtype, + producer=None, + users=[], + meta=dict(old.meta), + ) + + for value_id in graph.constants.keys(): + old = graph.values.get(value_id, IRValue(id=value_id)) + new_values[value_id] = IRValue( + id=value_id, + shape=old.shape, + dtype=old.dtype, + producer=None, + users=[], + meta=dict(old.meta), + ) + + for node_id in graph.order: + node = graph.nodes[node_id] + for output_id in node.outputs: + old = graph.values.get(output_id, IRValue(id=output_id)) + new_values[output_id] = IRValue( + id=output_id, + shape=old.shape, + dtype=old.dtype, + producer=node_id, + users=[], + meta=dict(old.meta), + ) + + for node_id in graph.order: + node = graph.nodes[node_id] + for input_id in node.inputs: + if input_id not in new_values: + old = graph.values.get(input_id, IRValue(id=input_id)) + new_values[input_id] = IRValue( + id=input_id, + shape=old.shape, + dtype=old.dtype, + producer=old.producer, + users=[], + meta=dict(old.meta), + ) + new_values[input_id].users.append(node_id) + + for output_id in graph.outputs: + if output_id not in new_values: + old = graph.values.get(output_id, IRValue(id=output_id)) + new_values[output_id] = IRValue( + id=output_id, + shape=old.shape, + dtype=old.dtype, + producer=old.producer, + users=[], + meta=dict(old.meta), + ) + + graph.values = new_values + verify_ir(graph) + + +def dce(graph: IRGraph) -> None: + live_values = set(graph.outputs) + live_nodes: set[str] = set() + + changed = True + while changed: + changed = False + for node_id in reversed(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if not any(output in live_values for output in node.outputs): + continue + if node_id not in live_nodes: + live_nodes.add(node_id) + changed = True + for input_id in node.inputs: + if input_id not in live_values: + live_values.add(input_id) + changed = True + + graph.order = [node_id for node_id in graph.order if node_id in live_nodes] + graph.nodes = {node_id: node for node_id, node in graph.nodes.items() if node_id in live_nodes} + + live_value_ids = set(graph.inputs) | set(graph.outputs) + for node in graph.nodes.values(): + live_value_ids.update(node.inputs) + live_value_ids.update(node.outputs) + + graph.values = {value_id: value for value_id, value in graph.values.items() if value_id in live_value_ids} + graph.constants = {value_id: value for value_id, value in graph.constants.items() if value_id in live_value_ids} + rebuild_graph(graph) + + +def remove_node(graph: IRGraph, node_id: str) -> None: + if node_id in graph.nodes: + del graph.nodes[node_id] + graph.order = [existing_id for existing_id in graph.order if existing_id != node_id] + + +def drop_value_if_unused(graph: IRGraph, value_id: str) -> None: + if value_id in graph.inputs or value_id in graph.outputs or value_id in graph.constants: + return + value = graph.values.get(value_id) + if value is None: + return + if value.producer is None and not value.users: + del graph.values[value_id] + + +def bypass_unary_node(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_id = node.inputs[0] + output_id = node.outputs[0] + + for other in graph.nodes.values(): + other.inputs = [input_id if value_id == output_id else value_id for value_id in other.inputs] + graph.outputs = [input_id if value_id == output_id else value_id for value_id in graph.outputs] + + remove_node(graph, node.id) + if output_id in graph.values: + del graph.values[output_id] + drop_value_if_unused(graph, input_id) + return True + + +def _is_gemma4_graph(graph: IRGraph) -> bool: + return str(graph.meta.get("adapter_family") or graph.meta.get("family") or "").lower() == "gemma4" + + +def _weight_binding_meta(value: IRValue | None) -> dict[str, object] | None: + if value is None: + return None + path = value.meta.get("path") + kind = value.meta.get("kind") + source_name = value.meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + return dict(value.meta) + return None + + +def _gemma4_materialized_binding_meta(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + if not _is_gemma4_graph(graph) or len(node.inputs) != 1: + return None + if node.op == "precision_cast": + pass + elif node.op == "scalar_add" and float(node.attrs.get("value", 0.0)) in (0.0, 1.0): + pass + else: + return None + + value_id = node.inputs[0] + for _ in range(4): + value = graph.values.get(value_id) + binding_meta = _weight_binding_meta(value) + if binding_meta is not None: + binding_meta["materialized_from_value_id"] = value_id + binding_meta["materialized_by_op"] = node.op + return binding_meta + producer_id = value.producer if value is not None else None + producer_node = graph.nodes.get(producer_id) if producer_id is not None else None + if producer_node is None or producer_node.op not in {"precision_cast", "type_as"}: + break + if len(producer_node.inputs) != 1: + break + value_id = producer_node.inputs[0] + return None + + +def materialize_constant_output(graph: IRGraph, node: IRNode, value: Any) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None: + return False + binding_meta = _gemma4_materialized_binding_meta(graph, node) + if binding_meta is not None: + output_value.meta.update(binding_meta) + graph.constants[output_id] = value + output_value.producer = None + remove_node(graph, node.id) + return True diff --git a/python/cactus/transpile/capture/import_semantics.py b/python/cactus/transpile/capture/import_semantics.py deleted file mode 100644 index 1afad4aaf..000000000 --- a/python/cactus/transpile/capture/import_semantics.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -import torch - -from cactus.transpile.capture.graph_ir import IRGraph -from cactus.transpile.capture.graph_ir import IRNode -from cactus.transpile.capture.graph_ir import verify_ir - -def apply_import_semantics(graph: IRGraph) -> IRGraph: - return graph diff --git a/python/cactus/transpile/capture/normalize.py b/python/cactus/transpile/capture/normalize.py deleted file mode 100644 index f9d23b546..000000000 --- a/python/cactus/transpile/capture/normalize.py +++ /dev/null @@ -1,155 +0,0 @@ -from __future__ import annotations - -from typing import Any - -import torch - - -def normalize_target(target: str) -> str: - if target.startswith("aten.mul_"): - return "multiply_inplace" - if target.startswith("aten._assert_tensor_metadata"): - return "identity" - if target.startswith("aten.lift_fresh_copy"): - return "identity" - if target.startswith("aten.clone"): - return "identity" - if target.startswith("aten.alias"): - return "identity" - if target.startswith("aten.detach"): - return "identity" - if target.startswith("aten.arange.start"): - return "arange" - if target.startswith("aten.arange"): - return "arange" - if target.startswith("aten.add"): - return "add" - if target.startswith("aten.sub"): - return "subtract" - if target.startswith("aten.mul"): - return "multiply" - if target.startswith("aten.div"): - return "divide" - if target.startswith("aten.to.dtype"): - return "precision_cast" - if target.startswith("aten.type_as"): - return "type_as" - if target.startswith("aten.neg"): - return "negate" - if target.startswith("aten.ne.") or target == "aten.ne": - return "not_equal" - if target.startswith("aten.abs"): - return "abs" - if target.startswith("aten.expand"): - return "expand" - if target.startswith("aten.exp"): - return "scalar_exp" - if target.startswith("aten.sqrt"): - return "scalar_sqrt" - if target.startswith("aten.log"): - return "scalar_log" - if target.startswith("aten.rsqrt"): - return "rsqrt" - if target.startswith("aten.cos"): - return "cos" - if target.startswith("aten.sin"): - return "sin" - if target in ("aten.view.default", "aten.reshape.default", "aten._unsafe_view.default"): - return "reshape" - if target == "aten.flatten.using_ints": - return "flatten" - if target.startswith("aten.unsqueeze"): - return "unsqueeze" - if target == "aten.t.default" or target.startswith("aten.transpose"): - return "transpose" - if target.startswith("aten.permute"): - return "permute" - if target.startswith("aten.contiguous"): - return "contiguous" - if target.startswith("aten.mm") or target.startswith("aten.matmul") or target.startswith("aten.bmm"): - return "matmul" - if target.startswith("aten.linear"): - return "linear" - if target.startswith("aten.addmm"): - return "addmm" - if target.startswith("aten.relu"): - return "relu" - if target.startswith("aten.silu"): - return "silu" - if target == "aten.gelu.erf": - return "gelu_erf" - if target.startswith("aten.gelu"): - return "gelu" - if target.startswith("aten.sigmoid"): - return "sigmoid" - if target.startswith("aten.softplus"): - return "softplus" - if target.startswith("aten.tanh"): - return "tanh" - if target.startswith("aten.softmax"): - return "softmax" - if target.startswith("aten.scaled_dot_product_attention"): - return "scaled_dot_product_attention" - if target.startswith("aten.sum"): - return "sum" - if target.startswith("aten.mean"): - return "mean" - if target.startswith("aten.var"): - return "variance" - if target.startswith("aten.min") or target.startswith("aten.amin"): - return "min" - if target.startswith("aten.max") or target.startswith("aten.amax"): - return "max" - if target.startswith("aten.cat"): - return "cat" - if target.startswith("aten.split_with_sizes"): - return "split_with_sizes" - if target.startswith("aten.chunk"): - return "chunk" - if target.startswith("aten.ones"): - return "ones" - if target.startswith("aten.slice"): - return "slice" - if target.startswith("aten.select"): - return "index" - if target.startswith("aten.gather"): - return "gather" - if target.startswith("aten.pad"): - return "pad" - if target.startswith("aten.embedding"): - return "embedding" - if target.startswith("aten.conv1d"): - return "conv1d" - if target.startswith("aten.pow"): - return "pow" - if target.startswith("aten.layer_norm"): - return "layer_norm" - if target.startswith("aten.native_layer_norm"): - return "layer_norm" - if target.startswith("aten.group_norm"): - return "group_norm" - if target.startswith("aten.batch_norm") or target.startswith("aten._native_batch_norm_legit_no_training"): - return "batch_norm" - if target.startswith("aten.rms_norm"): - return "rms_norm" - if target == "": - return "getitem" - return target - - -def dtype_to_ir(dtype: Any | None) -> str | None: - if dtype is None: - return None - - dtype_map = { - torch.float16: "fp16", - torch.float32: "fp32", - torch.float64: "fp64", - torch.bfloat16: "bf16", - torch.int8: "int8", - torch.int16: "int16", - torch.int32: "int32", - torch.int64: "int64", - torch.bool: "bool", - } - return dtype_map.get(dtype, str(dtype)) diff --git a/python/cactus/transpile/capture/weight_binding.py b/python/cactus/transpile/capture/weight_binding.py deleted file mode 100644 index 29c07dc81..000000000 --- a/python/cactus/transpile/capture/weight_binding.py +++ /dev/null @@ -1,185 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import json -import os -from pathlib import Path -import re - - -@dataclass(frozen=True) -class WeightBinding: - path: str - kind: str # "weight" | "embedding" - source_name: str - - -_PROJECT_ROOT = Path(__file__).resolve().parents[3] - - -def _candidate_model_dir_names(model_name_or_path: str) -> list[str]: - candidates: list[str] = [] - raw = model_name_or_path.strip() - if not raw: - return candidates - - def _add(name: str) -> None: - name = name.strip().lower() - if name and name not in candidates: - candidates.append(name) - - _add(raw.split("/")[-1]) - - path = Path(raw) - parts = path.parts - for part in parts: - if part.startswith("models--"): - _add(part[len("models--"):].split("--")[-1]) - break - - return candidates - - -def _default_weights_dir_for_model_name(model_name_or_path: str) -> str | None: - if not model_name_or_path: - return None - for model_dir_name in _candidate_model_dir_names(model_name_or_path): - candidate = _PROJECT_ROOT / "weights" / model_dir_name - if candidate.exists(): - return str(candidate) - return None - - -def _normalized_source_candidates(source_name: str) -> list[str]: - candidates: list[str] = [] - - def _add(name: str) -> None: - if name and name not in candidates: - candidates.append(name) - - raw = source_name.strip() - _add(raw) - - for prefix in ("p_", "b_", "c_"): - if raw.startswith(prefix): - _add(raw[len(prefix):]) - - stripped = candidates[-1] - - if stripped.startswith("module.backbone."): - tail = stripped[len("module.backbone."):] - _add(f"model.{tail}") - _add(f"model.language_model.{tail}") - if tail.startswith("layers."): - _add(f"model.{tail}") - _add(f"model.language_model.{tail}") - - layer_match = re.match( - r"^(?:module_)?backbone_layers_slice_none__\d+__none____modules__(\d+)___(.+)$", - stripped, - ) - if layer_match: - layer_index = int(layer_match.group(1)) - tail = layer_match.group(2) - tail_map = { - "self_attn_q_proj_weight": "self_attn.q_proj.weight", - "self_attn_k_proj_weight": "self_attn.k_proj.weight", - "self_attn_v_proj_weight": "self_attn.v_proj.weight", - "self_attn_o_proj_weight": "self_attn.o_proj.weight", - "self_attn_q_norm_weight": "self_attn.q_norm.weight", - "self_attn_k_norm_weight": "self_attn.k_norm.weight", - "mlp_gate_proj_weight": "mlp.gate_proj.weight", - "mlp_up_proj_weight": "mlp.up_proj.weight", - "mlp_down_proj_weight": "mlp.down_proj.weight", - "input_layernorm_weight": "input_layernorm.weight", - "post_attention_layernorm_weight": "post_attention_layernorm.weight", - "linear_attn_in_proj_qkv_weight": "linear_attn.in_proj_qkv.weight", - "linear_attn_conv1d_weight": "linear_attn.conv1d.weight", - "linear_attn_norm_weight": "linear_attn.norm.weight", - "linear_attn_dt_bias": "linear_attn.dt_bias", - "linear_attn_A_log": "linear_attn.A_log", - } - dotted_tail = tail_map.get(tail) - if dotted_tail is not None: - _add(f"model.layers.{layer_index}.{dotted_tail}") - _add(f"model.language_model.layers.{layer_index}.{dotted_tail}") - - embed_map = { - "module_backbone_embed_tokens_weight": [ - "model.embed_tokens.weight", - "model.language_model.embed_tokens.weight", - ], - "module_backbone_embed_tokens_per_layer_weight": [ - "model.embed_tokens_per_layer.weight", - "model.language_model.embed_tokens_per_layer.weight", - ], - "module_backbone_per_layer_model_projection_weight": [ - "model.per_layer_model_projection.weight", - "model.language_model.per_layer_model_projection.weight", - ], - "module_backbone_norm_weight": [ - "model.norm.weight", - "model.language_model.norm.weight", - ], - "module_model_lm_head_weight": [ - "lm_head.weight", - ], - } - for key, mapped in embed_map.items(): - if stripped == key: - for item in mapped: - _add(item) - - return candidates - - -def resolve_transpile_weights_dir(graph_meta: dict[str, object]) -> str | None: - explicit = graph_meta.get("weights_dir") - if isinstance(explicit, str) and explicit: - return explicit - - family = str(graph_meta.get("adapter_family", "")).upper() - family_env = f"CACTUS_TRANSPILER_WEIGHTS_DIR_{family}" - if family and family_env in os.environ and os.environ[family_env]: - return os.environ[family_env] - - generic = os.environ.get("CACTUS_TRANSPILER_WEIGHTS_DIR") - if generic: - return generic - - model_name_or_path = graph_meta.get("model_name_or_path") - if isinstance(model_name_or_path, str) and model_name_or_path: - default_dir = _default_weights_dir_for_model_name(model_name_or_path) - if default_dir: - return default_dir - return None - - -def resolve_weight_binding(*, weights_dir: str | None, source_name: str) -> WeightBinding | None: - if not weights_dir: - return None - root = Path(weights_dir) - if not root.exists(): - return None - manifest_path = root / "weights_manifest.json" - if not manifest_path.exists(): - return None - try: - manifest = json.loads(manifest_path.read_text()) - except Exception: - return None - if not isinstance(manifest, dict): - return None - for candidate_name in _normalized_source_candidates(source_name): - entry = manifest.get(candidate_name) - if not isinstance(entry, dict): - continue - filename = entry.get("filename") - kind = entry.get("kind", "weight") - if not isinstance(filename, str) or not isinstance(kind, str): - continue - candidate = root / filename - if not candidate.exists(): - continue - return WeightBinding(path=str(candidate), kind=kind, source_name=candidate_name) - return None diff --git a/python/cactus/transpile/capture/capture_pytorch.py b/python/cactus/transpile/capture_pytorch.py similarity index 75% rename from python/cactus/transpile/capture/capture_pytorch.py rename to python/cactus/transpile/capture_pytorch.py index 0955d4d8d..00190dd89 100644 --- a/python/cactus/transpile/capture/capture_pytorch.py +++ b/python/cactus/transpile/capture_pytorch.py @@ -1,12 +1,15 @@ from dataclasses import dataclass import inspect +from contextlib import contextmanager from typing import Any from typing import Iterable import torch from torch.export import export +from torch.fx.passes.shape_prop import ShapeProp -from cactus.transpile.capture.graph_ir import IRGraph +from cactus.transpile.aten_ops import canonical_torch_op +from cactus.transpile.graph_ir import IRGraph class CapturePhaseError(RuntimeError): @@ -128,7 +131,6 @@ def _call_transpile_metadata_provider(module: torch.nn.Module, args: tuple[Any, def _collect_transpile_metadata(model: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: graph_meta: dict[str, Any] = {} provider_graph_meta: dict[str, dict[str, Any]] = {} - import_hints: list[dict[str, Any]] = [] seen_modules: set[int] = set() for module_path, module in model.named_modules(): @@ -148,25 +150,57 @@ def _collect_transpile_metadata(model: torch.nn.Module, args: tuple[Any, ...], k if not module_path: graph_meta.update(provider_meta) - raw_hints = metadata.get("import_hints", ()) - for raw_hint in raw_hints: - if not isinstance(raw_hint, dict): - raise TypeError(f"{type(module).__name__} import hints must be dictionaries") - hint = dict(raw_hint) - hint.setdefault("provider", provider_key) - import_hints.append(hint) - if provider_graph_meta: graph_meta.setdefault("transpile_metadata_providers", provider_graph_meta) - if import_hints: - graph_meta["import_hint_count"] = len(import_hints) - if not graph_meta and not import_hints: + if not graph_meta: return {} - return { - "graph": graph_meta, - "import_hints": import_hints, - } + return {"graph": graph_meta} + + +def _prepare_import_graph_module(ep: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + graph_module = ep.graph_module + if kwargs: + return graph_module + + try: + graph_module = ep.module() + ShapeProp(graph_module).propagate(*args) + except Exception: + return ep.graph_module + return graph_module + + +@contextmanager +def _suppress_transformers_model_output_registration(): + """Avoid tracing a Transformers global-set mutation inside ModelOutput init. + + Recent Transformers versions lazily register ModelOutput dataclasses as + PyTrees from their __post_init__. That side effect is irrelevant for our + capture wrapper because the exported boundary returns tensors, but Dynamo + sees the intermediate set membership check and rejects it. + """ + + try: + import transformers.utils.generic as hf_generic + except Exception: + yield + return + + original = getattr(hf_generic, "_register_model_output_pytree_node", None) + if original is None: + yield + return + + def _noop_register_model_output_pytree_node(*_args: Any, **_kwargs: Any) -> None: + return None + + hf_generic._register_model_output_pytree_node = _noop_register_model_output_pytree_node + try: + yield + finally: + hf_generic._register_model_output_pytree_node = original + def capture_model(model, args, kwargs=None, *, strict=True) -> CapturedModel: if not isinstance(model, torch.nn.Module): @@ -183,19 +217,22 @@ def capture_model(model, args, kwargs=None, *, strict=True) -> CapturedModel: transpile_metadata = _collect_transpile_metadata(model, example_args, example_kwargs) try: - ep = export(model, args=example_args, kwargs=example_kwargs, strict=strict) + with _suppress_transformers_model_output_registration(): + ep = export(model, args=example_args, kwargs=example_kwargs, strict=strict) except Exception as exc: raise CapturePhaseError( "export", f"torch.export failed for model={type(model).__name__} strict={strict}: {exc}", cause=exc, ) from exc - from cactus.transpile.capture.import_ir import import_captured_to_ir + from cactus.transpile.import_ir import import_captured_to_ir + + import_graph_module = _prepare_import_graph_module(ep, example_args, example_kwargs) raw_captured = CapturedModel( exported_program=ep, - graph_module=ep.graph_module, - graph=ep.graph, + graph_module=import_graph_module, + graph=import_graph_module.graph, state_dict=dict(ep.state_dict), ir_graph=IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[]), source_module=model, @@ -242,16 +279,42 @@ def resolve_attr(root: Any, target: str) -> Any: def format_target(node: Any) -> str: - return str(node.target) + return canonical_torch_op(node.target) + + +def _try_materialize_int_tuple(values: Iterable[Any]) -> tuple[int, ...] | None: + materialized: list[int] = [] + try: + for value in values: + materialized.append(int(value)) + except Exception: + return None + return tuple(materialized) + def get_tensor_metadata(node: Any) -> TensorMetadata | None: meta = getattr(node, "meta", {}) or {} + tensor_meta = meta.get("tensor_meta") + if tensor_meta is not None: + shape = tuple(getattr(tensor_meta, "shape", ())) + dtype = getattr(tensor_meta, "dtype", None) + stride_value = getattr(tensor_meta, "stride", None) + stride = _try_materialize_int_tuple(stride_value) if isinstance(stride_value, Iterable) else None + device = getattr(tensor_meta, "device", None) + if shape or dtype is not None or stride is not None or device is not None: + return TensorMetadata( + shape=shape, + dtype=dtype, + stride=stride, + device=device, + ) + value = meta.get("val") if value is None: return None if isinstance(value, torch.Tensor): - stride = tuple(int(s) for s in value.stride()) if value.layout == torch.strided else None + stride = _try_materialize_int_tuple(value.stride()) if value.layout == torch.strided else None return TensorMetadata( shape=tuple(value.shape), dtype=value.dtype, diff --git a/python/cactus/transpile/component_bundle_runtime.py b/python/cactus/transpile/component_bundle_runtime.py new file mode 100644 index 000000000..5eafe86a4 --- /dev/null +++ b/python/cactus/transpile/component_bundle_runtime.py @@ -0,0 +1,3021 @@ +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field +import ctypes +import json +import os +from collections.abc import Mapping +from pathlib import Path +import re +import struct +import time +from typing import Any + +import numpy as np +import torch + +from cactus.convert.cactus_adapters.tensor_io import CACTUS_MAGIC +from cactus.convert.cactus_adapters.tensor_io import FLAG_INTERLEAVED +from cactus.convert.cactus_adapters.tensor_io import align_offset +from cactus.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features +from cactus.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform +from cactus.transpile.audio_preprocess import prepare_cactus_audio_features +from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph +from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs +from cactus.transpile.multimodal_runtime import _build_gemma4_chat_prompt +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.media_limits import resize_static_image +from cactus.transpile.lower import transpile_preoptimized_ir +from cactus.transpile.optimize_graph import optimize_graph +from cactus.transpile.runtime_support import ensure_transformers_supports_gemma4 +from cactus.transpile.runtime_support import patch_torch_flex_attention_compat +from cactus.transpile.runtime_support import patch_transformers_torchvision_probe +from cactus.transpile.runtime_support import PreparedInputs +from cactus.transpile.tdt_runtime import greedy_decode_parakeet_tdt_token_ids +from cactus.transpile.tdt_runtime import load_parakeet_tdt_config +from cactus.transpile.tdt_runtime import prepare_parakeet_tdt_audio_features +from cactus.transpile.runtime_compat import _lib +from cactus.transpile.runtime_compat import cactus_node_t +from cactus.transpile.runtime_compat import Graph +from cactus.transpile.runtime_compat import Tensor + + +_HEADER_SIZE = 84 +_FLAG_EXTENDED_SHAPE = 1 << 4 +_PRECISION_TO_DTYPE = { + Graph.INT8: np.int8, + Graph.FP16: np.float16, + Graph.FP32: np.float32, + Graph.INT4: np.uint8, + getattr(Graph, "CQ2", 4): np.uint8, + getattr(Graph, "CQ3", 5): np.uint8, + getattr(Graph, "CQ4", 6): np.uint8, +} + +_STATEFUL_DECODE_COMPONENTS = frozenset({"decoder_prefill_chunk", "decoder_step"}) +_COMPONENT_GRAPH_CACHE: dict[tuple[str, str | None, str], tuple[dict[str, "LoadedComponentGraph"], dict[str, object]]] = {} +_MULTIMODAL_ENCODER_FEATURE_CACHE: dict[tuple[str, str, tuple[str, ...], str | None], dict[str, np.ndarray]] = {} +_UNBOUNDED_GENERATION_GUARD_TOKENS = 512 + + +def _has_runtime_symbol(name: str) -> bool: + symbol = getattr(_lib, name, None) + return symbol is not None and not getattr(symbol, "_cactus_missing_symbol", False) + + +@dataclass +class LoadedTensorFile: + path: Path + precision: int + shape: tuple[int, ...] + data: np.memmap + scales: np.memmap | None + group_size: int + num_groups: int + is_interleaved: bool + original_n: int + + +@dataclass +class LoadedComponentGraph: + component: str + graph: Graph + runtime_inputs: list[Tensor] + outputs: list[Tensor] + bound_constant_bindings: list[dict[str, object]] + bound_tensor_files: list[object] + cache_state_tensors: list[tuple[str, Tensor, Tensor]] = field(default_factory=list) + external_input_refs: dict[int, np.ndarray] = field(default_factory=dict) + + def set_input(self, index: int, data: Any, *, dtype: int | None = None) -> None: + if index < 0 or index >= len(self.runtime_inputs): + raise IndexError( + f"runtime input index out of range: {index} (have {len(self.runtime_inputs)})" + ) + self.graph.set_input(self.runtime_inputs[index], data, dtype=dtype) + + def set_inputs(self, inputs: list[Any] | tuple[Any, ...]) -> None: + if len(inputs) != len(self.runtime_inputs): + raise ValueError( + f"expected {len(self.runtime_inputs)} runtime inputs, got {len(inputs)}" + ) + for index, value in enumerate(inputs): + self.set_input(index, value) + + def set_external_input(self, index: int, data: Any, *, dtype: int | None = None) -> np.ndarray: + if index < 0 or index >= len(self.runtime_inputs): + raise IndexError( + f"runtime input index out of range: {index} (have {len(self.runtime_inputs)})" + ) + tensor = self.runtime_inputs[index] + target_dtype = int(tensor.dtype if dtype is None else dtype) + array = self.graph._coerce_input_array(data, target_dtype) + self.graph.set_external_input(tensor, int(array.ctypes.data), dtype=target_dtype) + self.external_input_refs[index] = array + return array + + def set_external_inputs(self, inputs: list[Any] | tuple[Any, ...]) -> list[np.ndarray]: + if len(inputs) != len(self.runtime_inputs): + raise ValueError( + f"expected {len(self.runtime_inputs)} runtime inputs, got {len(inputs)}" + ) + bound: list[np.ndarray] = [] + for index, value in enumerate(inputs): + bound.append(self.set_external_input(index, value)) + return bound + + def execute(self) -> list[Tensor]: + self.graph.execute() + return self.outputs + + +def load_component_bundle_manifest(bundle_dir_or_manifest: str | Path) -> tuple[Path, dict[str, object]]: + path = Path(bundle_dir_or_manifest).expanduser().resolve() + if path.is_dir(): + candidate = path / "manifest.json" if path.name == "components" else path / "components" / "manifest.json" + if not candidate.exists(): + candidate = path / "manifest.json" + path = candidate + if not path.exists(): + raise FileNotFoundError(f"component bundle manifest not found: {path}") + manifest = json.loads(path.read_text()) + bundle_root = path.parent.parent if path.parent.name == "components" else path.parent + return bundle_root, manifest + + +def _component_cache_state_tensors( + graph: Graph, + component_entry: Mapping[str, object], +) -> list[tuple[str, Tensor, Tensor]]: + result: list[tuple[str, Tensor, Tensor]] = [] + for entry in component_entry.get("cache_state_node_ids", []) or []: + if not isinstance(entry, Mapping): + continue + layer_key = str(entry.get("layer_key", "")) + key_id = entry.get("key") + value_id = entry.get("value") + if not isinstance(key_id, int) or not isinstance(value_id, int): + continue + result.append((layer_key, graph._tensor_from_node(int(key_id)), graph._tensor_from_node(int(value_id)))) + return result + + +def load_saved_component_graph( + *, + bundle_root: str | Path, + component_entry: dict[str, object], + weights_dir: str | Path | None = None, +) -> LoadedComponentGraph: + root = Path(bundle_root).expanduser().resolve() + graph_relpath = component_entry.get("graph") + has_graph = isinstance(graph_relpath, str) and bool(graph_relpath) + graph_path = (root / str(graph_relpath)).resolve() if has_graph else None + prefer_saved_graph = os.environ.get("CACTUS_TRANSPILER_PREFER_SAVED_GRAPH", "1") != "0" + if prefer_saved_graph and graph_path is not None and graph_path.exists(): + try: + graph = Graph.load(graph_path) + + runtime_inputs = [ + graph._tensor_from_node(int(node_id)) + for node_id in component_entry.get("runtime_input_node_ids", []) + ] + outputs = [ + graph._tensor_from_node(int(node_id)) + for node_id in component_entry.get("output_node_ids", []) + ] + bound_constant_bindings = list(component_entry.get("bound_constant_bindings") or []) + bound_tensor_files = _rebind_bound_constants( + graph=graph, + bundle_root=root, + bindings=bound_constant_bindings, + weights_dir=weights_dir, + ) + return LoadedComponentGraph( + component=str(component_entry.get("component", "unknown")), + graph=graph, + runtime_inputs=runtime_inputs, + outputs=outputs, + bound_constant_bindings=bound_constant_bindings, + bound_tensor_files=bound_tensor_files, + cache_state_tensors=_component_cache_state_tensors(graph, component_entry), + ) + except Exception as exc: + if not ( + isinstance(component_entry.get("raw_ir"), str) + or isinstance(component_entry.get("optimized_ir"), str) + ): + raise + print( + f"note=component_graph_load_failed component={component_entry.get('component', 'unknown')} " + f"path={graph_path} fallback=ir reason={exc}", + flush=True, + ) + + raw_ir_relpath = component_entry.get("raw_ir") + optimized_ir_relpath = component_entry.get("optimized_ir") + has_raw_ir = isinstance(raw_ir_relpath, str) and bool(raw_ir_relpath) + has_optimized_ir = isinstance(optimized_ir_relpath, str) and bool(optimized_ir_relpath) + if has_raw_ir or has_optimized_ir: + return _load_component_graph_from_ir( + bundle_root=root, + component_entry=component_entry, + weights_dir=weights_dir, + ) + + if not isinstance(graph_relpath, str) or not graph_relpath: + raise ValueError(f"component entry is missing graph path: {component_entry}") + + graph_path = (root / graph_relpath).resolve() + graph = Graph.load(graph_path) + + runtime_inputs = [ + graph._tensor_from_node(int(node_id)) + for node_id in component_entry.get("runtime_input_node_ids", []) + ] + outputs = [ + graph._tensor_from_node(int(node_id)) + for node_id in component_entry.get("output_node_ids", []) + ] + bound_constant_bindings = list(component_entry.get("bound_constant_bindings") or []) + bound_tensor_files = _rebind_bound_constants( + graph=graph, + bundle_root=root, + bindings=bound_constant_bindings, + weights_dir=weights_dir, + ) + return LoadedComponentGraph( + component=str(component_entry.get("component", "unknown")), + graph=graph, + runtime_inputs=runtime_inputs, + outputs=outputs, + bound_constant_bindings=bound_constant_bindings, + bound_tensor_files=bound_tensor_files, + cache_state_tensors=_component_cache_state_tensors(graph, component_entry), + ) + + +def _load_component_graph_from_ir( + *, + bundle_root: Path, + component_entry: dict[str, object], + weights_dir: str | Path | None, +) -> LoadedComponentGraph: + raw_ir_relpath = component_entry.get("raw_ir") + optimized_ir_relpath = component_entry.get("optimized_ir") + has_raw_ir = isinstance(raw_ir_relpath, str) and bool(raw_ir_relpath) + has_optimized_ir = isinstance(optimized_ir_relpath, str) and bool(optimized_ir_relpath) + use_raw_ir = has_raw_ir and not has_optimized_ir + ir_relpath = str(raw_ir_relpath if use_raw_ir else optimized_ir_relpath) + ir_path = (bundle_root / ir_relpath).resolve() + payload = json.loads(ir_path.read_text()) + graph_payload = payload.get("graph") + if not isinstance(graph_payload, dict): + raise ValueError(f"saved IR payload is missing graph data: {ir_path}") + + ir_graph = _deserialize_saved_ir_graph( + graph_payload=graph_payload, + component_entry=component_entry, + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + if use_raw_ir: + canonicalize_exported_graph(ir_graph) + optimize_graph(ir_graph) + else: + family = str(ir_graph.meta.get("adapter_family") or ir_graph.meta.get("family") or "").strip().lower() + component = str(ir_graph.meta.get("component", "") or component_entry.get("component", "") or "").strip().lower() + if family == "gemma4" and component in {"decoder", "decoder_step"}: + optimize_graph(ir_graph) + transpiled = transpile_preoptimized_ir(ir_graph) + return LoadedComponentGraph( + component=str(component_entry.get("component", "unknown")), + graph=transpiled.graph, + runtime_inputs=list(transpiled.runtime_inputs), + outputs=list(transpiled.outputs), + bound_constant_bindings=list(component_entry.get("bound_constant_bindings") or []), + bound_tensor_files=[], + cache_state_tensors=list(getattr(transpiled, "cache_state_tensors", [])), + ) + + +def load_saved_component_graphs( + bundle_dir_or_manifest: str | Path, + *, + weights_dir: str | Path | None = None, + include_components: set[str] | frozenset[str] | tuple[str, ...] | list[str] | None = None, +) -> tuple[dict[str, LoadedComponentGraph], dict[str, object]]: + bundle_root, manifest = load_component_bundle_manifest(bundle_dir_or_manifest) + include_component_names = ( + frozenset(str(name) for name in include_components) + if include_components is not None + else None + ) + cache_key = ( + str(bundle_root), + None if weights_dir is None else str(Path(weights_dir).expanduser().resolve()), + os.environ.get("CACTUS_TRANSPILER_PREFER_SAVED_GRAPH", ""), + None if include_component_names is None else tuple(sorted(include_component_names)), + ) + has_stateful_decode_graph = any( + isinstance(component_entry, dict) + and str(component_entry.get("component", "")).strip() in _STATEFUL_DECODE_COMPONENTS + and ( + include_component_names is None + or str(component_entry.get("component", "")).strip() in include_component_names + ) + for component_entry in manifest.get("components", []) + ) + cache_components = os.environ.get("CACTUS_TRANSPILER_DISABLE_GRAPH_CACHE") != "1" + if cache_components: + cached = _COMPONENT_GRAPH_CACHE.get(cache_key) + if cached is not None: + if not has_stateful_decode_graph: + _attach_component_io_names(manifest, cached[0]) + return cached + loaded = dict(cached[0]) + for component_entry in manifest.get("components", []): + if not isinstance(component_entry, dict): + continue + component_name = str(component_entry.get("component", "")).strip() + if component_name not in _STATEFUL_DECODE_COMPONENTS: + continue + if include_component_names is not None and component_name not in include_component_names: + continue + loaded[component_name] = load_saved_component_graph( + bundle_root=bundle_root, + component_entry=component_entry, + weights_dir=weights_dir, + ) + _attach_component_io_names(manifest, loaded) + return loaded, manifest + + loaded: dict[str, LoadedComponentGraph] = {} + for component_entry in manifest.get("components", []): + if not isinstance(component_entry, dict): + continue + component_name = str(component_entry.get("component", "")).strip() + if not component_name: + continue + if include_component_names is not None and component_name not in include_component_names: + continue + loaded[component_name] = load_saved_component_graph( + bundle_root=bundle_root, + component_entry=component_entry, + weights_dir=weights_dir, + ) + _attach_component_io_names(manifest, loaded) + result = (loaded, manifest) + if cache_components: + if has_stateful_decode_graph: + _COMPONENT_GRAPH_CACHE[cache_key] = ( + {name: graph for name, graph in loaded.items() if name not in _STATEFUL_DECODE_COMPONENTS}, + manifest, + ) + else: + _COMPONENT_GRAPH_CACHE[cache_key] = result + return result + + +def run_transpiled_bundle( + bundle_dir_or_manifest: str | Path, + *, + audio_file: str | Path | None = None, + image_files: tuple[str, ...] = (), + prompt: str | None = None, + input_ids: str | list[int] | tuple[int, ...] | None = None, + weights_dir: str | Path | None = None, + torch_dtype: torch.dtype = torch.float16, + system_prompt: str | None = None, + enable_thinking: bool = False, + max_new_tokens: int | None = None, + stop_sequences: tuple[str, ...] = (), +) -> dict[str, object]: + bundle_root, manifest = load_component_bundle_manifest(bundle_dir_or_manifest) + manifest = dict(manifest) + manifest["_bundle_root"] = str(bundle_root) + resolved_weights_dir = _default_weights_dir_for_manifest(manifest, explicit=weights_dir) + family = str(manifest.get("family", "") or "") + task = str(manifest.get("task", "") or "") + + include_components = runtime_include_components_for_manifest( + family=family, + task=task, + manifest=manifest, + ) + if ( + family == "gemma4" + and task == "multimodal_causal_lm_logits" + and prompt is not None + and not image_files + and audio_file is None + ): + manifest_components = { + str(component_entry.get("component", "")).strip() + for component_entry in manifest.get("components", []) + if isinstance(component_entry, dict) + } + if {"lm_encoder_step", "decoder_step"}.issubset(manifest_components): + include_components = {"lm_encoder_step", "decoder_step"} + component_graphs, manifest = load_saved_component_graphs( + bundle_dir_or_manifest, + weights_dir=resolved_weights_dir, + include_components=include_components, + ) + manifest = dict(manifest) + manifest["_bundle_root"] = str(bundle_root) + if family == "parakeet_tdt" and task == "tdt_transcription": + if audio_file is None: + raise ValueError("audio_file is required for Parakeet TDT component bundles") + return _run_parakeet_tdt_bundle( + component_graphs=component_graphs, + manifest=manifest, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + if task == "multimodal_causal_lm_logits": + return _run_multimodal_causal_lm_bundle( + component_graphs=component_graphs, + manifest=manifest, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking=enable_thinking, + max_new_tokens=max_new_tokens, + stop_sequences=stop_sequences, + ) + if task == "causal_lm_logits": + return _run_causal_lm_logits_bundle( + component_graphs=component_graphs, + manifest=manifest, + prompt=prompt, + input_ids=input_ids, + enable_thinking=enable_thinking, + max_new_tokens=max_new_tokens, + stop_sequences=stop_sequences, + ) + if task == "seq2seq_transcription": + if audio_file is None: + inputs_meta = manifest.get("inputs") + if isinstance(inputs_meta, dict): + stored_audio = inputs_meta.get("audio_file") + if isinstance(stored_audio, str) and stored_audio: + audio_file = stored_audio + if audio_file is None: + raise ValueError("audio_file is required for seq2seq_transcription bundles") + return _run_seq2seq_transcription_bundle( + component_graphs=component_graphs, + manifest=manifest, + audio_file=audio_file, + prompt=prompt, + torch_dtype=torch_dtype, + max_new_tokens=max_new_tokens, + stop_sequences=stop_sequences, + ) + if task == "encoder_hidden_states": + if audio_file is None: + inputs_meta = manifest.get("inputs") + if isinstance(inputs_meta, dict): + stored_audio = inputs_meta.get("audio_file") + if isinstance(stored_audio, str) and stored_audio: + audio_file = stored_audio + if audio_file is None: + raise ValueError("audio_file is required for encoder_hidden_states bundles") + return _run_encoder_hidden_states_bundle( + component_graphs=component_graphs, + manifest=manifest, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + raise NotImplementedError( + f"saved transpiled bundle execution is not implemented for family={family!r} task={task!r}" + ) + + +def runtime_include_components_for_manifest( + *, + family: str, + task: str, + manifest: Mapping[str, object], +) -> set[str] | None: + manifest_components = { + str(component_entry.get("component", "")).strip() + for component_entry in manifest.get("components", []) + if isinstance(component_entry, dict) + } + if ( + family == "gemma4" + and task == "multimodal_causal_lm_logits" + and {"lm_encoder_step", "decoder_step"}.issubset(manifest_components) + and os.environ.get("CACTUS_TRANSPILER_DISABLE_CACHED_STEP_DECODE") != "1" + ): + # Older Gemma bundles may include a full static decoder. Cached decode + # should not pay to load it when step graphs are available. + include_components = set(manifest_components) + include_components.discard("decoder") + return include_components + return None + + +def _default_weights_dir_for_manifest( + manifest: Mapping[str, object], + *, + explicit: str | Path | None, +) -> str | Path | None: + if explicit is not None: + return explicit + model_id = str(manifest.get("model_id", "") or "").strip() + if not model_id: + return None + try: + from cactus.cli.download import get_weights_dir + + candidate = get_weights_dir(model_id) + except Exception: + return None + return candidate if candidate.exists() else None + + +def _bundle_root_from_manifest(manifest: Mapping[str, object]) -> Path | None: + raw_root = manifest.get("_bundle_root") + if not isinstance(raw_root, str) or not raw_root: + return None + root = Path(raw_root).expanduser().resolve() + return root if root.exists() else None + + +def _looks_like_tokenizer_source(path: Path) -> bool: + return any( + (path / filename).exists() + for filename in ( + "tokenizer.json", + "tokenizer_config.json", + "vocab.txt", + "vocab.json", + "merges.txt", + "sentencepiece.bpe.model", + "tokenizer.model", + ) + ) + + +def _looks_like_processor_source(path: Path) -> bool: + return any( + (path / filename).exists() + for filename in ( + "processor_config.json", + "preprocessor_config.json", + "image_processor_config.json", + "feature_extractor_config.json", + ) + ) + + +def _pretrained_source_candidates( + manifest: Mapping[str, object], + *, + processor: bool, +) -> list[str]: + candidates: list[str] = [] + seen: set[str] = set() + + def add(value: object) -> None: + if not isinstance(value, str) or not value: + return + if value in seen: + return + seen.add(value) + candidates.append(value) + + bundle_root = _bundle_root_from_manifest(manifest) + if bundle_root is not None: + if processor: + if _looks_like_processor_source(bundle_root): + add(str(bundle_root)) + elif _looks_like_tokenizer_source(bundle_root): + add(str(bundle_root)) + + model_source = str(manifest.get("model_source", "") or "") + if model_source: + source_path = Path(model_source).expanduser() + if source_path.exists(): + add(str(source_path.resolve())) + elif not source_path.is_absolute(): + add(model_source) + + add(manifest.get("model_id")) + return candidates + + +def _deserialize_saved_ir_graph( + *, + graph_payload: Mapping[str, object], + component_entry: Mapping[str, object], + bundle_root: Path, + weights_dir: str | Path | None, +) -> IRGraph: + values_payload = graph_payload.get("values") + nodes_payload = graph_payload.get("nodes") + if not isinstance(values_payload, dict) or not isinstance(nodes_payload, list): + raise ValueError("saved optimized IR graph is missing values or nodes payload") + + values: dict[str, IRValue] = {} + for value_id, raw_value in values_payload.items(): + if not isinstance(value_id, str) or not isinstance(raw_value, dict): + continue + shape = raw_value.get("shape") + values[value_id] = IRValue( + id=str(raw_value.get("id", value_id)), + shape=None if shape is None else tuple(int(dim) for dim in shape), + dtype=None if raw_value.get("dtype") is None else str(raw_value.get("dtype")), + producer=None if raw_value.get("producer") is None else str(raw_value.get("producer")), + users=[str(item) for item in raw_value.get("users", [])], + meta=dict(raw_value.get("meta") or {}), + ) + + nodes: dict[str, IRNode] = {} + order: list[str] = [] + for raw_node in nodes_payload: + if not isinstance(raw_node, dict): + continue + node_id = str(raw_node["id"]) + node = IRNode( + id=node_id, + op=str(raw_node["op"]), + inputs=[str(item) for item in raw_node.get("inputs", [])], + outputs=[str(item) for item in raw_node.get("outputs", [])], + attrs=dict(raw_node.get("attrs") or {}), + meta=dict(raw_node.get("meta") or {}), + kind=str(raw_node.get("kind", "generic")), + ) + nodes[node_id] = node + order.append(node_id) + + constants_payload = graph_payload.get("constants") + if not isinstance(constants_payload, dict): + raise ValueError("saved optimized IR graph is missing constants payload") + + constant_bindings_by_value_id = { + str(binding.get("value_id")): binding + for binding in component_entry.get("bound_constant_bindings", []) + if isinstance(binding, dict) and isinstance(binding.get("value_id"), str) + } + + constants: dict[str, object] = {} + for value_id, serialized in constants_payload.items(): + if not isinstance(value_id, str): + continue + value = values.get(value_id) + if value is None: + continue + constants[value_id] = _deserialize_saved_ir_constant( + value=value, + serialized=serialized, + binding=constant_bindings_by_value_id.get(value_id), + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + + return IRGraph( + values=values, + nodes=nodes, + order=order, + inputs=[str(item) for item in graph_payload.get("inputs", [])], + outputs=[str(item) for item in graph_payload.get("outputs", [])], + constants=constants, + meta=dict(graph_payload.get("meta") or {}), + ) + + +def _deserialize_saved_ir_constant( + *, + value: IRValue, + serialized: object, + binding: Mapping[str, object] | None, + bundle_root: Path, + weights_dir: str | Path | None, +) -> object: + def _torch_dtype_from_name(name: str) -> torch.dtype | None: + return { + "torch.bool": torch.bool, + "torch.uint8": torch.uint8, + "torch.int8": torch.int8, + "torch.int16": torch.int16, + "torch.int32": torch.int32, + "torch.int64": torch.int64, + "torch.float16": torch.float16, + "torch.float32": torch.float32, + "torch.float64": torch.float64, + }.get(name) + + meta = value.meta if isinstance(value.meta, dict) else {} + if isinstance(meta.get("path"), str) and isinstance(meta.get("kind"), str): + # The lowerer ignores the constant payload when a weight binding is present + # in IRValue metadata and re-attaches the mmap-backed tensor directly. + return 0 + + if isinstance(binding, Mapping): + binding_format = str(binding.get("format", "tensor_io") or "tensor_io") + if binding_format != "tensor_io": + raise RuntimeError( + f"unsupported bound constant format {binding_format!r}; re-run cactus convert to rebuild the bundle" + ) + tensor_path = _resolve_bound_tensor_path( + str(binding["path"]), + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + value.meta = { + **meta, + "path": str(tensor_path), + "kind": str(binding.get("kind", "saved_constant") or "saved_constant"), + "source_name": str(binding.get("source_name", value.id) or value.id), + } + return 0 + + if isinstance(serialized, (str, int, float, bool)) or serialized is None: + return serialized + + if isinstance(serialized, list): + return serialized + + if isinstance(serialized, dict): + value_type = str(serialized.get("type", "")) + if value_type in {"torch.Tensor", "numpy.ndarray"}: + shape = tuple(int(dim) for dim in serialized.get("shape", []) or []) + if not shape: + dtype = _torch_dtype_from_name(str(serialized.get("dtype", ""))) or torch.float32 + zero_scalar = False if dtype is torch.bool else 0 + return torch.tensor(zero_scalar, dtype=dtype) + raise ValueError( + "saved optimized IR is missing a materialized constant payload for " + f"{value.id} with shape={shape}; expected a bound_constants entry" + ) + return dict(serialized) + + return serialized + + +def execute_loaded_component_pipeline( + components: list[LoadedComponentGraph], + *, + initial_store: dict[str, Any], +) -> tuple[dict[str, np.ndarray], dict[str, list[np.ndarray]]]: + store: dict[str, np.ndarray] = {} + for key, value in initial_store.items(): + store[key] = _to_numpy(value) + + outputs_by_component: dict[str, list[np.ndarray]] = {} + for component in components: + runtime_inputs = [] + input_names = component_input_names(component) + for input_name in input_names: + if input_name not in store: + raise KeyError( + f"component {component.component} is missing pipeline input {input_name!r}" + ) + runtime_inputs.append(store[input_name]) + for tensor, value, input_name in zip( + component.runtime_inputs, + runtime_inputs, + input_names, + strict=True, + ): + expected_shape = tuple(int(dim) for dim in tensor.shape) + actual_shape = tuple(int(dim) for dim in np.asarray(value).shape) + if actual_shape != expected_shape: + raise ValueError( + f"component {component.component} input {input_name!r} shape mismatch: " + f"expected {expected_shape}, got {actual_shape}" + ) + component.set_external_inputs(runtime_inputs) + raw_outputs = component.execute() + numpy_outputs = [output.numpy().copy() for output in raw_outputs] + output_names = component_output_names(component) + if len(numpy_outputs) != len(output_names): + raise ValueError( + f"component {component.component} produced {len(numpy_outputs)} outputs, " + f"expected {len(output_names)}" + ) + for output_name, value in zip(output_names, numpy_outputs, strict=True): + store[output_name] = value + outputs_by_component[component.component] = numpy_outputs + return store, outputs_by_component + + +def execute_loaded_component( + component: LoadedComponentGraph, + store: dict[str, np.ndarray], +) -> list[np.ndarray]: + runtime_inputs = [] + input_names = component_input_names(component) + for input_name in input_names: + if input_name not in store: + raise KeyError( + f"component {component.component} is missing pipeline input {input_name!r}" + ) + runtime_inputs.append(store[input_name]) + for tensor, value, input_name in zip( + component.runtime_inputs, + runtime_inputs, + input_names, + strict=True, + ): + expected_shape = tuple(int(dim) for dim in tensor.shape) + actual_shape = tuple(int(dim) for dim in np.asarray(value).shape) + if actual_shape != expected_shape: + raise ValueError( + f"component {component.component} input {input_name!r} shape mismatch: " + f"expected {expected_shape}, got {actual_shape}" + ) + component.set_external_inputs(runtime_inputs) + raw_outputs = component.execute() + numpy_outputs = [output.numpy().copy() for output in raw_outputs] + output_names = component_output_names(component) + if len(numpy_outputs) != len(output_names): + raise ValueError( + f"component {component.component} produced {len(numpy_outputs)} outputs, " + f"expected {len(output_names)}" + ) + for output_name, value in zip(output_names, numpy_outputs, strict=True): + store[output_name] = value + return numpy_outputs + + +def component_input_names(component: LoadedComponentGraph) -> tuple[str, ...]: + return tuple(str(value) for value in getattr(component, "_input_names", ())) + + +def component_output_names(component: LoadedComponentGraph) -> tuple[str, ...]: + return tuple(str(value) for value in getattr(component, "_output_names", ())) + + +def _zero_component_outputs( + component: LoadedComponentGraph, +) -> dict[str, np.ndarray]: + outputs: dict[str, np.ndarray] = {} + for name, tensor in zip(component_output_names(component), component.outputs, strict=True): + dtype = _PRECISION_TO_DTYPE.get(int(tensor.dtype), np.float16) + shape = tuple(int(dim) for dim in tensor.shape) + outputs[name] = np.zeros(shape, dtype=dtype) + return outputs + + +def _seed_skipped_component_outputs( + store: dict[str, np.ndarray], + *, + component_graphs: dict[str, LoadedComponentGraph], + component_names: tuple[str, ...], +) -> None: + for component_name in component_names: + component = component_graphs.get(component_name) + if component is None: + continue + for output_name, value in _zero_component_outputs(component).items(): + store.setdefault(output_name, value) + + +def _bind_zero_input_buffers( + component: LoadedComponentGraph, + dtype_by_name: Mapping[str, np.dtype | type], +) -> dict[str, np.ndarray]: + input_names = component_input_names(component) + buffers: dict[str, np.ndarray] = {} + for name, tensor in zip(input_names, component.runtime_inputs, strict=True): + if name not in dtype_by_name: + raise RuntimeError(f"missing dtype for {component.component} input {name!r}") + shape = tuple(int(dim) for dim in tensor.shape) + buffers[name] = np.zeros(shape, dtype=dtype_by_name[name]) + bound = component.set_external_inputs([buffers[name] for name in input_names]) + return {name: bound[index] for index, name in enumerate(input_names)} + + +def _tensor_data_ptr(tensor: Tensor) -> int: + out_ptr = ctypes.c_void_p() + rc = _lib.cactus_graph_get_output_ptr(tensor.g.h, cactus_node_t(tensor.id), ctypes.byref(out_ptr)) + if rc != 0 or not out_ptr.value: + raise RuntimeError(f"graph_get_output_ptr failed for node {tensor.id}") + return int(out_ptr.value) + + +def _tensor_byte_size(tensor: Tensor) -> int: + info = tensor.g._get_output_info(tensor.id) + return int(info["byte_size"]) + + +def _copy_component_cache_states( + source: LoadedComponentGraph, + target: LoadedComponentGraph, +) -> None: + source_states = list(source.cache_state_tensors) + target_states = list(target.cache_state_tensors) + if not source_states or not target_states: + raise RuntimeError( + f"cannot transfer KV cache from {source.component} to {target.component}: " + "cache state metadata is missing" + ) + if len(source_states) != len(target_states): + raise RuntimeError( + f"cache state count mismatch: {source.component} has {len(source_states)}, " + f"{target.component} has {len(target_states)}" + ) + for (source_layer, source_k, source_v), (target_layer, target_k, target_v) in zip(source_states, target_states, strict=True): + if source_layer != target_layer: + raise RuntimeError(f"cache layer mismatch: {source_layer!r} != {target_layer!r}") + for source_tensor, target_tensor in ((source_k, target_k), (source_v, target_v)): + byte_size = _tensor_byte_size(source_tensor) + target_byte_size = _tensor_byte_size(target_tensor) + if byte_size != target_byte_size: + raise RuntimeError( + f"cache byte-size mismatch for layer {source_layer}: " + f"{byte_size} != {target_byte_size}" + ) + ctypes.memmove(_tensor_data_ptr(target_tensor), _tensor_data_ptr(source_tensor), byte_size) + + +def _copy_gemma4_decoder_inputs( + buffers: Mapping[str, np.ndarray], + *, + inputs_embeds: np.ndarray, + per_layer_inputs: np.ndarray, + position_ids: np.ndarray, +) -> None: + np.copyto( + buffers["inputs_embeds"], + np.asarray(inputs_embeds, dtype=buffers["inputs_embeds"].dtype), + ) + np.copyto( + buffers["per_layer_inputs"], + np.asarray(per_layer_inputs, dtype=buffers["per_layer_inputs"].dtype), + ) + np.copyto( + buffers["position_ids"], + np.asarray(position_ids, dtype=buffers["position_ids"].dtype), + ) + + +def _run_parakeet_tdt_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + audio_file: str | Path, + torch_dtype: torch.dtype, +) -> dict[str, object]: + if "audio_encoder" not in component_graphs or "decoder" not in component_graphs: + raise ValueError("Parakeet TDT component bundle must include audio_encoder and decoder graphs") + + inputs_meta = manifest.get("inputs") or {} + input_shapes = inputs_meta.get("input_shapes") if isinstance(inputs_meta, dict) else {} + if not isinstance(input_shapes, dict): + input_shapes = {} + expected_shape = input_shapes.get("input_features") + if not (isinstance(expected_shape, list) and len(expected_shape) == 3): + raise ValueError("Parakeet TDT bundle manifest is missing inputs.input_shapes.input_features") + + model_source = str(manifest.get("model_source", "") or "") + config = load_parakeet_tdt_config(model_source) + preprocess_start = time.perf_counter() + input_features, active_frames = prepare_parakeet_tdt_audio_features( + audio_file=audio_file, + expected_frames=int(expected_shape[1]), + expected_mels=int(expected_shape[2]), + torch_dtype=torch_dtype, + ) + preprocess_end = time.perf_counter() + + _attach_component_io_names(manifest, component_graphs) + encoder_start = time.perf_counter() + store, _ = execute_loaded_component_pipeline( + [component_graphs["audio_encoder"]], + initial_store={"input_features": input_features}, + ) + encoder_end = time.perf_counter() + encoder_hidden_states = np.asarray(store["encoder_hidden_states"]) + batch_size = int(encoder_hidden_states.shape[0]) + if batch_size != 1: + raise ValueError("Parakeet TDT saved bundle runtime currently expects batch size 1") + + state_dtype = np.float16 if torch_dtype == torch.float16 else np.float32 + initial_states: list[np.ndarray] = [] + for _ in range(config.predictor_num_layers): + state_shape = (batch_size, config.predictor_hidden_dim) + initial_states.append(np.zeros(state_shape, dtype=state_dtype)) + initial_states.append(np.zeros(state_shape, dtype=state_dtype)) + + decoder_component = component_graphs["decoder"] + decoder_steps = 0 + decoder_input_names = component_input_names(decoder_component) + decoder_input_buffers: dict[str, np.ndarray] = { + "encoder_frame": np.zeros((batch_size, int(encoder_hidden_states.shape[-1])), dtype=encoder_hidden_states.dtype), + "token_ids": np.zeros((batch_size,), dtype=np.int64), + } + for index in range(config.predictor_num_layers): + decoder_input_buffers[f"state_h_{index}"] = np.zeros( + (batch_size, config.predictor_hidden_dim), + dtype=state_dtype, + ) + decoder_input_buffers[f"state_c_{index}"] = np.zeros( + (batch_size, config.predictor_hidden_dim), + dtype=state_dtype, + ) + bound_decoder_inputs = decoder_component.set_external_inputs( + [decoder_input_buffers[name] for name in decoder_input_names] + ) + decoder_input_buffers = { + name: bound_decoder_inputs[index] for index, name in enumerate(decoder_input_names) + } + + def _step( + frame: np.ndarray, + token_id: int, + state_values: tuple[np.ndarray, ...], + ) -> tuple[np.ndarray, tuple[np.ndarray, ...]]: + nonlocal decoder_steps + decoder_steps += 1 + np.copyto(decoder_input_buffers["encoder_frame"], np.asarray(frame, dtype=encoder_hidden_states.dtype)) + decoder_input_buffers["token_ids"].fill(int(token_id)) + for index in range(config.predictor_num_layers): + np.copyto(decoder_input_buffers[f"state_h_{index}"], np.asarray(state_values[index * 2], dtype=state_dtype)) + np.copyto(decoder_input_buffers[f"state_c_{index}"], np.asarray(state_values[index * 2 + 1], dtype=state_dtype)) + outputs = decoder_component.execute() + logits = outputs[0].numpy().astype(np.float32, copy=False) + next_states = tuple(output.numpy() for output in outputs[1:]) + return logits, next_states + + decoder_start = time.perf_counter() + emitted = greedy_decode_parakeet_tdt_token_ids( + config=config, + encoder_hidden_states=encoder_hidden_states, + initial_states=tuple(initial_states), + step=_step, + ) + decoder_end = time.perf_counter() + total_end = decoder_end + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "audio_file": str(Path(audio_file).expanduser().resolve()), + "preprocess_ms": (preprocess_end - preprocess_start) * 1000.0, + "encoder_ms": (encoder_end - encoder_start) * 1000.0, + "decoder_ms": (decoder_end - decoder_start) * 1000.0, + "total_ms": (total_end - preprocess_start) * 1000.0, + "decoder_steps": decoder_steps, + "active_feature_frames": active_frames, + "token_ids": emitted, + "transcript": _decode_parakeet_tdt_token_ids(config.vocabulary, emitted), + "encoder_hidden_shape": list(encoder_hidden_states.shape), + "component_order": list(manifest.get("component_order", [])), + } + + +def _execute_multimodal_component_pipeline_for_generation( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + required_components: tuple[str, ...], + initial_store: dict[str, Any], + prompt_token_count: int, + image_files: tuple[str, ...], + audio_file: str | None, +) -> tuple[dict[str, np.ndarray], dict[str, list[np.ndarray]]]: + store: dict[str, np.ndarray] = { + key: _to_numpy(value) + for key, value in initial_store.items() + } + outputs_by_component: dict[str, list[np.ndarray]] = {} + family = str(manifest.get("family", "") or "").strip().lower() + cache_key = ( + str(manifest.get("model_id", "") or manifest.get("model_source", "") or ""), + family, + tuple(str(path) for path in image_files), + None if audio_file is None else str(audio_file), + ) + cached_features = _MULTIMODAL_ENCODER_FEATURE_CACHE.get(cache_key) + if cached_features is not None: + for key, value in cached_features.items(): + store[key] = np.ascontiguousarray(value) + + for component_name in required_components: + component = component_graphs[component_name] + output_names = component_output_names(component) + if ( + cached_features is not None + and component_name in {"vision_encoder", "audio_encoder"} + and output_names + and all(output_name in cached_features for output_name in output_names) + ): + for output_name in output_names: + store[output_name] = np.ascontiguousarray(cached_features[output_name]) + continue + if component_name == "decoder": + _right_align_decoder_inputs_to_static_tail( + store, + component=component, + prompt_token_count=prompt_token_count, + ) + outputs = execute_loaded_component(component, store) + outputs_by_component[component_name] = outputs + + if component_name in {"vision_encoder", "audio_encoder"}: + if all(name in store for name in output_names): + feature_payload = _MULTIMODAL_ENCODER_FEATURE_CACHE.setdefault(cache_key, {}) + for output_name in output_names: + feature_payload[output_name] = np.asarray(store[output_name]).copy() + + return store, outputs_by_component + + +def _right_align_decoder_inputs_to_static_tail( + store: dict[str, np.ndarray], + *, + component: LoadedComponentGraph, + prompt_token_count: int, +) -> None: + if prompt_token_count <= 0: + return + input_names = component_input_names(component) + if "inputs_embeds" not in input_names: + return + embeds = store.get("inputs_embeds") + if not isinstance(embeds, np.ndarray) or embeds.ndim < 2: + return + static_token_count = int(embeds.shape[1]) + valid_tokens = min(int(prompt_token_count), static_token_count) + if valid_tokens <= 0 or valid_tokens == static_token_count: + return + + for key in input_names: + value = store.get(key) + if not isinstance(value, np.ndarray) or value.ndim < 2: + continue + if int(value.shape[0]) != 1 or int(value.shape[1]) != static_token_count: + continue + shifted = np.zeros_like(value) + shifted[:, static_token_count - valid_tokens :, ...] = value[:, :valid_tokens, ...] + store[key] = np.ascontiguousarray(shifted) + + +def _run_multimodal_causal_lm_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + prompt: str | None, + image_files: tuple[str, ...], + audio_file: str | Path | None, + torch_dtype: torch.dtype, + system_prompt: str | None, + enable_thinking: bool, + max_new_tokens: int | None, + stop_sequences: tuple[str, ...], +) -> dict[str, object]: + family = str(manifest.get("family", "") or "").strip().lower() + use_cached_step_decode = ( + family == "gemma4" + and "lm_encoder_step" in component_graphs + and "decoder_step" in component_graphs + and os.environ.get("CACTUS_TRANSPILER_DISABLE_CACHED_STEP_DECODE") != "1" + ) + use_chunk_prefill = use_cached_step_decode and "decoder_prefill_chunk" in component_graphs + inputs_meta = manifest.get("inputs") if isinstance(manifest.get("inputs"), dict) else {} + resolved_prompt = prompt if prompt is not None else str(inputs_meta.get("prompt", "") or "") + if not resolved_prompt: + raise ValueError("provide --prompt for multimodal causal-LM bundles") + + allow_stored_media = prompt is None + resolved_image_files: tuple[str, ...] + if image_files: + resolved_image_files = tuple(str(Path(path).expanduser().resolve()) for path in image_files) + elif allow_stored_media: + stored_images = inputs_meta.get("image_files") + if isinstance(stored_images, list): + resolved_image_files = tuple(str(Path(path).expanduser().resolve()) for path in stored_images if isinstance(path, str) and path) + else: + resolved_image_files = () + else: + resolved_image_files = () + + resolved_audio: str | None = None + if audio_file is not None: + resolved_audio = str(Path(audio_file).expanduser().resolve()) + elif allow_stored_media: + stored_audio = inputs_meta.get("audio_file") + if isinstance(stored_audio, str) and stored_audio: + resolved_audio = str(Path(stored_audio).expanduser().resolve()) + + has_image = bool(resolved_image_files) + has_audio = resolved_audio is not None + if family == "gemma4" and not has_image and not has_audio: + if not use_cached_step_decode: + raise ValueError( + "Gemma4 text-only execution from a multimodal bundle requires " + "lm_encoder_step and decoder_step components; re-run cactus convert." + ) + return _run_gemma4_text_only_cached_bundle( + component_graphs=component_graphs, + manifest=manifest, + prompt=resolved_prompt, + input_ids=None, + enable_thinking=enable_thinking, + max_new_tokens=max_new_tokens, + stop_sequences=stop_sequences, + ) + + encoder_components: list[str] = [] + skipped_encoder_components: list[str] = [] + if has_image: + encoder_components.append("vision_encoder") + elif "vision_encoder" in component_graphs: + skipped_encoder_components.append("vision_encoder") + if has_audio: + encoder_components.append("audio_encoder") + elif "audio_encoder" in component_graphs: + skipped_encoder_components.append("audio_encoder") + if family == "lfm2_vl" and not has_image: + raise ValueError("provide --image or --image-file for LFM2-VL multimodal bundles") + + if use_cached_step_decode: + required_components = tuple(encoder_components) + ( + ("lm_encoder", "decoder_prefill_chunk") if use_chunk_prefill else ("lm_encoder",) + ) + else: + required_components = tuple(encoder_components) + ("lm_encoder", "decoder") + missing = [name for name in required_components if name not in component_graphs] + if missing: + raise ValueError( + "multimodal causal LM bundle requires components " + f"{required_components!r}, missing {missing!r}" + ) + + if family != "lfm2_vl": + external_transformers_site_packages = ensure_transformers_supports_gemma4() + if external_transformers_site_packages: + print( + "note=using external transformers install for gemma4 runtime: " + f"{external_transformers_site_packages}" + ) + patch_transformers_torchvision_probe() + patch_torch_flex_attention_compat() + + from transformers import AutoProcessor + + processor_sources = _pretrained_source_candidates(manifest, processor=True) + if not processor_sources: + raise ValueError("bundle manifest is missing model_source/model_id for multimodal preprocessing") + processor_errors: list[str] = [] + processor = None + for source in processor_sources: + try: + processor = AutoProcessor.from_pretrained( + source, + local_files_only=Path(source).exists(), + trust_remote_code=True, + ) + break + except Exception as exc: + processor_errors.append(f"{source}: {exc}") + if processor is None: + raise RuntimeError( + "failed to load tokenizer/processor assets for multimodal preprocessing. " + "Re-run cactus convert so processor files are copied into the weights folder. " + f"Tried: {'; '.join(processor_errors)}" + ) + if family == "lfm2_vl": + prepared = _prepare_lfm2_vl_multimodal_inputs_for_runtime( + processor, + prompt=resolved_prompt, + image_files=resolved_image_files, + torch_dtype=torch_dtype, + system_prompt=system_prompt or "", + enable_thinking_if_supported=enable_thinking, + ) + else: + prepared = prepare_gemma4_multimodal_inputs( + processor, + prompt=resolved_prompt, + image_files=resolved_image_files, + audio_file=resolved_audio, + torch_dtype=torch_dtype, + system_prompt=system_prompt or "", + enable_thinking_if_supported=enable_thinking, + use_gemma4_chat_template=True, + ) + + _attach_component_io_names(manifest, component_graphs) + prepared_store = { + name: tensor.detach().cpu().numpy() + for name, tensor in zip(prepared.names, prepared.tensors, strict=True) + } + unpadded_input_ids = prepared_store.get("input_ids") + unpadded_token_count = ( + int(unpadded_input_ids.shape[1]) + if isinstance(unpadded_input_ids, np.ndarray) and unpadded_input_ids.ndim >= 2 + else 0 + ) + tokenizer = getattr(processor, "tokenizer", processor) + _pad_prepared_store_to_static_input_shapes( + prepared_store, + inputs_meta=inputs_meta, + tokenizer=tokenizer, + ) + unpadded_token_count = _infer_multimodal_token_count( + prepared_store, + tokenizer=tokenizer, + inputs_meta=inputs_meta, + fallback=unpadded_token_count, + ) + _seed_skipped_component_outputs( + prepared_store, + component_graphs=component_graphs, + component_names=tuple(skipped_encoder_components), + ) + initial_components = ( + tuple(encoder_components) + ("lm_encoder",) + if use_cached_step_decode + else required_components + ) + + start = time.perf_counter() + store, _ = _execute_multimodal_component_pipeline_for_generation( + component_graphs=component_graphs, + manifest=manifest, + required_components=initial_components, + initial_store=prepared_store, + prompt_token_count=unpadded_token_count, + image_files=resolved_image_files, + audio_file=resolved_audio, + ) + tokenizer = getattr(processor, "tokenizer", processor) + generated_ids: list[int] = [] + logits_shape: list[int] | None = None + first_token_ms = 0.0 + stop_reason = "max_new_tokens" + + current_length = _infer_multimodal_token_count( + prepared_store, + tokenizer=tokenizer, + inputs_meta=inputs_meta, + fallback=unpadded_token_count, + ) + if use_cached_step_decode: + return _run_gemma4_cached_step_multimodal_decode( + component_graphs=component_graphs, + manifest=manifest, + store=store, + prepared_store=prepared_store, + tokenizer=tokenizer, + prompt=resolved_prompt, + image_files=resolved_image_files, + audio_file=resolved_audio, + current_length=current_length, + start=start, + max_new_tokens=max_new_tokens, + stop_sequences=stop_sequences, + ) + + static_token_count = _static_multimodal_token_count(prepared_store) + available_headroom = max(0, static_token_count - current_length) + requested_tokens = ( + _UNBOUNDED_GENERATION_GUARD_TOKENS + if max_new_tokens is None + else max(0, int(max_new_tokens)) + ) + token_budget = max(0, min(requested_tokens, max(1, available_headroom + 1))) + stop_token_ids = _bundle_stop_token_ids(manifest=manifest, tokenizer=tokenizer) + encoded_stop_sequences = _encode_stop_sequences(tokenizer, stop_sequences) + + for step_index in range(token_budget): + logits = np.asarray(store["logits"], dtype=np.float32) + logits_shape = list(logits.shape) + if logits.ndim != 3: + raise RuntimeError(f"expected logits with shape [batch, seq, vocab], got {list(logits.shape)}") + next_token_id = int(np.argmax(logits[0, -1])) + generated_ids.append(next_token_id) + if step_index == 0: + first_token_ms = (time.perf_counter() - start) * 1000.0 + + if next_token_id in stop_token_ids: + stop_reason = "stop_token" + break + if _trim_stop_suffix(generated_ids, encoded_stop_sequences): + stop_reason = "stop_sequence" + break + if step_index + 1 >= token_budget: + break + if current_length >= static_token_count: + stop_reason = "context_limit" + break + + _append_multimodal_token_in_place(store, current_length=current_length, token_id=next_token_id) + current_length += 1 + store, _ = _execute_multimodal_component_pipeline_for_generation( + component_graphs=component_graphs, + manifest=manifest, + required_components=("lm_encoder", "decoder"), + initial_store=store, + prompt_token_count=current_length, + image_files=resolved_image_files, + audio_file=resolved_audio, + ) + + end = time.perf_counter() + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() + if not response: + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False).strip() + decode_time_ms = max(0.0, (end - start) * 1000.0 - first_token_ms) + decode_tps = ( + ((len(generated_ids) - 1) * 1000.0) / decode_time_ms + if len(generated_ids) > 1 and decode_time_ms > 0.0 + else 0.0 + ) + first_generated_token_id = generated_ids[0] if generated_ids else None + first_generated_token = ( + _decode_generated_text(tokenizer, [first_generated_token_id], skip_special_tokens=False) + if first_generated_token_id is not None + else None + ) + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "component_order": list(manifest.get("component_order", [])), + "prompt": resolved_prompt, + "image_files": list(resolved_image_files), + "audio_file": resolved_audio, + "input_shapes": { + name: list(np.asarray(value).shape) + for name, value in prepared_store.items() + }, + "output_shape": logits_shape or [], + "total_ms": (end - start) * 1000.0, + "time_to_first_token_ms": first_token_ms, + "decode_tps": decode_tps, + "decode_tokens": len(generated_ids), + "generated_token_ids": generated_ids, + "response": response, + "stop_reason": stop_reason, + "next_token_id": first_generated_token_id, + "next_token": first_generated_token, + } + + +def _run_gemma4_text_only_cached_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + prompt: str | None, + input_ids: str | list[int] | tuple[int, ...] | None, + enable_thinking: bool, + max_new_tokens: int | None, + stop_sequences: tuple[str, ...], +) -> dict[str, object]: + if "lm_encoder_step" not in component_graphs or "decoder_step" not in component_graphs: + raise ValueError("Gemma4 text-only multimodal bundle requires lm_encoder_step and decoder_step") + + prompt_token_ids, tokenizer = _resolve_causal_lm_input_ids( + manifest=manifest, + prompt=prompt, + input_ids=input_ids, + enable_thinking=enable_thinking, + ) + if not prompt_token_ids: + raise ValueError("Gemma4 text-only bundle input token ids are empty") + if tokenizer is None: + try: + tokenizer = _load_bundle_tokenizer(manifest) + except Exception: + tokenizer = None + + _attach_component_io_names(manifest, component_graphs) + lm_encoder_step = component_graphs["lm_encoder_step"] + decoder_step = component_graphs["decoder_step"] + lm_encoder_input_buffers = _bind_zero_input_buffers( + lm_encoder_step, + {"input_ids": np.int64, "position_ids": np.int64}, + ) + decoder_dtypes = { + name: _PRECISION_TO_DTYPE.get(int(tensor.dtype), np.float16) + for name, tensor in zip(component_input_names(decoder_step), decoder_step.runtime_inputs, strict=True) + } + decoder_input_buffers = _bind_zero_input_buffers(decoder_step, decoder_dtypes) + + def _run_step_token(token_id: int, position_id: int, *, read_logits: bool) -> np.ndarray | None: + lm_encoder_input_buffers["input_ids"].fill(int(token_id)) + lm_encoder_input_buffers["position_ids"].fill(int(position_id)) + lm_encoder_step.graph.execute() + _copy_gemma4_decoder_inputs( + decoder_input_buffers, + inputs_embeds=lm_encoder_step.outputs[0].numpy(), + per_layer_inputs=lm_encoder_step.outputs[1].numpy(), + position_ids=lm_encoder_step.outputs[2].numpy(), + ) + decoder_step.graph.execute() + if not read_logits: + return None + return np.asarray(decoder_step.outputs[0].numpy()) + + requested_tokens = ( + _UNBOUNDED_GENERATION_GUARD_TOKENS + if max_new_tokens is None + else max(0, int(max_new_tokens)) + ) + stop_token_ids = _bundle_stop_token_ids(manifest=manifest, tokenizer=tokenizer) + encoded_stop_sequences = _encode_stop_sequences(tokenizer, stop_sequences) + generated_ids: list[int] = [] + logits_shape: list[int] | None = None + first_token_ms = 0.0 + stop_reason = "max_new_tokens" + logits: np.ndarray | None = None + + start = time.perf_counter() + for position_id, token_id in enumerate(prompt_token_ids): + logits = _run_step_token( + int(token_id), + int(position_id), + read_logits=position_id + 1 == len(prompt_token_ids), + ) + if logits is None: + raise RuntimeError("Gemma4 text-only cached decoder did not produce logits") + + token_position = len(prompt_token_ids) + for step_index in range(requested_tokens): + logits_shape = list(logits.shape) + if logits.ndim != 3: + raise RuntimeError(f"expected logits with shape [batch, seq, vocab], got {list(logits.shape)}") + next_token_id = int(np.argmax(logits[0, -1])) + generated_ids.append(next_token_id) + if step_index == 0: + first_token_ms = (time.perf_counter() - start) * 1000.0 + + if next_token_id in stop_token_ids: + stop_reason = "stop_token" + break + if _trim_stop_suffix(generated_ids, encoded_stop_sequences): + stop_reason = "stop_sequence" + break + if step_index + 1 >= requested_tokens: + if max_new_tokens is None: + stop_reason = "generation_guard" + break + + logits = _run_step_token( + next_token_id, + token_position, + read_logits=True, + ) + if logits is None: + raise RuntimeError("Gemma4 text-only cached decoder did not produce decode logits") + token_position += 1 + + end = time.perf_counter() + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() + if not response: + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False).strip() + decode_time_ms = max(0.0, (end - start) * 1000.0 - first_token_ms) + decode_tps = ( + ((len(generated_ids) - 1) * 1000.0) / decode_time_ms + if len(generated_ids) > 1 and decode_time_ms > 0.0 + else 0.0 + ) + prefill_tps = ( + (len(prompt_token_ids) * 1000.0) / first_token_ms + if first_token_ms > 0.0 + else 0.0 + ) + first_generated_token_id = generated_ids[0] if generated_ids else None + first_generated_token = ( + _decode_generated_text(tokenizer, [first_generated_token_id], skip_special_tokens=False) + if first_generated_token_id is not None + else None + ) + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "component_order": list(manifest.get("component_order", [])), + "prompt": prompt, + "input_ids": prompt_token_ids, + "output_shape": logits_shape or [], + "total_ms": (end - start) * 1000.0, + "time_to_first_token_ms": first_token_ms, + "prefill_tps": prefill_tps, + "decode_tps": decode_tps, + "decode_tokens": len(generated_ids), + "total_tokens": len(prompt_token_ids) + len(generated_ids), + "generated_token_ids": generated_ids, + "response": response, + "stop_reason": stop_reason, + "next_token_id": first_generated_token_id, + "next_token": first_generated_token, + "decode_mode": "cached_step_text", + } + + +def _run_gemma4_cached_step_multimodal_decode( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + store: dict[str, np.ndarray], + prepared_store: dict[str, np.ndarray], + tokenizer: object, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + current_length: int, + start: float, + max_new_tokens: int | None, + stop_sequences: tuple[str, ...], +) -> dict[str, object]: + """Run Gemma4 generation through a one-token decoder graph with internal KV state. + + The full-context decoder recomputes the entire text stack for every token. + The chunk graph primes K/V in fixed windows, then the step graph appends one + token at a time through the cached attention kernel. + """ + + lm_encoder_step = component_graphs["lm_encoder_step"] + decoder_step = component_graphs["decoder_step"] + decoder_prefill_chunk = component_graphs.get("decoder_prefill_chunk") + stop_token_ids = _bundle_stop_token_ids(manifest=manifest, tokenizer=tokenizer) + encoded_stop_sequences = _encode_stop_sequences(tokenizer, stop_sequences) + requested_tokens = ( + _UNBOUNDED_GENERATION_GUARD_TOKENS + if max_new_tokens is None + else max(0, int(max_new_tokens)) + ) + + inputs_embeds = np.asarray(store["inputs_embeds"]) + per_layer_inputs = np.asarray(store["per_layer_inputs"]) + position_ids = np.asarray(store["position_ids"]) + prompt_tokens = max(0, min(int(current_length), int(inputs_embeds.shape[1]))) + if prompt_tokens <= 0: + raise RuntimeError("Gemma4 cached decode requires at least one prompt token") + + generated_ids: list[int] = [] + logits_shape: list[int] | None = None + first_token_ms = 0.0 + stop_reason = "max_new_tokens" + + decoder_dtypes = { + "inputs_embeds": inputs_embeds.dtype, + "per_layer_inputs": per_layer_inputs.dtype, + "position_ids": position_ids.dtype, + } + decoder_input_buffers = _bind_zero_input_buffers(decoder_step, decoder_dtypes) + + def _run_decoder_step( + *, + step_inputs_embeds: np.ndarray, + step_per_layer_inputs: np.ndarray, + step_position_ids: np.ndarray, + read_logits: bool, + ) -> np.ndarray | None: + _copy_gemma4_decoder_inputs( + decoder_input_buffers, + inputs_embeds=step_inputs_embeds, + per_layer_inputs=step_per_layer_inputs, + position_ids=step_position_ids, + ) + decoder_step.graph.execute() + if not read_logits: + return None + return np.asarray(decoder_step.outputs[0].numpy()) + + lm_encoder_input_buffers = _bind_zero_input_buffers( + lm_encoder_step, + {"input_ids": np.int64, "position_ids": np.int64}, + ) + + prefill_input_buffers: dict[str, np.ndarray] | None = None + prefill_chunk_tokens = 0 + if decoder_prefill_chunk is not None: + prefill_input_buffers = _bind_zero_input_buffers(decoder_prefill_chunk, decoder_dtypes) + prefill_chunk_tokens = int(prefill_input_buffers["inputs_embeds"].shape[1]) + + logits: np.ndarray | None = None + prime_start = time.perf_counter() + token_index = 0 + if ( + decoder_prefill_chunk is not None + and prefill_input_buffers is not None + and prefill_chunk_tokens > 1 + and prompt_tokens >= prefill_chunk_tokens + ): + chunked_tokens = (prompt_tokens // prefill_chunk_tokens) * prefill_chunk_tokens + for chunk_start in range(0, chunked_tokens, prefill_chunk_tokens): + chunk_end = chunk_start + prefill_chunk_tokens + _copy_gemma4_decoder_inputs( + prefill_input_buffers, + inputs_embeds=inputs_embeds[:, chunk_start:chunk_end, :], + per_layer_inputs=per_layer_inputs[:, chunk_start:chunk_end, ...], + position_ids=position_ids[:, chunk_start:chunk_end], + ) + decoder_prefill_chunk.graph.execute() + if chunk_end == prompt_tokens: + logits = np.asarray(decoder_prefill_chunk.outputs[0].numpy()) + _copy_component_cache_states(decoder_prefill_chunk, decoder_step) + token_index = chunked_tokens + + while token_index < prompt_tokens: + logits = _run_decoder_step( + step_inputs_embeds=inputs_embeds[:, token_index : token_index + 1, :], + step_per_layer_inputs=per_layer_inputs[:, token_index : token_index + 1, ...], + step_position_ids=position_ids[:, token_index : token_index + 1], + read_logits=token_index + 1 == prompt_tokens, + ) + token_index += 1 + prime_end = time.perf_counter() + + if logits is None: + raise RuntimeError("Gemma4 cached decoder did not produce logits") + + token_position = prompt_tokens + for step_index in range(requested_tokens): + logits_shape = list(logits.shape) + if logits.ndim != 3: + raise RuntimeError(f"expected logits with shape [batch, seq, vocab], got {list(logits.shape)}") + next_token_id = int(np.argmax(logits[0, -1])) + generated_ids.append(next_token_id) + if step_index == 0: + first_token_ms = (time.perf_counter() - start) * 1000.0 + + if next_token_id in stop_token_ids: + stop_reason = "stop_token" + break + if _trim_stop_suffix(generated_ids, encoded_stop_sequences): + stop_reason = "stop_sequence" + break + if step_index + 1 >= requested_tokens: + if max_new_tokens is None: + stop_reason = "generation_guard" + break + + lm_encoder_input_buffers["input_ids"].fill(int(next_token_id)) + lm_encoder_input_buffers["position_ids"].fill(int(token_position)) + lm_encoder_step.graph.execute() + logits = _run_decoder_step( + step_inputs_embeds=lm_encoder_step.outputs[0].numpy(), + step_per_layer_inputs=lm_encoder_step.outputs[1].numpy(), + step_position_ids=lm_encoder_step.outputs[2].numpy(), + read_logits=True, + ) + if logits is None: + raise RuntimeError("Gemma4 cached decoder did not produce logits during decode") + token_position += 1 + + end = time.perf_counter() + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() + if not response: + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False).strip() + decode_time_ms = max(0.0, (end - start) * 1000.0 - first_token_ms) + decode_tps = ( + ((len(generated_ids) - 1) * 1000.0) / decode_time_ms + if len(generated_ids) > 1 and decode_time_ms > 0.0 + else 0.0 + ) + first_generated_token_id = generated_ids[0] if generated_ids else None + first_generated_token = ( + _decode_generated_text(tokenizer, [first_generated_token_id], skip_special_tokens=False) + if first_generated_token_id is not None + else None + ) + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "component_order": list(manifest.get("component_order", [])), + "prompt": prompt, + "image_files": list(image_files), + "audio_file": audio_file, + "input_shapes": { + name: list(np.asarray(value).shape) + for name, value in prepared_store.items() + }, + "output_shape": logits_shape or [], + "total_ms": (end - start) * 1000.0, + "time_to_first_token_ms": first_token_ms, + "cache_prime_ms": (prime_end - prime_start) * 1000.0, + "cache_prime_tokens": prompt_tokens, + "cache_prime_chunk_tokens": prefill_chunk_tokens, + "decode_tps": decode_tps, + "decode_tokens": len(generated_ids), + "generated_token_ids": generated_ids, + "response": response, + "stop_reason": stop_reason, + "next_token_id": first_generated_token_id, + "next_token": first_generated_token, + "decode_mode": "cached_step", + } + + +def _static_multimodal_token_count(store: Mapping[str, np.ndarray]) -> int: + for key in ("input_ids", "attention_mask", "token_type_ids", "inputs_embeds"): + value = store.get(key) + if isinstance(value, np.ndarray) and value.ndim >= 2: + return int(value.shape[1]) + return 0 + + +def _infer_multimodal_token_count( + store: Mapping[str, np.ndarray], + *, + tokenizer: object | None, + inputs_meta: Mapping[str, object], + fallback: int, +) -> int: + attention_mask = store.get("attention_mask") + if isinstance(attention_mask, np.ndarray) and attention_mask.ndim >= 2: + return int(np.count_nonzero(attention_mask[0])) + input_ids = store.get("input_ids") + if isinstance(input_ids, np.ndarray) and input_ids.ndim >= 2: + padding_token_id = _resolve_bundle_padding_token_id(inputs_meta, tokenizer) + return int(np.count_nonzero(input_ids[0] != int(padding_token_id))) + return max(0, int(fallback)) + + +def _append_multimodal_token_in_place( + store: dict[str, np.ndarray], + *, + current_length: int, + token_id: int, +) -> None: + input_ids = store.get("input_ids") + if isinstance(input_ids, np.ndarray) and input_ids.ndim >= 2 and current_length < input_ids.shape[1]: + input_ids[0, current_length] = int(token_id) + attention_mask = store.get("attention_mask") + if isinstance(attention_mask, np.ndarray) and attention_mask.ndim >= 2 and current_length < attention_mask.shape[1]: + attention_mask[0, current_length] = 1 + token_type_ids = store.get("token_type_ids") + if isinstance(token_type_ids, np.ndarray) and token_type_ids.ndim >= 2 and current_length < token_type_ids.shape[1]: + token_type_ids[0, current_length] = 0 + + +def _load_image_inputs_for_runtime(image_files: tuple[str, ...]) -> list[object]: + if not image_files: + return [] + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for --image: {exc}") from exc + + images: list[object] = [] + for image_file in image_files: + path = Path(image_file).resolve() + if not path.exists(): + raise RuntimeError(f"image file does not exist: {path}") + with Image.open(path) as image: + images.append(resize_static_image(image.convert("RGB")).copy()) + return images + + +def _prepare_lfm2_vl_multimodal_inputs_for_runtime( + processor: object | None, + *, + prompt: str, + image_files: tuple[str, ...], + torch_dtype: torch.dtype, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +): + if processor is None: + raise RuntimeError("LFM2-VL multimodal bundle execution requires an AutoProcessor with image support") + images = _load_image_inputs_for_runtime(image_files) + if not images: + raise RuntimeError("LFM2-VL multimodal bundle execution requires at least one --image") + + user_content: list[dict[str, object]] = [{"type": "image", "image": image} for image in images] + user_content.append({"type": "text", "text": prompt.strip()}) + + messages: list[dict[str, object]] = [] + normalized_system = system_prompt.strip() + if normalized_system: + messages.append({"role": "system", "content": normalized_system}) + messages.append({"role": "user", "content": user_content}) + + apply_chat_template = getattr(processor, "apply_chat_template", None) + image_placeholders = "\n".join("" for _ in images) + fallback_text = ( + f"<|startoftext|><|im_start|>user\n" + f"{image_placeholders}\n" + f"{prompt.strip()}<|im_end|>\n" + f"<|im_start|>assistant\n" + ) + if callable(apply_chat_template): + try: + batch = apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ) + except ValueError as exc: + if "chat template" not in str(exc).lower(): + raise + batch = processor(text=fallback_text, images=images, return_tensors="pt") + else: + batch = processor(text=fallback_text, images=images, return_tensors="pt") + + names = ("input_ids", "attention_mask", "pixel_values", "spatial_shapes", "pixel_attention_mask") + tensors: list[torch.Tensor] = [] + input_shapes: dict[str, list[int]] = {} + for key in names: + value = batch.get(key) if hasattr(batch, "get") else None + if not isinstance(value, torch.Tensor): + raise RuntimeError(f"LFM2-VL processor did not return required tensor input: {key}") + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + elif key == "pixel_attention_mask": + value = value.to(dtype=torch.int64) + else: + value = value.to(dtype=torch.long) + tensors.append(value) + input_shapes[key] = [int(dim) for dim in value.shape] + + return PreparedInputs( + names=names, + tensors=tuple(tensors), + metadata={ + "prompt": prompt, + "system_prompt": system_prompt, + "image_files": [str(Path(path).resolve()) for path in image_files], + "input_shapes": input_shapes, + "enable_thinking": bool(enable_thinking_if_supported), + }, + ) + + +def _run_causal_lm_logits_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + prompt: str | None, + input_ids: str | list[int] | tuple[int, ...] | None, + enable_thinking: bool, + max_new_tokens: int | None, + stop_sequences: tuple[str, ...], +) -> dict[str, object]: + if "decoder" not in component_graphs: + raise ValueError("causal LM component bundle must include a decoder graph") + + prompt_token_ids, tokenizer = _resolve_causal_lm_input_ids( + manifest=manifest, + prompt=prompt, + input_ids=input_ids, + enable_thinking=enable_thinking, + ) + if not prompt_token_ids: + raise ValueError("causal LM bundle input token ids are empty") + if tokenizer is None: + try: + tokenizer = _load_bundle_tokenizer(manifest) + except Exception: + tokenizer = None + + _attach_component_io_names(manifest, component_graphs) + decoder = component_graphs["decoder"] + runtime_inputs = component_input_names(decoder) + if runtime_inputs and runtime_inputs != ("input_ids",): + raise ValueError( + "causal LM bundle runner currently expects decoder logical input ('input_ids',), " + f"got {runtime_inputs!r}" + ) + + inputs_meta = manifest.get("inputs") + if not isinstance(inputs_meta, dict): + inputs_meta = {} + stored_input_ids = _parse_nested_manifest_input_ids(inputs_meta.get("input_ids")) or [] + stored_target_token_count = int(inputs_meta.get("target_token_count", 0) or 0) + target_token_count = max( + len(prompt_token_ids), + stored_target_token_count, + len(stored_input_ids), + ) + if target_token_count <= 0: + raise ValueError("causal LM bundle manifest did not provide a valid target token count") + if len(prompt_token_ids) > target_token_count: + raise ValueError( + f"prompt token length {len(prompt_token_ids)} exceeds transpiled bundle context {target_token_count}; " + "re-transpile with a larger --max-new-tokens budget or use a shorter prompt" + ) + + padding_token_id = _resolve_bundle_padding_token_id(inputs_meta, tokenizer) + input_array = np.full((1, target_token_count), padding_token_id, dtype=np.int64) + input_array[0, : len(prompt_token_ids)] = np.asarray(prompt_token_ids, dtype=np.int64) + input_array = decoder.set_external_input(0, input_array) + + available_headroom = max(0, target_token_count - len(prompt_token_ids)) + if max_new_tokens is None: + token_budget = available_headroom if available_headroom > 0 else 1 + else: + requested = max(0, int(max_new_tokens)) + if available_headroom > 0: + token_budget = min(requested, available_headroom) + else: + token_budget = 1 if requested > 0 else 0 + + stop_token_ids = _bundle_stop_token_ids(manifest=manifest, tokenizer=tokenizer) + encoded_stop_sequences = _encode_stop_sequences(tokenizer, stop_sequences) + generated_ids: list[int] = [] + logits_shape: list[int] | None = None + current_length = len(prompt_token_ids) + first_token_ms = 0.0 + stop_reason = "max_new_tokens" + start = time.perf_counter() + + for step_index in range(token_budget): + outputs = decoder.execute() + if not outputs: + raise RuntimeError("causal LM decoder graph produced no outputs") + logits = outputs[0].numpy() + logits_shape = list(logits.shape) + if logits.ndim != 3: + raise RuntimeError(f"expected logits with shape [batch, seq, vocab], got {list(logits.shape)}") + token_position = current_length - 1 + if logits.shape[1] == 1: + token_position = 0 + next_token_id = int(np.argmax(logits[0, token_position])) + generated_ids.append(next_token_id) + if step_index == 0: + first_token_ms = (time.perf_counter() - start) * 1000.0 + + if next_token_id in stop_token_ids: + stop_reason = "stop_token" + break + if _trim_stop_suffix(generated_ids, encoded_stop_sequences): + stop_reason = "stop_sequence" + break + + if current_length >= target_token_count: + stop_reason = "context_limit" + break + if step_index + 1 >= token_budget: + break + + input_array[0, current_length] = next_token_id + current_length += 1 + + end = time.perf_counter() + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() + if not response: + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False).strip() + decode_time_ms = max(0.0, (end - start) * 1000.0 - first_token_ms) + decode_tps = ( + ((len(generated_ids) - 1) * 1000.0) / decode_time_ms + if len(generated_ids) > 1 and decode_time_ms > 0.0 + else 0.0 + ) + prefill_tps = ( + (len(prompt_token_ids) * 1000.0) / first_token_ms + if first_token_ms > 0.0 + else 0.0 + ) + first_generated_token_id = generated_ids[0] if generated_ids else None + first_generated_token = None + if first_generated_token_id is not None: + first_generated_token = _decode_generated_text( + tokenizer, + [int(first_generated_token_id)], + skip_special_tokens=False, + ) + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "component_order": list(manifest.get("component_order", [])), + "input_ids": prompt_token_ids, + "input_shape": list(input_array.shape), + "output_shape": logits_shape or [], + "decoder_ms": (end - start) * 1000.0, + "total_ms": (end - start) * 1000.0, + "time_to_first_token_ms": first_token_ms, + "prefill_tps": prefill_tps, + "decode_tps": decode_tps, + "decode_tokens": len(generated_ids), + "total_tokens": len(prompt_token_ids) + len(generated_ids), + "generated_token_ids": generated_ids, + "response": response, + "stop_reason": stop_reason, + "next_token_id": first_generated_token_id, + "next_token": first_generated_token, + } + + +def _run_seq2seq_transcription_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + audio_file: str | Path, + prompt: str | None, + torch_dtype: torch.dtype, + max_new_tokens: int | None, + stop_sequences: tuple[str, ...], +) -> dict[str, object]: + if "audio_encoder" not in component_graphs or "decoder" not in component_graphs: + raise ValueError("seq2seq_transcription bundle must include audio_encoder and decoder graphs") + + inputs_meta = manifest.get("inputs") + if not isinstance(inputs_meta, dict): + inputs_meta = {} + input_shapes = inputs_meta.get("input_shapes") if isinstance(inputs_meta, dict) else {} + if not isinstance(input_shapes, dict): + input_shapes = {} + expected_shape = input_shapes.get("input_features") + if not (isinstance(expected_shape, list) and len(expected_shape) == 3): + raise ValueError("seq2seq_transcription bundle manifest is missing inputs.input_shapes.input_features") + + tokenizer = None + try: + tokenizer = _load_bundle_tokenizer(manifest) + except Exception: + tokenizer = None + + prompt_token_ids = _resolve_seq2seq_prompt_token_ids( + manifest=manifest, + prompt=prompt, + tokenizer=tokenizer, + ) + if not prompt_token_ids: + raise ValueError("seq2seq_transcription bundle input token ids are empty") + + _attach_component_io_names(manifest, component_graphs) + encoder = component_graphs["audio_encoder"] + decoder = component_graphs["decoder"] + encoder_inputs = component_input_names(encoder) + decoder_inputs = component_input_names(decoder) + if encoder_inputs and encoder_inputs != ("input_features",): + raise ValueError( + "seq2seq_transcription audio_encoder must accept logical input ('input_features',), " + f"got {encoder_inputs!r}" + ) + if decoder_inputs and decoder_inputs != ("decoder_input_ids", "encoder_hidden_states"): + raise ValueError( + "seq2seq_transcription decoder must accept logical inputs " + "('decoder_input_ids', 'encoder_hidden_states'), " + f"got {decoder_inputs!r}" + ) + + preprocess_start = time.perf_counter() + input_features, active_frames = _prepare_generic_audio_encoder_features( + audio_file=audio_file, + manifest=manifest, + expected_shape=expected_shape, + torch_dtype=torch_dtype, + ) + preprocess_end = time.perf_counter() + + encoder_start = time.perf_counter() + encoder.set_inputs([input_features]) + encoder_outputs = encoder.execute() + encoder_end = time.perf_counter() + if not encoder_outputs: + raise RuntimeError("seq2seq_transcription encoder graph produced no outputs") + encoder_hidden_states = np.asarray(encoder_outputs[0].numpy()) + + stored_target_token_count = int(inputs_meta.get("target_token_count", 0) or 0) + target_token_count = max(stored_target_token_count, len(prompt_token_ids)) + if target_token_count <= 0: + raise ValueError("seq2seq_transcription bundle manifest did not provide a valid target token count") + if len(prompt_token_ids) > target_token_count: + raise ValueError( + f"prompt token length {len(prompt_token_ids)} exceeds transpiled bundle context {target_token_count}; " + "re-transpile with a larger --max-new-tokens budget or use a shorter prompt" + ) + + padding_token_id = _resolve_bundle_padding_token_id(inputs_meta, tokenizer) + input_array = np.full((1, target_token_count), padding_token_id, dtype=np.int64) + input_array[0, : len(prompt_token_ids)] = np.asarray(prompt_token_ids, dtype=np.int64) + if hasattr(decoder, "set_external_inputs"): + bound_decoder_inputs = decoder.set_external_inputs([input_array, encoder_hidden_states]) + input_array = bound_decoder_inputs[0] + encoder_hidden_states = bound_decoder_inputs[1] + else: + decoder.set_inputs([input_array, encoder_hidden_states]) + + available_headroom = max(0, target_token_count - len(prompt_token_ids)) + if max_new_tokens is None: + token_budget = available_headroom if available_headroom > 0 else 1 + else: + requested = max(0, int(max_new_tokens)) + if available_headroom > 0: + token_budget = min(requested, available_headroom) + else: + token_budget = 1 if requested > 0 else 0 + + default_stop_sequences = ("<|endoftext|>", "<|endoftranscript|>", "", "") + resolved_stop_sequences = stop_sequences or default_stop_sequences + encoded_stop_sequences = _encode_stop_sequences(tokenizer, resolved_stop_sequences) + eos_token_id = inputs_meta.get("eos_token_id", getattr(tokenizer, "eos_token_id", None)) + suppress_tokens = [int(value) for value in inputs_meta.get("suppress_tokens", []) if isinstance(value, int)] + begin_suppress_tokens = [int(value) for value in inputs_meta.get("begin_suppress_tokens", []) if isinstance(value, int)] + if tokenizer is not None and ( + "whisper" in str(manifest.get("family", "") or "").lower() + or "whisper" in str(manifest.get("model_id", "") or "").lower() + ): + eos_int = int(eos_token_id) if isinstance(eos_token_id, int) else None + whisper_special_ids = [ + int(token_id) + for token_id in getattr(tokenizer, "all_special_ids", []) or [] + if eos_int is None or int(token_id) != eos_int + ] + suppress_tokens = sorted(set(suppress_tokens).union(whisper_special_ids)) + + generated_ids: list[int] = [] + logits_shape: list[int] | None = None + current_length = len(prompt_token_ids) + first_token_ms = 0.0 + stop_reason = "max_new_tokens" + decoder_start = time.perf_counter() + + for step_index in range(token_budget): + outputs = decoder.execute() + if not outputs: + raise RuntimeError("seq2seq_transcription decoder graph produced no outputs") + logits = outputs[0].numpy() + logits_shape = list(logits.shape) + if logits.ndim != 3: + raise RuntimeError(f"expected decoder logits with shape [batch, seq, vocab], got {list(logits.shape)}") + token_position = current_length - 1 + if logits.shape[1] == 1: + token_position = 0 + next_token_id = _select_next_token_with_suppression( + np.asarray(logits[0, token_position]), + suppress_tokens=suppress_tokens, + begin_suppress_tokens=begin_suppress_tokens if step_index == 0 else (), + ) + generated_ids.append(next_token_id) + if step_index == 0: + first_token_ms = (time.perf_counter() - decoder_start) * 1000.0 + + if eos_token_id is not None and next_token_id == int(eos_token_id): + stop_reason = "eos_token" + break + if _trim_stop_suffix(generated_ids, encoded_stop_sequences): + stop_reason = "stop_sequence" + break + if current_length >= target_token_count: + stop_reason = "context_limit" + break + if step_index + 1 >= token_budget: + break + + input_array[0, current_length] = next_token_id + current_length += 1 + + decoder_end = time.perf_counter() + transcript = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() + if not transcript: + transcript = _strip_whisper_control_tokens( + _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False) + ).strip() + decode_time_ms = max(0.0, (decoder_end - decoder_start) * 1000.0 - first_token_ms) + decode_tps = ( + ((len(generated_ids) - 1) * 1000.0) / decode_time_ms + if len(generated_ids) > 1 and decode_time_ms > 0.0 + else 0.0 + ) + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "audio_file": str(Path(audio_file).expanduser().resolve()), + "component_order": list(manifest.get("component_order", [])), + "active_feature_frames": active_frames, + "input_shape": list(input_features.shape), + "encoder_hidden_shape": list(encoder_hidden_states.shape), + "output_shape": logits_shape or [], + "input_ids": prompt_token_ids, + "generated_token_ids": generated_ids, + "transcript": transcript, + "response": transcript, + "preprocess_ms": (preprocess_end - preprocess_start) * 1000.0, + "encoder_ms": (encoder_end - encoder_start) * 1000.0, + "decoder_ms": (decoder_end - decoder_start) * 1000.0, + "total_ms": (decoder_end - preprocess_start) * 1000.0, + "time_to_first_token_ms": first_token_ms, + "decode_tps": decode_tps, + "decode_tokens": len(generated_ids), + "stop_reason": stop_reason, + } + + +def _run_encoder_hidden_states_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + audio_file: str | Path, + torch_dtype: torch.dtype, +) -> dict[str, object]: + component_name = "audio_encoder" if "audio_encoder" in component_graphs else "encoder" + if component_name not in component_graphs: + raise ValueError("encoder_hidden_states bundle must include an audio_encoder or encoder graph") + + inputs_meta = manifest.get("inputs") or {} + input_shapes = inputs_meta.get("input_shapes") if isinstance(inputs_meta, dict) else {} + if not isinstance(input_shapes, dict): + input_shapes = {} + expected_shape = input_shapes.get("input_features") + if not (isinstance(expected_shape, list) and len(expected_shape) == 3): + raise ValueError("encoder bundle manifest is missing inputs.input_shapes.input_features") + + preprocess_start = time.perf_counter() + input_features, active_frames = _prepare_generic_audio_encoder_features( + audio_file=audio_file, + manifest=manifest, + expected_shape=expected_shape, + torch_dtype=torch_dtype, + ) + preprocess_end = time.perf_counter() + + _attach_component_io_names(manifest, component_graphs) + encoder = component_graphs[component_name] + encoder_start = time.perf_counter() + encoder.set_inputs([input_features]) + outputs = encoder.execute() + encoder_end = time.perf_counter() + if not outputs: + raise RuntimeError("encoder graph produced no outputs") + hidden = outputs[0].numpy() + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "audio_file": str(Path(audio_file).expanduser().resolve()), + "component_order": list(manifest.get("component_order", [])), + "active_feature_frames": active_frames, + "input_shape": list(input_features.shape), + "encoder_hidden_shape": list(hidden.shape), + "preprocess_ms": (preprocess_end - preprocess_start) * 1000.0, + "encoder_ms": (encoder_end - encoder_start) * 1000.0, + "total_ms": (encoder_end - preprocess_start) * 1000.0, + } + + +def _prepare_generic_audio_encoder_features( + *, + audio_file: str | Path, + manifest: dict[str, object], + expected_shape: list[object], + torch_dtype: torch.dtype, +) -> tuple[np.ndarray, int]: + family = str(manifest.get("family", "") or "") + family_lower = family.strip().lower() + inputs_meta = manifest.get("inputs") if isinstance(manifest.get("inputs"), dict) else {} + sample_rate = int(inputs_meta.get("sample_rate", 16000) if isinstance(inputs_meta, dict) else 16000) + batch = int(expected_shape[0]) + if batch != 1: + raise ValueError("saved audio encoder bundle runtime currently expects batch size 1") + + if "whisper" in family_lower: + expected_mels = int(expected_shape[1]) + expected_frames = int(expected_shape[2]) + try: + features, active_frames = prepare_cactus_audio_features( + audio_file, + model_type="whisper", + expected_frames=expected_frames, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + layout="mels_frames", + ) + return np.ascontiguousarray(features.detach().cpu().numpy()), active_frames + except Exception: + pass + else: + expected_frames = int(expected_shape[1]) + expected_mels = int(expected_shape[2]) + + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + features, feature_length = _generic_log_mel_features( + waveform, + sample_rate=sample_rate, + num_mels=expected_mels, + n_fft=400, + hop_length=160, + frame_length=400, + preemphasis=None, + ) + active_frames = min(feature_length, expected_frames) + features = features[:active_frames, :] + if expected_frames > active_frames: + features = np.pad(features, ((0, expected_frames - active_frames), (0, 0)), mode="constant") + if "whisper" in family_lower: + features = np.ascontiguousarray(features.T) + features = np.ascontiguousarray(features, dtype=np.float16 if torch_dtype == torch.float16 else np.float32) + return np.expand_dims(features, axis=0), active_frames + + +def _resolve_causal_lm_input_ids( + *, + manifest: dict[str, object], + prompt: str | None, + input_ids: str | list[int] | tuple[int, ...] | None, + enable_thinking: bool = False, +) -> tuple[list[int], object | None]: + if input_ids is not None: + return _parse_input_ids(input_ids), None + + if prompt is None: + inputs_meta = manifest.get("inputs") + if isinstance(inputs_meta, dict): + stored_prompt_ids = inputs_meta.get("prompt_input_ids") + parsed_prompt_ids = _parse_nested_manifest_input_ids(stored_prompt_ids) + if parsed_prompt_ids: + return parsed_prompt_ids, None + stored_ids = inputs_meta.get("input_ids") + parsed = _parse_nested_manifest_input_ids(stored_ids) + if parsed: + return parsed, None + stored_prompt = inputs_meta.get("prompt") + if isinstance(stored_prompt, str) and stored_prompt: + prompt = stored_prompt + if not prompt: + raise ValueError("provide --input-ids or --prompt for causal LM component bundles") + + tokenizer = _load_bundle_tokenizer(manifest) + token_ids = _tokenize_bundle_prompt_for_manifest( + manifest, + tokenizer, + prompt, + enable_thinking_if_supported=enable_thinking, + ) + return token_ids, tokenizer + + +def _parse_input_ids(input_ids: str | list[int] | tuple[int, ...]) -> list[int]: + if isinstance(input_ids, str): + parsed = [int(part.strip()) for part in input_ids.split(",") if part.strip()] + else: + parsed = [int(value) for value in input_ids] + if not parsed: + raise ValueError("input_ids was provided but no token ids were parsed") + return parsed + + +def _parse_nested_manifest_input_ids(value: object) -> list[int] | None: + if isinstance(value, list) and value: + if all(isinstance(item, int) for item in value): + return [int(item) for item in value] + first = value[0] + if isinstance(first, list) and all(isinstance(item, int) for item in first): + return [int(item) for item in first] + return None + + +def _patch_missing_lzma_backport() -> str | None: + try: + import importlib.util + import sys + + if importlib.util.find_spec("_lzma") is not None: + return None + if importlib.util.find_spec("backports.lzma") is None: + return None + import backports.lzma as backports_lzma # type: ignore + + sys.modules.setdefault("lzma", backports_lzma) + return "using backports.lzma because this Python build is missing _lzma" + except Exception: + return None + + +def _load_bundle_tokenizer(manifest: dict[str, object]): + _patch_missing_lzma_backport() + try: + from transformers import AutoTokenizer # type: ignore + except Exception as exc: + raise RuntimeError(f"transformers is required to tokenize --prompt: {exc}") from exc + + tokenizer_sources = _pretrained_source_candidates(manifest, processor=False) + if not tokenizer_sources: + raise ValueError("bundle manifest is missing model_source/model_id; provide --input-ids instead") + errors: list[str] = [] + for source in tokenizer_sources: + try: + return AutoTokenizer.from_pretrained( + source, + local_files_only=Path(source).exists(), + trust_remote_code=True, + ) + except Exception as exc: + errors.append(f"{source}: {exc}") + raise RuntimeError( + "failed to load tokenizer assets for prompt tokenization. " + "The CQ weights are present, but tokenizer files are also required for text prompts. " + f"Tried: {'; '.join(errors)}" + ) + + +def _tokenize_bundle_prompt( + tokenizer: object, + prompt: str, + *, + enable_thinking_if_supported: bool = False, +) -> list[int]: + apply_chat_template = getattr(tokenizer, "apply_chat_template", None) + if callable(apply_chat_template): + try: + encoded = apply_chat_template( + [{"role": "user", "content": prompt}], + tokenize=True, + add_generation_prompt=True, + return_dict=True, + enable_thinking=bool(enable_thinking_if_supported), + ) + ids = encoded["input_ids"] if isinstance(encoded, Mapping) else encoded + if ids and isinstance(ids[0], list): + ids = ids[0] + return [int(value) for value in ids] + except Exception: + pass + + encoded = tokenizer(prompt, return_tensors=None) # type: ignore[operator] + ids = encoded["input_ids"] if isinstance(encoded, Mapping) else encoded + if ids and isinstance(ids[0], list): + ids = ids[0] + return [int(value) for value in ids] + + +def _tokenize_bundle_prompt_for_manifest( + manifest: Mapping[str, object], + tokenizer: object, + prompt: str, + *, + enable_thinking_if_supported: bool = False, +) -> list[int]: + family = str(manifest.get("family", "") or "").strip().lower() + if family in {"qwen", "qwen3", "qwen3_5", "qwen3.5"}: + return _encode_prompt_text( + tokenizer, + f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n", + ) + if family in {"lfm2", "lfm2_vl", "lfm"}: + return _encode_prompt_text( + tokenizer, + f"<|startoftext|><|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n", + ) + if family in {"gemma4", "gemma"}: + return _encode_prompt_text( + tokenizer, + _build_gemma4_chat_prompt( + prompt=prompt, + image_token=None, + num_images=0, + audio_token=None, + num_audio_segments=0, + enable_thinking_if_supported=enable_thinking_if_supported, + ), + ) + return _tokenize_bundle_prompt( + tokenizer, + prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + + +def _encode_prompt_text(tokenizer: object, prompt_text: str) -> list[int]: + encode = getattr(tokenizer, "encode", None) + if callable(encode): + try: + return [int(value) for value in encode(prompt_text, add_special_tokens=False)] + except TypeError: + return [int(value) for value in encode(prompt_text)] + encoded = tokenizer(prompt_text, return_tensors=None, add_special_tokens=False) # type: ignore[operator] + ids = encoded["input_ids"] if isinstance(encoded, Mapping) else encoded + if ids and isinstance(ids[0], list): + ids = ids[0] + return [int(value) for value in ids] + + +def _resolve_bundle_padding_token_id(inputs_meta: Mapping[str, object] | None, tokenizer: object | None) -> int: + if isinstance(inputs_meta, Mapping): + value = inputs_meta.get("padding_token_id") + if isinstance(value, int) and value >= 0: + return int(value) + for attr_name in ("pad_token_id", "eos_token_id", "bos_token_id"): + token_id = getattr(tokenizer, attr_name, None) if tokenizer is not None else None + if isinstance(token_id, int) and token_id >= 0: + return int(token_id) + return 0 + + +def _static_input_pad_value(name: str, *, padding_token_id: int) -> int | float: + normalized = name.strip().lower() + if normalized in {"input_ids", "decoder_input_ids"}: + return int(padding_token_id) + if normalized.endswith("position_ids") and "pixel" in normalized: + return -1 + if normalized.endswith("mask") or normalized in {"attention_mask", "token_type_ids"}: + return 0 + return 0.0 + + +def _pad_prepared_store_to_static_input_shapes( + prepared_store: dict[str, np.ndarray], + *, + inputs_meta: Mapping[str, object], + tokenizer: object | None, +) -> None: + input_shapes = inputs_meta.get("input_shapes") if isinstance(inputs_meta, Mapping) else None + if not isinstance(input_shapes, Mapping): + return + + padding_token_id = _resolve_bundle_padding_token_id(inputs_meta, tokenizer) + for name, raw_target_shape in input_shapes.items(): + if not isinstance(name, str): + continue + value = prepared_store.get(name) + if not isinstance(value, np.ndarray): + continue + if not isinstance(raw_target_shape, (list, tuple)): + continue + target_shape = tuple(int(dim) for dim in raw_target_shape) + if tuple(int(dim) for dim in value.shape) == target_shape: + continue + if value.ndim != len(target_shape): + raise ValueError( + f"{name} rank {value.ndim} does not match transpiled bundle input rank {len(target_shape)}; " + "re-transpile with representative inputs for this model." + ) + if any(int(current) > int(target) for current, target in zip(value.shape, target_shape, strict=True)): + if ( + name in {"input_ids", "attention_mask", "token_type_ids", "decoder_input_ids"} + and value.ndim == 2 + and len(target_shape) == 2 + and int(value.shape[0]) == int(target_shape[0]) + and int(value.shape[1]) > int(target_shape[1]) + ): + value = np.ascontiguousarray(value[:, -int(target_shape[1]) :]) + prepared_store[name] = value + elif ( + name in {"input_features", "input_features_mask"} + and value.ndim == len(target_shape) + and len(target_shape) >= 2 + and int(value.shape[0]) == int(target_shape[0]) + and int(value.shape[1]) > int(target_shape[1]) + and all( + int(current) <= int(target) + for current, target in zip(value.shape[2:], target_shape[2:], strict=True) + ) + ): + slices = [slice(None)] * value.ndim + slices[1] = slice(0, int(target_shape[1])) + value = np.ascontiguousarray(value[tuple(slices)]) + prepared_store[name] = value + else: + raise ValueError( + f"{name} shape {list(value.shape)} exceeds transpiled bundle input shape {list(target_shape)}; " + "re-transpile with a longer representative prompt/media sample." + ) + if tuple(int(dim) for dim in value.shape) == target_shape: + continue + if any(int(current) > int(target) for current, target in zip(value.shape, target_shape, strict=True)): + raise ValueError( + f"{name} shape {list(value.shape)} exceeds transpiled bundle input shape {list(target_shape)}; " + "re-transpile with a longer representative prompt/media sample." + ) + + pad_value = _static_input_pad_value(name, padding_token_id=padding_token_id) + padded = np.full(target_shape, pad_value, dtype=value.dtype) + copy_slices = tuple(slice(0, int(dim)) for dim in value.shape) + padded[copy_slices] = value + prepared_store[name] = np.ascontiguousarray(padded) + + +def _encode_stop_sequences(tokenizer: object | None, stop_sequences: tuple[str, ...]) -> list[list[int]]: + if tokenizer is None or not stop_sequences: + return [] + encode = getattr(tokenizer, "encode", None) + if not callable(encode): + return [] + encoded: list[list[int]] = [] + for stop_sequence in stop_sequences: + try: + token_ids = list(encode(stop_sequence, add_special_tokens=False)) + except TypeError: + token_ids = list(encode(stop_sequence)) + if token_ids: + encoded.append([int(token_id) for token_id in token_ids]) + return encoded + + +def _bundle_stop_token_ids( + *, + manifest: Mapping[str, object], + tokenizer: object | None, +) -> set[int]: + token_ids: set[int] = set() + eos_token_id = getattr(tokenizer, "eos_token_id", None) + if isinstance(eos_token_id, int): + token_ids.add(int(eos_token_id)) + + family = str(manifest.get("family", "") or "").strip().lower() + stop_tokens: tuple[str, ...] = () + if family in {"gemma4", "gemma"}: + stop_tokens = ("", "") + elif family in {"qwen", "qwen3", "qwen3_5", "qwen3.5"}: + stop_tokens = ("<|im_end|>",) + + convert = getattr(tokenizer, "convert_tokens_to_ids", None) + encode = getattr(tokenizer, "encode", None) + for token in stop_tokens: + token_id = None + if callable(convert): + try: + token_id = convert(token) + except Exception: + token_id = None + if isinstance(token_id, int) and token_id >= 0: + token_ids.add(int(token_id)) + continue + if callable(encode): + try: + encoded = encode(token, add_special_tokens=False) + except TypeError: + encoded = encode(token) + except Exception: + encoded = [] + if isinstance(encoded, list) and len(encoded) == 1: + token_ids.add(int(encoded[0])) + return token_ids + + +def _has_token_suffix(token_ids: list[int], suffix: list[int]) -> bool: + if not suffix or len(token_ids) < len(suffix): + return False + return token_ids[-len(suffix) :] == suffix + + +def _trim_stop_suffix(token_ids: list[int], stop_sequences: list[list[int]]) -> bool: + for stop_sequence in stop_sequences: + if _has_token_suffix(token_ids, stop_sequence): + del token_ids[-len(stop_sequence) :] + return True + return False + + +def _decode_generated_text(tokenizer: object | None, token_ids: list[int], *, skip_special_tokens: bool) -> str: + if tokenizer is None: + return "" + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + return "" + try: + return str(decode(token_ids, skip_special_tokens=skip_special_tokens)) + except TypeError: + return str(decode(token_ids)) + + +def _resolve_seq2seq_prompt_token_ids( + *, + manifest: dict[str, object], + prompt: str | None, + tokenizer: object | None, +) -> list[int]: + if prompt: + if tokenizer is None: + raise ValueError("transformers tokenizer is required when providing --prompt for seq2seq bundles") + return _tokenize_bundle_prompt(tokenizer, prompt, enable_thinking_if_supported=False) + + inputs_meta = manifest.get("inputs") + if isinstance(inputs_meta, dict): + stored_ids = _parse_nested_manifest_input_ids(inputs_meta.get("decoder_input_ids")) + if stored_ids: + return stored_ids + decoder_start_token_id = inputs_meta.get("decoder_start_token_id") + if isinstance(decoder_start_token_id, int): + return [int(decoder_start_token_id)] + return [] + + +def _select_next_token_with_suppression( + logits: np.ndarray, + *, + suppress_tokens: list[int] | tuple[int, ...], + begin_suppress_tokens: list[int] | tuple[int, ...], +) -> int: + masked = np.asarray(logits, dtype=np.float32).copy() + vocab_size = masked.shape[-1] + for token_id in (*suppress_tokens, *begin_suppress_tokens): + token_index = int(token_id) + if 0 <= token_index < vocab_size: + masked[token_index] = -np.inf + return int(np.argmax(masked)) + + +def _strip_whisper_control_tokens(text: str) -> str: + cleaned = re.sub(r"<\|\d+(?:\.\d+)?\|>", " ", text) + cleaned = re.sub(r"<\|[^|>]+?\|>", " ", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned) + return cleaned.strip() + + +def _attach_component_io_names( + manifest: dict[str, object], + component_graphs: dict[str, LoadedComponentGraph], +) -> None: + for component_entry in manifest.get("components", []): + if not isinstance(component_entry, dict): + continue + name = str(component_entry.get("component", "")).strip() + if not name or name not in component_graphs: + continue + component = component_graphs[name] + logical_inputs = tuple(str(value) for value in component_entry.get("logical_inputs", [])) + logical_outputs = tuple(str(value) for value in component_entry.get("logical_outputs", [])) + if not logical_inputs or not logical_outputs: + raise ValueError( + f"component bundle manifest is missing logical IO names for component={name!r}" + ) + component._input_names = logical_inputs + component._output_names = logical_outputs + + +def _rebind_bound_constants( + *, + graph: Graph, + bundle_root: Path, + bindings: list[dict[str, object]], + weights_dir: str | Path | None, +) -> list[object]: + loaded: list[object] = [] + for binding in bindings: + node_id = int(binding["node_id"]) + if node_id < 0: + continue + raw_path = str(binding["path"]) + tensor_path = _resolve_bound_tensor_path( + raw_path, + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + tensor = graph._tensor_from_node(node_id) + binding_format = str(binding.get("format", "tensor_io") or "tensor_io") + if binding_format != "tensor_io": + raise RuntimeError( + f"unsupported bound constant format {binding_format!r}; re-run cactus convert to rebuild the bundle" + ) + try: + tensor_info = graph._get_output_info(tensor.id) + except Exception: + tensor_info = {} + if int(tensor_info.get("num_elements", 1) or 0) == 0: + # Empty constants carry only shape information. There is no data + # payload to bind, and mmap rebinding rejects zero-byte weights. + continue + if _has_runtime_symbol("cactus_graph_bind_mmap_weights"): + rc = _lib.cactus_graph_bind_mmap_weights( + graph.h, + cactus_node_t(tensor.id), + str(tensor_path).encode(), + ) + if rc != 0: + raise RuntimeError("graph_bind_mmap_weights failed") + loaded.append(tensor_path) + continue + + tensor_file = _open_cactus_tensor_file(tensor_path) + graph.set_external_input(tensor, int(tensor_file.data.ctypes.data), dtype=tensor_file.precision) + if tensor_file.scales is not None and tensor_file.group_size > 0: + rc = _lib.cactus_graph_set_grouped_scales( + graph.h, + cactus_node_t(tensor.id), + int(tensor_file.group_size), + int(tensor_file.num_groups), + tensor_file.scales.ctypes.data_as(ctypes.c_void_p), + ) + if rc != 0: + raise RuntimeError("graph_set_grouped_scales failed") + if tensor_file.is_interleaved: + graph.set_interleaved(tensor, True, tensor_file.original_n) + loaded.append(tensor_file) + return loaded + + +def _resolve_bound_tensor_path( + raw_path: str, + *, + bundle_root: Path, + weights_dir: str | Path | None, +) -> Path: + candidates: list[Path] = [] + explicit = Path(raw_path).expanduser() + if explicit.is_absolute(): + candidates.append(explicit) + else: + candidates.append((bundle_root / explicit).resolve()) + candidates.append(explicit.resolve()) + + if weights_dir is not None: + weights_root = Path(weights_dir).expanduser().resolve() + raw_parts = Path(raw_path).parts + for index in range(len(raw_parts)): + candidates.append(weights_root.joinpath(*raw_parts[index:])) + candidates.append(weights_root / Path(raw_path).name) + + for candidate in candidates: + if candidate.exists(): + return candidate.resolve() + raise FileNotFoundError( + f"could not resolve bound tensor file {raw_path!r} from bundle_root={bundle_root}" + + ("" if weights_dir is None else f" weights_dir={Path(weights_dir).expanduser().resolve()}") + ) + + +def _open_cactus_tensor_file(path: str | Path) -> LoadedTensorFile: + tensor_path = Path(path).expanduser().resolve() + with tensor_path.open("rb") as handle: + header = handle.read(_HEADER_SIZE) + if len(header) < _HEADER_SIZE: + raise RuntimeError(f"tensor file is too small for a Cactus header: {tensor_path}") + + magic = header[:4] + if magic != CACTUS_MAGIC: + raise RuntimeError(f"tensor file is missing the CACT header: {tensor_path}") + + flags = struct.unpack_from(" 0) + + dtype = _PRECISION_TO_DTYPE.get(precision) + if dtype is None: + raise RuntimeError(f"unsupported tensor precision {precision} in {tensor_path}") + + aligned_header = align_offset(header_size, alignment) + scales_offset = aligned_header if scales_bytes > 0 else 0 + data_offset = ( + align_offset(scales_offset + scales_bytes, alignment) + if scales_bytes > 0 + else aligned_header + ) + + data_element_count = byte_size // np.dtype(dtype).itemsize + data = np.memmap(tensor_path, mode="r", dtype=dtype, offset=data_offset, shape=(data_element_count,)) + scales = None + if scales_bytes > 0: + scales = np.memmap( + tensor_path, + mode="r", + dtype=np.float16, + offset=scales_offset, + shape=(scales_bytes // np.dtype(np.float16).itemsize,), + ) + return LoadedTensorFile( + path=tensor_path, + precision=precision, + shape=shape, + data=data, + scales=scales, + group_size=group_size, + num_groups=num_groups, + is_interleaved=bool(flags & FLAG_INTERLEAVED), + original_n=original_n, + ) + + +def _decode_parakeet_tdt_token_ids(vocabulary: tuple[str, ...], token_ids: list[int]) -> str: + pieces: list[str] = [] + for token_id in token_ids: + if token_id < 0 or token_id >= len(vocabulary): + continue + piece = vocabulary[token_id] + if piece.startswith("<|") and piece.endswith("|>"): + continue + pieces.append(piece) + text = "".join(pieces).replace("▁", " ") + return re.sub(r"\s+", " ", text).strip() + + +def _to_numpy(value: Any) -> np.ndarray: + if isinstance(value, np.ndarray): + return np.ascontiguousarray(value) + if isinstance(value, torch.Tensor): + return np.ascontiguousarray(value.detach().cpu().numpy()) + raise TypeError(f"unsupported runtime value type: {type(value).__name__}") diff --git a/python/cactus/transpile/component_partition.py b/python/cactus/transpile/component_partition.py new file mode 100644 index 000000000..090224bb5 --- /dev/null +++ b/python/cactus/transpile/component_partition.py @@ -0,0 +1,333 @@ +from __future__ import annotations + +import copy +from collections import Counter + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.graph_ir import verify_ir + + +COMPONENT_AUDIO_ENCODER = "audio_encoder" +COMPONENT_VISION_ENCODER = "vision_encoder" +COMPONENT_LM_ENCODER = "lm_encoder" +COMPONENT_DECODER = "decoder" +COMPONENT_UNSPECIFIED = "unspecified" + +COMPONENT_ORDER = ( + COMPONENT_AUDIO_ENCODER, + COMPONENT_VISION_ENCODER, + COMPONENT_LM_ENCODER, + COMPONENT_DECODER, +) + + +def _normalized_name_candidates( + *, + value_id: str | None = None, + meta: dict[str, object] | None = None, +) -> tuple[str, ...]: + names: list[str] = [] + + def _add(value: object) -> None: + if not isinstance(value, str): + return + normalized = value.strip().lower() + if normalized and normalized not in names: + names.append(normalized) + + if value_id is not None: + _add(value_id) + if isinstance(meta, dict): + for key in ("logical_name", "input_name", "output_name", "component_role"): + _add(meta.get(key)) + return tuple(names) + + +def _contains_any(name: str, needles: tuple[str, ...]) -> bool: + return any(needle in name for needle in needles) + + +def classify_component_semantic_candidates( + candidates: tuple[str, ...], + *, + task: str, +) -> str | None: + if not candidates: + return None + + for name in candidates: + if _contains_any( + name, + ( + "pixel_values", + "image_features", + ), + ): + return COMPONENT_VISION_ENCODER + + for name in candidates: + if _contains_any( + name, + ( + "input_features", + "audio_features", + "speech_features", + "mel_features", + ), + ): + return COMPONENT_AUDIO_ENCODER + + for name in candidates: + if _contains_any( + name, + ( + "logits", + "next_token", + "ctc_head", + "ctc_logits", + "decoder_output", + ), + ): + return COMPONENT_DECODER + + for name in candidates: + if task == "multimodal_causal_lm_logits" and _contains_any( + name, + ( + "input_ids", + "inputs_embeds", + "text_embeds", + "position_ids", + "placeholder", + "merge", + "masked_scatter", + "token_type", + "attention_mask", + "pixel_position", + ), + ): + return COMPONENT_LM_ENCODER + + return None + + +def classify_value_component(value_id: str, value: IRValue, *, family: str, task: str) -> str | None: + return classify_component_semantic_candidates( + _normalized_name_candidates( + value_id=value_id, + meta=value.meta, + ), + task=task, + ) + + +def classify_node_component(node: IRNode, *, family: str, task: str) -> str | None: + if task == "multimodal_causal_lm_logits" and node.op in {"embedding", "masked_scatter", "advanced_index"}: + return COMPONENT_LM_ENCODER + return classify_component_semantic_candidates( + _normalized_name_candidates(meta=node.meta), + task=task, + ) + + +def _task_default_component(*, family: str, task: str) -> str: + if task == "multimodal_causal_lm_logits": + return COMPONENT_DECODER + if task == "causal_lm_logits": + return COMPONENT_DECODER + if task in {"ctc_logits", "encoder_hidden_states"}: + return COMPONENT_AUDIO_ENCODER + if family == "parakeet_tdt": + return COMPONENT_AUDIO_ENCODER + return COMPONENT_UNSPECIFIED + + +def _rebuild_users(graph: IRGraph) -> None: + for value in graph.values.values(): + value.users = [] + for value_id in graph.inputs: + graph.values[value_id].producer = None + for value_id in graph.constants: + graph.values[value_id].producer = None + for node_id in graph.order: + node = graph.nodes[node_id] + for output_id in node.outputs: + graph.values[output_id].producer = node_id + for input_id in node.inputs: + graph.values[input_id].users.append(node_id) + + +def annotate_ir_components(graph: IRGraph) -> None: + family = str(graph.meta.get("adapter_family", "") or "").lower() + task = str(graph.meta.get("task", "") or "").lower() + graph_component = str(graph.meta.get("component", "") or "").strip() + if graph_component in COMPONENT_ORDER: + for value in graph.values.values(): + value.meta["component"] = graph_component + for node_id in graph.order: + node = graph.nodes[node_id] + node.meta["component"] = graph_component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = graph_component + graph.meta["component_counts"] = {graph_component: len(graph.order)} + return + + default_component = _task_default_component(family=family, task=task) + + for value_id, value in graph.values.items(): + component = classify_value_component(value_id, value, family=family, task=task) + if component is not None: + value.meta["component"] = component + + for node_id in graph.order: + node = graph.nodes[node_id] + component = classify_node_component(node, family=family, task=task) + if component is not None: + node.meta["component"] = component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = component + + for _ in range(8): + changed = False + for node_id in graph.order: + node = graph.nodes[node_id] + component = node.meta.get("component") + if component not in COMPONENT_ORDER: + input_components = { + graph.values[input_id].meta.get("component") + for input_id in node.inputs + if graph.values[input_id].meta.get("component") in COMPONENT_ORDER + } + if len(input_components) == 1: + component = next(iter(input_components)) + node.meta["component"] = component + changed = True + component = node.meta.get("component") + if component in COMPONENT_ORDER: + for output_id in node.outputs: + if graph.values[output_id].meta.get("component") != component: + graph.values[output_id].meta["component"] = component + changed = True + + for node_id in reversed(graph.order): + node = graph.nodes[node_id] + component = node.meta.get("component") + if component in COMPONENT_ORDER: + continue + output_components: set[str] = set() + for output_id in node.outputs: + value = graph.values[output_id] + for user_id in value.users: + user_component = graph.nodes[user_id].meta.get("component") + if user_component in COMPONENT_ORDER: + output_components.add(user_component) + if len(output_components) == 1: + component = next(iter(output_components)) + node.meta["component"] = component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = component + changed = True + + if not changed: + break + + for value in graph.values.values(): + component = value.meta.get("component") + if component not in COMPONENT_ORDER and default_component != COMPONENT_UNSPECIFIED: + value.meta["component"] = default_component + + for node_id in graph.order: + node = graph.nodes[node_id] + component = node.meta.get("component") + if component not in COMPONENT_ORDER and default_component != COMPONENT_UNSPECIFIED: + node.meta["component"] = default_component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = default_component + + counts = Counter( + str(graph.nodes[node_id].meta.get("component", COMPONENT_UNSPECIFIED)) + for node_id in graph.order + ) + graph.meta["component_counts"] = dict(sorted(counts.items())) + + +def summarize_ir_components(graph: IRGraph) -> dict[str, int]: + annotate_ir_components(graph) + counts = graph.meta.get("component_counts", {}) + if isinstance(counts, dict): + return {str(key): int(value) for key, value in counts.items()} + return {} + + +def _clone_value_for_subgraph(value: IRValue, *, as_input: bool) -> IRValue: + cloned = copy.deepcopy(value) + if as_input: + cloned.producer = None + cloned.users = [] + return cloned + + +def extract_component_subgraphs(graph: IRGraph) -> dict[str, IRGraph]: + annotate_ir_components(graph) + component_graphs: dict[str, IRGraph] = {} + + for component in COMPONENT_ORDER: + node_ids = [ + node_id + for node_id in graph.order + if graph.nodes[node_id].meta.get("component") == component + ] + if not node_ids: + continue + + node_id_set = set(node_ids) + inputs: list[str] = [] + outputs: list[str] = [] + constants: dict[str, object] = {} + values: dict[str, IRValue] = {} + nodes: dict[str, IRNode] = {} + + for node_id in node_ids: + node = copy.deepcopy(graph.nodes[node_id]) + nodes[node_id] = node + for input_id in node.inputs: + if input_id in graph.constants: + constants[input_id] = graph.constants[input_id] + if input_id not in values: + values[input_id] = _clone_value_for_subgraph(graph.values[input_id], as_input=True) + continue + producer = graph.values[input_id].producer + if producer not in node_id_set and input_id not in inputs: + inputs.append(input_id) + values[input_id] = _clone_value_for_subgraph(graph.values[input_id], as_input=True) + for output_id in node.outputs: + if output_id not in values: + values[output_id] = _clone_value_for_subgraph(graph.values[output_id], as_input=False) + + for node_id in node_ids: + node = graph.nodes[node_id] + for output_id in node.outputs: + value = graph.values[output_id] + if output_id in graph.outputs or any(user_id not in node_id_set for user_id in value.users): + if output_id not in outputs: + outputs.append(output_id) + + component_graph = IRGraph( + values=values, + nodes=nodes, + order=list(node_ids), + inputs=inputs, + outputs=outputs, + constants=constants, + meta={ + **copy.deepcopy(graph.meta), + "component": component, + }, + ) + _rebuild_users(component_graph) + verify_ir(component_graph) + component_graphs[component] = component_graph + + return component_graphs diff --git a/python/cactus/transpile/component_pipeline.py b/python/cactus/transpile/component_pipeline.py new file mode 100644 index 000000000..60394ebc6 --- /dev/null +++ b/python/cactus/transpile/component_pipeline.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import copy +from dataclasses import dataclass +from dataclasses import field +import re +from typing import Any + +import numpy as np +import torch + +from cactus.transpile.capture_pytorch import capture_model +from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.lower import TranspiledGraph +from cactus.transpile.lower import transpile_preoptimized_ir +from cactus.transpile.optimize_graph import FusionConfig +from cactus.transpile.optimize_graph import optimize_graph + + +@dataclass +class ComponentModuleSpec: + component: str + module: torch.nn.Module + example_inputs: tuple[torch.Tensor, ...] + input_keys: tuple[str, ...] + output_keys: tuple[str, ...] + graph_meta: dict[str, object] = field(default_factory=dict) + metadata: dict[str, object] = field(default_factory=dict) + + +@dataclass +class CapturedComponent: + component: str + input_keys: tuple[str, ...] + output_keys: tuple[str, ...] + raw_ir_graph: IRGraph + optimized_ir_graph: IRGraph + transpiled_graph: TranspiledGraph + metadata: dict[str, object] = field(default_factory=dict) + + +_GRAPH_ARG_INPUT_RE = re.compile(r"^v_args_(\d+)$") + + +def _align_input_keys_to_graph_inputs( + declared_input_keys: tuple[str, ...], + graph_inputs: tuple[str, ...] | list[str], +) -> tuple[str, ...]: + aligned: list[str] = [] + for value_id in graph_inputs: + match = _GRAPH_ARG_INPUT_RE.match(str(value_id)) + if match is None: + return declared_input_keys + input_index = int(match.group(1)) + if input_index < 0 or input_index >= len(declared_input_keys): + return declared_input_keys + aligned.append(declared_input_keys[input_index]) + return tuple(aligned) + + +def _call_optional_capture_hook(spec: ComponentModuleSpec, attr_name: str) -> None: + hook = getattr(spec.module, attr_name, None) + if not callable(hook): + return + try: + hook( + component=spec.component, + example_inputs=spec.example_inputs, + input_keys=spec.input_keys, + output_keys=spec.output_keys, + ) + except TypeError: + hook() + + +def capture_component_spec( + spec: ComponentModuleSpec, + *, + fusion_config: FusionConfig | None = None, +) -> CapturedComponent: + wrapped = _ComponentCaptureWrapper(spec.module, graph_meta=spec.graph_meta).eval() + capture_error: Exception | None = None + _call_optional_capture_hook(spec, "prepare_for_capture") + try: + captured = capture_model(wrapped, spec.example_inputs) + except Exception as exc: + capture_error = exc + raise + finally: + try: + _call_optional_capture_hook(spec, "restore_after_capture") + except Exception: + if capture_error is None: + raise + raw_ir_graph = copy.deepcopy(captured.ir_graph) + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + canonicalize_exported_graph(optimized_ir_graph) + optimize_graph(optimized_ir_graph, config=fusion_config) + transpiled_graph = transpile_preoptimized_ir(copy.deepcopy(optimized_ir_graph)) + if len(spec.output_keys) != len(transpiled_graph.outputs): + raise ValueError( + f"component {spec.component} declared {len(spec.output_keys)} output keys but lowered " + f"{len(transpiled_graph.outputs)} graph outputs" + ) + aligned_input_keys = _align_input_keys_to_graph_inputs(spec.input_keys, optimized_ir_graph.inputs) + return CapturedComponent( + component=spec.component, + input_keys=aligned_input_keys, + output_keys=spec.output_keys, + raw_ir_graph=raw_ir_graph, + optimized_ir_graph=optimized_ir_graph, + transpiled_graph=transpiled_graph, + metadata=dict(spec.metadata), + ) + + +def execute_component_pipeline( + components: list[CapturedComponent], + *, + initial_store: dict[str, Any], +) -> tuple[dict[str, np.ndarray], dict[str, list[np.ndarray]]]: + store: dict[str, np.ndarray] = {} + for key, value in initial_store.items(): + store[key] = _to_numpy(value) + + outputs_by_component: dict[str, list[np.ndarray]] = {} + for component in components: + runtime_inputs = [] + for input_key in component.input_keys: + if input_key not in store: + raise KeyError( + f"component {component.component} is missing pipeline input {input_key!r}" + ) + runtime_inputs.append(store[input_key]) + component.transpiled_graph.set_inputs(runtime_inputs) + raw_outputs = component.transpiled_graph.execute() + numpy_outputs = [output.numpy().copy() for output in raw_outputs] + if len(numpy_outputs) != len(component.output_keys): + raise ValueError( + f"component {component.component} produced {len(numpy_outputs)} outputs, " + f"expected {len(component.output_keys)}" + ) + for output_key, value in zip(component.output_keys, numpy_outputs, strict=True): + store[output_key] = value + outputs_by_component[component.component] = numpy_outputs + + return store, outputs_by_component + + +def initial_store_from_named_tensors( + keys: tuple[str, ...] | list[str], + tensors: tuple[torch.Tensor, ...] | list[torch.Tensor], +) -> dict[str, np.ndarray]: + if len(keys) != len(tensors): + raise ValueError(f"expected {len(keys)} tensors, got {len(tensors)}") + return { + str(key): _to_numpy(tensor) + for key, tensor in zip(keys, tensors, strict=True) + } + + +def _to_numpy(value: Any) -> np.ndarray: + if isinstance(value, np.ndarray): + return np.ascontiguousarray(value) + if isinstance(value, torch.Tensor): + return np.ascontiguousarray(value.detach().cpu().numpy()) + raise TypeError(f"unsupported pipeline value type: {type(value).__name__}") + + +class _ComponentCaptureWrapper(torch.nn.Module): + def __init__(self, module: torch.nn.Module, *, graph_meta: dict[str, object]): + super().__init__() + self.module = module + self.graph_meta = dict(graph_meta) + + def forward(self, *args: torch.Tensor) -> Any: + return self.module(*args) + + def get_transpile_metadata(self) -> dict[str, object]: + metadata: dict[str, object] = {} + provider = getattr(self.module, "get_transpile_metadata", None) + if callable(provider): + provided = provider() + if isinstance(provided, dict): + metadata.update(provided) + graph_meta = {} + existing = metadata.get("graph", {}) + if isinstance(existing, dict): + graph_meta.update(existing) + graph_meta.update(self.graph_meta) + metadata["graph"] = graph_meta + return metadata diff --git a/python/cactus/transpile/component_plan.py b/python/cactus/transpile/component_plan.py new file mode 100644 index 000000000..9702c48ca --- /dev/null +++ b/python/cactus/transpile/component_plan.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Mapping + + +@dataclass(frozen=True) +class ComponentPlan: + task: str + components: tuple[str, ...] = () + needs_image: bool = False + needs_audio: bool = False + force_component_pipeline: bool = False + + +def _load_json(path: Path) -> dict[str, object]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + return value if isinstance(value, dict) else {} + + +def _load_config_txt(path: Path) -> dict[str, str]: + config: dict[str, str] = {} + try: + lines = path.read_text(encoding="utf-8").splitlines() + except Exception: + return config + for line in lines: + if "=" not in line: + continue + key, value = line.split("=", 1) + config[key.strip()] = value.strip() + return config + + +def _architecture_names(config: Mapping[str, object], config_txt: Mapping[str, str] | None = None) -> tuple[str, ...]: + raw_architectures = config.get("architectures", []) + names = tuple(str(value) for value in raw_architectures if isinstance(value, str)) + if names or config_txt is None: + return names + raw_txt = config_txt.get("architectures") + if not raw_txt: + return () + return tuple(value.strip() for value in raw_txt.split(",") if value.strip()) + + +def _has_dict_config(config: Mapping[str, object], *keys: str) -> bool: + return any(isinstance(config.get(key), dict) for key in keys) + + +def _has_positive_txt_int(config_txt: Mapping[str, str] | None, *keys: str) -> bool: + if config_txt is None: + return False + for key in keys: + try: + if int(config_txt.get(key, "0") or 0) > 0: + return True + except ValueError: + continue + return False + + +def _model_type(config: Mapping[str, object], config_txt: Mapping[str, str] | None = None) -> str: + value = config.get("model_type") + if value is None and config_txt is not None: + value = config_txt.get("model_type") + return str(value or "").strip().lower() + + +def _lowered_architectures(config: Mapping[str, object], config_txt: Mapping[str, str] | None = None) -> tuple[str, ...]: + return tuple(value.lower() for value in _architecture_names(config, config_txt)) + + +def _is_tdt_config(config: Mapping[str, object], model_type: str, lowered_id: str) -> bool: + decoding_cfg = config.get("decoding") + if isinstance(decoding_cfg, Mapping) and str(decoding_cfg.get("model_type", "") or "").lower() == "tdt": + return True + loss_cfg = config.get("loss") + if isinstance(loss_cfg, Mapping) and str(loss_cfg.get("loss_name", "") or "").lower() == "tdt": + return True + return model_type == "parakeet_tdt" or "parakeet-tdt" in lowered_id + + +def _looks_like_vision_language_model( + *, + model_type: str, + architectures: tuple[str, ...], + lowered_id: str, + has_vision: bool, +) -> bool: + if not has_vision: + return False + if any(token in model_type for token in ("vl", "vision", "image", "multimodal")): + return True + if any( + token in architecture + for architecture in architectures + for token in ("vision", "image", "vl", "imagetexttotext", "conditionalgeneration") + ): + return True + return any(token in lowered_id for token in ("-vl", "_vl", "vision", "image")) + + +def infer_component_plan_from_config( + config: Mapping[str, object], + *, + model_id: str = "", + config_txt: Mapping[str, str] | None = None, +) -> ComponentPlan | None: + model_type = _model_type(config, config_txt) + lowered_id = model_id.lower() + architectures = _lowered_architectures(config, config_txt) + + if _is_tdt_config(config, model_type, lowered_id): + return ComponentPlan( + task="tdt_transcription", + components=("audio_encoder", "decoder"), + needs_audio=True, + force_component_pipeline=True, + ) + + if model_type == "whisper" or "whisper" in lowered_id or any("whisper" in value for value in architectures): + return ComponentPlan( + task="seq2seq_transcription", + components=("audio_encoder", "decoder"), + needs_audio=True, + force_component_pipeline=True, + ) + + if any("ctc" in value for value in architectures) or "ctc" in lowered_id: + return ComponentPlan( + task="ctc_logits", + components=("audio_encoder",), + needs_audio=True, + ) + + has_vision = ( + _has_dict_config(config, "vision_config", "visual_config", "image_config") + or _has_positive_txt_int(config_txt, "vision_num_layers") + ) + has_audio = ( + _has_dict_config(config, "audio_config", "speech_config", "acoustic_config") + or ( + _has_dict_config(config, "encoder_config") + and ( + "audio" in model_type + or "speech" in model_type + or "audio_token_index" in config + or any(token in lowered_id for token in ("speech", "audio")) + or any(token in value for value in architectures for token in ("speech", "audio")) + ) + ) + or _has_positive_txt_int(config_txt, "audio_num_layers") + ) + if has_vision or has_audio: + if has_audio or _looks_like_vision_language_model( + model_type=model_type, + architectures=architectures, + lowered_id=lowered_id, + has_vision=has_vision, + ): + components: list[str] = [] + if has_vision: + components.append("vision_encoder") + if has_audio: + components.append("audio_encoder") + components.extend(("lm_encoder", "decoder")) + return ComponentPlan( + task="multimodal_causal_lm_logits", + components=tuple(components), + needs_image=has_vision, + needs_audio=has_audio, + force_component_pipeline=True, + ) + + if any("causallm" in value for value in architectures): + return ComponentPlan(task="causal_lm_logits", components=("decoder",)) + if any(token in lowered_id or token in model_type for token in ("qwen", "gemma", "llama", "mistral", "lfm")): + return ComponentPlan(task="causal_lm_logits", components=("decoder",)) + return None + + +def infer_component_plan_from_output(output_dir: str | Path, *, model_id: str = "") -> ComponentPlan | None: + root = Path(output_dir) + hf_config = _load_json(root / "hf_config.json") + config_txt = _load_config_txt(root / "config.txt") + return infer_component_plan_from_config(hf_config, model_id=model_id, config_txt=config_txt) diff --git a/python/cactus/transpile/fusion/__init__.py b/python/cactus/transpile/fusion/__init__.py new file mode 100644 index 000000000..463478d1c --- /dev/null +++ b/python/cactus/transpile/fusion/__init__.py @@ -0,0 +1,47 @@ +from cactus.transpile.fusion.conv import ConvModuleMatch +from cactus.transpile.fusion.conv import match_conv_module +from cactus.transpile.fusion.attention import AttentionBlockMatch +from cactus.transpile.fusion.attention import AttentionMatch +from cactus.transpile.fusion.attention import SelfAttentionBlockMatch +from cactus.transpile.fusion.attention import match_attention +from cactus.transpile.fusion.attention import match_attention_block +from cactus.transpile.fusion.attention import match_self_attention_block +from cactus.transpile.fusion.deltanet import GatedDeltaNetMatch +from cactus.transpile.fusion.deltanet import match_gated_deltanet +from cactus.transpile.fusion.linear import LinearMatch +from cactus.transpile.fusion.linear import match_linear +from cactus.transpile.fusion.lstm import LSTMCellMatch +from cactus.transpile.fusion.lstm import match_lstm_cell +from cactus.transpile.fusion.mlp import GatedMLPMatch +from cactus.transpile.fusion.mlp import match_gated_mlp +from cactus.transpile.fusion.rel_pos_bias import RelPosBiasMatch +from cactus.transpile.fusion.rel_pos_bias import match_rel_pos_bias +from cactus.transpile.fusion.rms_norm import RMSNormMatch +from cactus.transpile.fusion.rms_norm import match_rms_norm +from cactus.transpile.fusion.rope import RoPEMatch +from cactus.transpile.fusion.rope import match_rope + +__all__ = [ + "ConvModuleMatch", + "AttentionBlockMatch", + "AttentionMatch", + "SelfAttentionBlockMatch", + "GatedDeltaNetMatch", + "GatedMLPMatch", + "LinearMatch", + "LSTMCellMatch", + "RelPosBiasMatch", + "RMSNormMatch", + "RoPEMatch", + "match_attention", + "match_attention_block", + "match_self_attention_block", + "match_conv_module", + "match_gated_deltanet", + "match_gated_mlp", + "match_linear", + "match_lstm_cell", + "match_rel_pos_bias", + "match_rms_norm", + "match_rope", +] diff --git a/python/cactus/transpile/fusion/attention.py b/python/cactus/transpile/fusion/attention.py new file mode 100644 index 000000000..d6a8e14e0 --- /dev/null +++ b/python/cactus/transpile/fusion/attention.py @@ -0,0 +1,550 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.fusion.common import collect_node_ids +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_passthrough +from cactus.transpile.fusion.linear import match_linear +from cactus.transpile.fusion.rms_norm import match_rms_norm +from cactus.transpile.fusion.rope import match_rope + + +@dataclass(frozen=True) +class AttentionMatch: + query_value_id: str + key_value_id: str + value_value_id: str + source_input_value_ids: tuple[str, str, str] + weight_value_ids: tuple[str | None, str | None, str | None] + has_rope: bool + has_qk_norm: bool + has_gqa_repeat: bool + is_causal: bool + scale: float + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class AttentionBlockMatch: + attention_node_id: str + query_value_id: str + key_value_id: str + value_value_id: str + mask_value_id: str | None + gate_value_id: str | None + output_projection_weight_value_id: str + output_projection_bias_value_id: str | None + attention_output_shape: tuple[int, ...] + is_causal: bool + additive_mask: bool + scale: float + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class SelfAttentionBlockMatch: + attention_node_id: str + hidden_value_id: str + query_weight_value_id: str + query_projection_bias_value_id: str | None + query_add_value_id: str | None + rel_query_add_value_id: str | None + key_weight_value_id: str + key_projection_bias_value_id: str | None + value_weight_value_id: str + value_projection_bias_value_id: str | None + mask_value_id: str | None + relative_key_input_value_id: str | None + relative_key_weight_value_id: str | None + relative_key_projection_bias_value_id: str | None + gate_value_id: str | None + output_projection_weight_value_id: str + output_projection_bias_value_id: str | None + query_shape: tuple[int, ...] + key_shape: tuple[int, ...] + value_shape: tuple[int, ...] + relative_key_shape: tuple[int, ...] | None + attention_output_shape: tuple[int, ...] + is_causal: bool + additive_mask: bool + scale: float + rel_pos_scale: float | None + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class _ProjectedAttentionInputMatch: + hidden_value_id: str + weight_value_id: str + projection_bias_value_id: str | None + projected_shape: tuple[int, ...] + add_value_id: str | None + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class _RelativeKeyProjectionMatch: + input_value_id: str + weight_value_id: str + projection_bias_value_id: str | None + projected_shape: tuple[int, ...] + node_ids: tuple[str, ...] + + +def match_attention(graph: IRGraph, node: IRNode) -> AttentionMatch | None: + if node.op not in {"attention", "scaled_dot_product_attention"} or len(node.inputs) < 3: + return None + + q_info = _extract_attention_input(graph, node.inputs[0], role="q") + k_info = _extract_attention_input(graph, node.inputs[1], role="k") + v_info = _extract_attention_input(graph, node.inputs[2], role="v") + if q_info is None or k_info is None or v_info is None: + return None + + has_rope = bool(q_info["has_rope"] and k_info["has_rope"]) + has_qk_norm = bool(q_info["has_rms_norm"] and k_info["has_rms_norm"]) + has_gqa = bool(k_info["has_gqa_repeat"] or v_info["has_gqa_repeat"] or node.attrs.get("enable_gqa", False)) + node_ids = {node.id, *q_info["node_ids"], *k_info["node_ids"], *v_info["node_ids"]} + + return AttentionMatch( + query_value_id=node.inputs[0], + key_value_id=node.inputs[1], + value_value_id=node.inputs[2], + source_input_value_ids=(q_info["source_input"], k_info["source_input"], v_info["source_input"]), + weight_value_ids=(q_info.get("weight_value_id"), k_info.get("weight_value_id"), v_info.get("weight_value_id")), + has_rope=has_rope, + has_qk_norm=has_qk_norm, + has_gqa_repeat=has_gqa, + is_causal=bool(node.attrs.get("is_causal", False)), + scale=float(node.attrs.get("scale", 0.0)), + window_size=int(node.attrs.get("window_size", 0)), + node_ids=tuple(sorted(node_ids)), + ) + + +def match_self_attention_block(graph: IRGraph, node: IRNode) -> SelfAttentionBlockMatch | None: + projection = match_linear(graph, node) + if projection is None: + return None + + output_path = _extract_attention_output_path(graph, projection.input_value_id) + if output_path is None: + return None + attention_node = graph.nodes.get(output_path["attention_node_id"]) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: + return None + + query_match = _extract_projected_attention_input(graph, attention_node.inputs[0], role="q") + key_match = _extract_projected_attention_input(graph, attention_node.inputs[1], role="k") + value_match = _extract_projected_attention_input(graph, attention_node.inputs[2], role="v") + if query_match is None or key_match is None or value_match is None: + return None + + if not ( + query_match.hidden_value_id == key_match.hidden_value_id == value_match.hidden_value_id + ): + return None + + mask_value_id = attention_node.inputs[3] if len(attention_node.inputs) > 3 else None + rel_query_add_value_id: str | None = None + relative_key_input_value_id: str | None = None + relative_key_weight_value_id: str | None = None + relative_key_projection_bias_value_id: str | None = None + relative_key_shape: tuple[int, ...] | None = None + rel_pos_scale: float | None = None + rel_pos_node_ids: tuple[str, ...] = () + + if mask_value_id is not None: + rel_pos_node = producer(graph, strip_passthrough(graph, mask_value_id)) + if rel_pos_node is not None and rel_pos_node.op == "rel_pos_bias" and len(rel_pos_node.inputs) == 2: + rel_query_match = _extract_projected_attention_input(graph, rel_pos_node.inputs[0], role="q") + relative_key_match = _extract_relative_key_projection(graph, rel_pos_node.inputs[1]) + if rel_query_match is None or relative_key_match is None: + return None + if not ( + rel_query_match.hidden_value_id == query_match.hidden_value_id + and rel_query_match.weight_value_id == query_match.weight_value_id + and rel_query_match.projection_bias_value_id == query_match.projection_bias_value_id + and rel_query_match.projected_shape == query_match.projected_shape + ): + return None + rel_query_add_value_id = rel_query_match.add_value_id + relative_key_input_value_id = relative_key_match.input_value_id + relative_key_weight_value_id = relative_key_match.weight_value_id + relative_key_projection_bias_value_id = relative_key_match.projection_bias_value_id + relative_key_shape = relative_key_match.projected_shape + rel_pos_scale = float(rel_pos_node.attrs.get("scale", 1.0)) + rel_pos_node_ids = collect_node_ids( + rel_pos_node.id, + rel_pos_node.meta.get("rel_pos_bias_nodes", ()), + rel_query_match.node_ids, + relative_key_match.node_ids, + ) + mask_value_id = None + + return SelfAttentionBlockMatch( + attention_node_id=attention_node.id, + hidden_value_id=query_match.hidden_value_id, + query_weight_value_id=query_match.weight_value_id, + query_projection_bias_value_id=query_match.projection_bias_value_id, + query_add_value_id=query_match.add_value_id, + rel_query_add_value_id=rel_query_add_value_id, + key_weight_value_id=key_match.weight_value_id, + key_projection_bias_value_id=key_match.projection_bias_value_id, + value_weight_value_id=value_match.weight_value_id, + value_projection_bias_value_id=value_match.projection_bias_value_id, + mask_value_id=mask_value_id, + relative_key_input_value_id=relative_key_input_value_id, + relative_key_weight_value_id=relative_key_weight_value_id, + relative_key_projection_bias_value_id=relative_key_projection_bias_value_id, + gate_value_id=output_path["gate_value_id"], + output_projection_weight_value_id=projection.weight_value_id, + output_projection_bias_value_id=projection.bias_value_id, + query_shape=query_match.projected_shape, + key_shape=key_match.projected_shape, + value_shape=value_match.projected_shape, + relative_key_shape=relative_key_shape, + attention_output_shape=output_path["attention_output_shape"], + is_causal=bool(attention_node.attrs.get("is_causal", True)), + additive_mask=bool(attention_node.attrs.get("additive_mask", False)), + scale=float(attention_node.attrs.get("scale", 0.0)), + rel_pos_scale=rel_pos_scale, + window_size=int(attention_node.attrs.get("window_size", 0)), + node_ids=collect_node_ids( + attention_node.id, + projection.node_ids, + output_path["node_ids"], + query_match.node_ids, + key_match.node_ids, + value_match.node_ids, + rel_pos_node_ids, + ), + ) + + +def match_attention_block(graph: IRGraph, node: IRNode) -> AttentionBlockMatch | None: + projection = match_linear(graph, node) + if projection is None: + return None + + output_path = _extract_attention_output_path(graph, projection.input_value_id) + if output_path is None: + return None + attention_node = graph.nodes.get(output_path["attention_node_id"]) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: + return None + + return AttentionBlockMatch( + attention_node_id=attention_node.id, + query_value_id=attention_node.inputs[0], + key_value_id=attention_node.inputs[1], + value_value_id=attention_node.inputs[2], + mask_value_id=attention_node.inputs[3] if len(attention_node.inputs) > 3 else None, + gate_value_id=output_path["gate_value_id"], + output_projection_weight_value_id=projection.weight_value_id, + output_projection_bias_value_id=projection.bias_value_id, + attention_output_shape=output_path["attention_output_shape"], + is_causal=bool(attention_node.attrs.get("is_causal", True)), + additive_mask=bool(attention_node.attrs.get("additive_mask", False)), + scale=float(attention_node.attrs.get("scale", 0.0)), + window_size=int(attention_node.attrs.get("window_size", 0)), + node_ids=tuple(sorted({attention_node.id, *projection.node_ids, *output_path["node_ids"]})), + ) + + +def _extract_attention_output_path(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + gate_value_id: str | None = None + + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + break + + node = producer(graph, current) + if node is not None and node.op == "multiply" and len(node.inputs) == 2: + for attn_candidate, gate_candidate in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + attn_match = _extract_attention_output_path(graph, attn_candidate) + if attn_match is None: + continue + attn_match["gate_value_id"] = gate_candidate + attn_match["node_ids"] = tuple(sorted({node.id, *attn_match["node_ids"]})) + return attn_match + + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + attention_output_shape = tuple(int(v) for v in reshape_node.attrs.get("shape", ())) + current = reshape_node.inputs[0] + transpose_node = producer(graph, current) + if transpose_node is None: + return None + + if transpose_node.op == "transpose": + if int(transpose_node.attrs.get("dim0", -1)) != 1 or int(transpose_node.attrs.get("dim1", -1)) != 2: + return None + current = transpose_node.inputs[0] + elif transpose_node.op == "permute": + permutation = tuple(int(v) for v in transpose_node.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + return None + current = transpose_node.inputs[0] + else: + return None + + attention_node = producer(graph, current) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"}: + return None + + return { + "attention_node_id": attention_node.id, + "gate_value_id": gate_value_id, + "attention_output_shape": attention_output_shape, + "node_ids": tuple(sorted({reshape_node.id, transpose_node.id})), + } + + +def _extract_projected_attention_input(graph: IRGraph, value_id: str, *, role: str) -> _ProjectedAttentionInputMatch | None: + current = strip_passthrough(graph, value_id) + add_node = producer(graph, current) + if add_node is not None and add_node.op == "add" and len(add_node.inputs) == 2: + for projected_candidate, add_candidate in ((add_node.inputs[0], add_node.inputs[1]), (add_node.inputs[1], add_node.inputs[0])): + projected = _extract_transposed_projection(graph, projected_candidate) + if projected is None: + continue + if not _looks_like_attention_addend(graph, add_candidate, projected.projected_shape): + continue + return _ProjectedAttentionInputMatch( + hidden_value_id=projected.hidden_value_id, + weight_value_id=projected.weight_value_id, + projection_bias_value_id=projected.projection_bias_value_id, + projected_shape=projected.projected_shape, + add_value_id=strip_passthrough(graph, add_candidate), + node_ids=collect_node_ids(add_node.id, projected.node_ids), + ) + + if role == "q": + return None + + return _extract_transposed_projection(graph, current) + + +def _extract_transposed_projection(graph: IRGraph, value_id: str) -> _ProjectedAttentionInputMatch | None: + current = strip_passthrough(graph, value_id) + layout_node = producer(graph, current) + if layout_node is None or len(layout_node.inputs) != 1: + return None + + if layout_node.op == "permute": + permutation = tuple(int(v) for v in layout_node.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + return None + elif layout_node.op == "transpose": + if int(layout_node.attrs.get("dim0", -1)) != 1 or int(layout_node.attrs.get("dim1", -1)) != 2: + return None + else: + return None + + projected_value_id = strip_passthrough(graph, layout_node.inputs[0]) + projected_value = graph.values.get(projected_value_id) + if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: + return None + + reshape_node = producer(graph, projected_value_id) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + linear_node = producer(graph, reshape_node.inputs[0]) + if linear_node is None: + return None + linear = match_linear(graph, linear_node) + if linear is None: + return None + + return _ProjectedAttentionInputMatch( + hidden_value_id=linear.input_value_id, + weight_value_id=linear.weight_value_id, + projection_bias_value_id=linear.bias_value_id, + projected_shape=tuple(int(v) for v in projected_value.shape), + add_value_id=None, + node_ids=collect_node_ids(layout_node.id, reshape_node.id, linear.node_ids), + ) + + +def _extract_relative_key_projection(graph: IRGraph, value_id: str) -> _RelativeKeyProjectionMatch | None: + current = strip_passthrough(graph, value_id) + projected_value = graph.values.get(current) + if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: + return None + + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + linear_node = producer(graph, reshape_node.inputs[0]) + if linear_node is None: + return None + linear = match_linear(graph, linear_node) + if linear is None: + return None + + return _RelativeKeyProjectionMatch( + input_value_id=linear.input_value_id, + weight_value_id=linear.weight_value_id, + projection_bias_value_id=linear.bias_value_id, + projected_shape=tuple(int(v) for v in projected_value.shape), + node_ids=collect_node_ids(reshape_node.id, linear.node_ids), + ) + + +def _looks_like_attention_addend(graph: IRGraph, value_id: str, target_shape: tuple[int, ...]) -> bool: + value = graph.values.get(strip_passthrough(graph, value_id)) + if value is None or value.shape is None: + return False + + shape = tuple(int(v) for v in value.shape) + if len(shape) != 4 or len(target_shape) != 4: + return False + + batch, seq_len, heads, head_dim = target_shape + if shape[0] not in (1, batch) or shape[3] != head_dim: + return False + + # Allow either BHSD-style tensors like [B, H, 1, D] or BSHD-style tensors like [B, 1, H, D]. + if shape[1] == heads and shape[2] in (1, seq_len): + return True + if shape[1] in (1, seq_len) and shape[2] == heads: + return True + return False + + +def _extract_attention_input(graph: IRGraph, value_id: str, *, role: str) -> dict[str, object] | None: + node_ids: set[str] = set() + current = strip_passthrough(graph, value_id) + has_gqa_repeat = False + + while True: + node = producer(graph, current) + if node is None: + return { + "source_input": current, + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + node_ids.add(node.id) + + if node.op in {"reshape", "view", "transpose", "permute"}: + current = node.inputs[0] + continue + + rope = match_rope(graph, current) + if rope is not None: + rope_info = _extract_attention_input(graph, rope.input_value_id, role=role) + if rope_info is None: + return None + rope_info["has_rope"] = True + rope_info["node_ids"] = tuple(sorted(set(rope_info["node_ids"]) | node_ids | set(rope.node_ids))) + rope_info["has_gqa_repeat"] = bool(rope_info["has_gqa_repeat"] or has_gqa_repeat) + return rope_info + + rms = match_rms_norm(graph, node) + if rms is not None: + if role == "v": + return None + rms_info = _extract_attention_input(graph, rms.input_value_id, role=role) + if rms_info is None: + return None + rms_info["has_rms_norm"] = True + rms_info["node_ids"] = tuple(sorted(set(rms_info["node_ids"]) | node_ids | set(rms.node_ids))) + rms_info["has_gqa_repeat"] = bool(rms_info["has_gqa_repeat"] or has_gqa_repeat) + return rms_info + + if _looks_like_gqa_repeat(graph, current): + has_gqa_repeat = True + current = _unwrap_gqa_repeat(graph, current) + if current is None: + return None + continue + + linear_node = producer(graph, current) + if linear_node is None: + return { + "source_input": strip_passthrough(graph, current), + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + linear = match_linear(graph, linear_node) + if linear is None: + return { + "source_input": strip_passthrough(graph, current), + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + return { + "source_input": strip_passthrough(graph, linear.input_value_id), + "weight_value_id": linear.weight_value_id, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(set(node_ids) | set(linear.node_ids))), + } + + +def _looks_like_gqa_repeat(graph: IRGraph, value_id: str) -> bool: + return _unwrap_gqa_repeat(graph, value_id) is not None + + +def _unwrap_gqa_repeat(graph: IRGraph, value_id: str) -> str | None: + current = strip_passthrough(graph, value_id) + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"}: + return None + current = reshape_node.inputs[0] + expand_node = producer(graph, current) + if expand_node is None or expand_node.op != "expand": + return None + current = expand_node.inputs[0] + while True: + node = producer(graph, current) + if node is None: + return None + if node.op == "slice": + current = node.inputs[0] + continue + if node.op != "unsqueeze": + return None + if int(node.attrs.get("dim", -999)) != 2: + return None + break + base_value_id = strip_passthrough(graph, node.inputs[0]) + base_shape = graph.values.get(base_value_id).shape if graph.values.get(base_value_id) is not None else None + target_shape = tuple(int(v) for v in expand_node.attrs.get("shape", ())) + if base_shape is None or len(base_shape) != 4 or len(target_shape) != 5: + return None + expected = (int(base_shape[0]), int(base_shape[1]), int(target_shape[2]), int(base_shape[2]), int(base_shape[3])) + if expected != target_shape: + return None + return base_value_id diff --git a/python/cactus/transpile/fusion/common.py b/python/cactus/transpile/fusion/common.py new file mode 100644 index 000000000..d463cb804 --- /dev/null +++ b/python/cactus/transpile/fusion/common.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class FusionMatch: + kind: str + anchor_node_id: str + node_ids: tuple[str, ...] + value_ids: tuple[str, ...] + attrs: dict[str, object] + + +def producer(graph: IRGraph, value_id: str) -> IRNode | None: + value = graph.values.get(value_id) + if value is None or value.producer is None: + return None + return graph.nodes.get(value.producer) + + +def strip_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def collect_node_ids(*parts: Any) -> tuple[str, ...]: + node_ids: set[str] = set() + for part in parts: + if part is None: + continue + if isinstance(part, str): + node_ids.add(part) + continue + if isinstance(part, IRNode): + node_ids.add(part.id) + continue + if isinstance(part, (list, tuple, set)): + for item in part: + if isinstance(item, str): + node_ids.add(item) + elif isinstance(item, IRNode): + node_ids.add(item.id) + return tuple(sorted(node_ids)) diff --git a/python/cactus/transpile/fusion/conv.py b/python/cactus/transpile/fusion/conv.py new file mode 100644 index 000000000..265501f9a --- /dev/null +++ b/python/cactus/transpile/fusion/conv.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from cactus.transpile.fusion.common import collect_node_ids +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_passthrough +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class ConvModuleMatch: + input_value_id: str + pointwise1_weight_value_id: str + pointwise1_bias_value_id: str | None + depthwise_weight_value_id: str + depthwise_bias_value_id: str | None + batch_norm_weight_value_id: str + batch_norm_bias_value_id: str + batch_norm_running_mean_value_id: str + batch_norm_running_var_value_id: str + pointwise2_weight_value_id: str + pointwise2_bias_value_id: str | None + eps: float + depthwise_kernel_size: int + depthwise_padding: int + node_ids: tuple[str, ...] + + +def match_conv_module(graph: IRGraph, node: IRNode) -> ConvModuleMatch | None: + if node.op != "permute" or len(node.inputs) != 1 or len(node.outputs) != 1: + return None + if tuple(node.attrs.get("permutation", ())) != (0, 2, 1): + return None + + pointwise2 = producer(graph, strip_passthrough(graph, node.inputs[0])) + if not _is_pointwise_conv1d(pointwise2): + return None + + silu = producer(graph, strip_passthrough(graph, pointwise2.inputs[0])) + if silu is None or silu.op != "silu" or len(silu.inputs) != 1: + return None + + batch_norm = producer(graph, strip_passthrough(graph, silu.inputs[0])) + if batch_norm is None or batch_norm.op != "batch_norm" or len(batch_norm.inputs) != 5: + return None + + depthwise = producer(graph, strip_passthrough(graph, batch_norm.inputs[0])) + if not _is_same_depthwise_conv1d(depthwise): + return None + + glu = producer(graph, strip_passthrough(graph, depthwise.inputs[0])) + if glu is None or glu.op != "glu" or len(glu.inputs) != 1: + return None + if int(glu.attrs.get("axis", -1)) not in {1, -2}: + return None + + pointwise1 = producer(graph, strip_passthrough(graph, glu.inputs[0])) + if not _is_pointwise_conv1d(pointwise1): + return None + + input_permute = producer(graph, strip_passthrough(graph, pointwise1.inputs[0])) + if input_permute is None or input_permute.op != "permute" or len(input_permute.inputs) != 1: + return None + if tuple(input_permute.attrs.get("permutation", ())) != (0, 2, 1): + return None + + depthwise_weight_shape = graph.values.get(depthwise.inputs[1]).shape if depthwise.inputs[1] in graph.values else None + if depthwise_weight_shape is None or len(depthwise_weight_shape) != 3: + return None + depthwise_kernel_size = int(depthwise_weight_shape[2]) + + return ConvModuleMatch( + input_value_id=strip_passthrough(graph, input_permute.inputs[0]), + pointwise1_weight_value_id=pointwise1.inputs[1], + pointwise1_bias_value_id=pointwise1.inputs[2] if len(pointwise1.inputs) > 2 else None, + depthwise_weight_value_id=depthwise.inputs[1], + depthwise_bias_value_id=depthwise.inputs[2] if len(depthwise.inputs) > 2 else None, + batch_norm_weight_value_id=batch_norm.inputs[1], + batch_norm_bias_value_id=batch_norm.inputs[2], + batch_norm_running_mean_value_id=batch_norm.inputs[3], + batch_norm_running_var_value_id=batch_norm.inputs[4], + pointwise2_weight_value_id=pointwise2.inputs[1], + pointwise2_bias_value_id=pointwise2.inputs[2] if len(pointwise2.inputs) > 2 else None, + eps=float(batch_norm.attrs.get("eps", 1e-5)), + depthwise_kernel_size=depthwise_kernel_size, + depthwise_padding=int(depthwise.attrs.get("padding", 0)), + node_ids=collect_node_ids( + node, + pointwise2, + silu, + batch_norm, + depthwise, + glu, + pointwise1, + input_permute, + ), + ) + + +def _is_pointwise_conv1d(node: IRNode | None) -> bool: + return ( + node is not None + and node.op == "conv1d" + and len(node.inputs) >= 2 + and int(node.attrs.get("stride", 1)) == 1 + and int(node.attrs.get("padding", 0)) == 0 + and int(node.attrs.get("dilation", 1)) == 1 + and int(node.attrs.get("groups", 1)) == 1 + ) + + +def _is_same_depthwise_conv1d(node: IRNode | None) -> bool: + if node is None or node.op != "conv1d" or len(node.inputs) < 2: + return False + groups = int(node.attrs.get("groups", 1)) + padding = int(node.attrs.get("padding", 0)) + stride = int(node.attrs.get("stride", 1)) + dilation = int(node.attrs.get("dilation", 1)) + if stride != 1 or dilation != 1: + return False + return groups > 1 and padding >= 0 diff --git a/python/cactus/transpile/fusion/deltanet.py b/python/cactus/transpile/fusion/deltanet.py new file mode 100644 index 000000000..94f4e9299 --- /dev/null +++ b/python/cactus/transpile/fusion/deltanet.py @@ -0,0 +1,357 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_passthrough +from cactus.transpile.fusion.linear import match_linear +from cactus.transpile.fusion.rms_norm import match_rms_norm +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class GatedDeltaNetMatch: + output_value_id: str + normalized_input_value_id: str + qkv_weight_value_id: str + a_weight_value_id: str + b_weight_value_id: str + norm_weight_value_id: str + z_weight_value_id: str | None + dt_bias_value_id: str | None + a_log_value_id: str | None + conv_weight_value_id: str | None + num_k_heads: int + num_v_heads: int + key_dim: int + value_dim: int + eps: float + chunk_size: int + mode: str + node_ids: tuple[str, ...] + + +def match_gated_deltanet(graph: IRGraph, node: IRNode) -> GatedDeltaNetMatch | None: + if node.op not in {"view", "reshape"} or len(node.outputs) != 1: + return None + output_value = graph.values.get(node.outputs[0]) + if output_value is None or output_value.shape is None or len(output_value.shape) != 3: + return None + + body_value_id = strip_passthrough(graph, node.inputs[0]) + body_node = producer(graph, body_value_id) + if body_node is None: + return None + + z_weight_value_id: str | None = None + rms_match = None + normalized_input_value_id: str | None = None + node_ids: set[str] = {node.id} + + if body_node.op == "multiply" and len(body_node.inputs) == 2: + for rms_value_id, z_value_id in ((body_node.inputs[0], body_node.inputs[1]), (body_node.inputs[1], body_node.inputs[0])): + rms_match = _match_rms_from_value(graph, rms_value_id) + if rms_match is None: + continue + z_weight_value_id, normalized_input_value_id, z_node_ids = _extract_z_branch(graph, z_value_id) + if normalized_input_value_id is None: + continue + node_ids.update(z_node_ids) + node_ids.add(body_node.id) + break + if rms_match is None or normalized_input_value_id is None: + return None + else: + rms_match = _match_rms_from_value(graph, body_value_id) + if rms_match is None: + return None + + y2d_value_id = rms_match.input_value_id + if rms_match.weight_value_id is None: + return None + norm_weight_value_id = rms_match.weight_value_id + eps = float(rms_match.eps) + node_ids.update(rms_match.node_ids) + + if normalized_input_value_id is None: + normalized_input_value_id = _infer_normalized_input_from_y_branch(graph, y2d_value_id) + if normalized_input_value_id is None: + return None + + qkv_info = _find_qkv_linear(graph, normalized_input_value_id) + if qkv_info is None: + return None + qkv_weight_value_id, conv_weight_value_id, num_k_heads, key_dim, num_v_heads, value_dim, qkv_node_ids = qkv_info + node_ids.update(qkv_node_ids) + + a_info = _find_a_linear(graph, normalized_input_value_id) + b_info = _find_b_linear(graph, normalized_input_value_id) + if a_info is None or b_info is None: + return None + a_weight_value_id, dt_bias_value_id, a_log_value_id, a_node_ids = a_info + b_weight_value_id, b_node_ids = b_info + node_ids.update(a_node_ids) + node_ids.update(b_node_ids) + + mode, chunk_size = _classify_mode(graph, y2d_value_id, match_node_ids=node_ids) + + return GatedDeltaNetMatch( + output_value_id=node.outputs[0], + normalized_input_value_id=normalized_input_value_id, + qkv_weight_value_id=qkv_weight_value_id, + a_weight_value_id=a_weight_value_id, + b_weight_value_id=b_weight_value_id, + norm_weight_value_id=norm_weight_value_id, + z_weight_value_id=z_weight_value_id, + dt_bias_value_id=dt_bias_value_id, + a_log_value_id=a_log_value_id, + conv_weight_value_id=conv_weight_value_id, + num_k_heads=num_k_heads, + num_v_heads=num_v_heads, + key_dim=key_dim, + value_dim=value_dim, + eps=eps, + chunk_size=chunk_size, + mode=mode, + node_ids=tuple(sorted(node_ids)), + ) + + +def _match_rms_from_value(graph: IRGraph, value_id: str): + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + return None + return match_rms_norm(graph, node) + + +def _extract_z_branch(graph: IRGraph, value_id: str) -> tuple[str | None, str | None, tuple[str, ...]]: + current = strip_passthrough(graph, value_id) + silu_node = producer(graph, current) + if silu_node is None or silu_node.op != "silu" or len(silu_node.inputs) != 1: + return None, None, () + linear_value_id = strip_passthrough(graph, silu_node.inputs[0]) + linear_node = producer(graph, linear_value_id) + while linear_node is not None and linear_node.op in {"view", "reshape"} and len(linear_node.inputs) == 1: + linear_value_id = strip_passthrough(graph, linear_node.inputs[0]) + linear_node = producer(graph, linear_value_id) + if linear_node is None: + return None, None, () + linear_match = match_linear(graph, linear_node) + if linear_match is None: + return None, None, () + return ( + linear_match.weight_value_id, + linear_match.input_value_id, + tuple(sorted({silu_node.id, *linear_match.node_ids})), + ) + + +def _infer_normalized_input_from_y_branch(graph: IRGraph, value_id: str) -> str | None: + current = strip_passthrough(graph, value_id) + node = producer(graph, current) + visited: set[str] = set() + while node is not None and node.id not in visited: + visited.add(node.id) + for input_value_id in node.inputs: + candidate = producer(graph, strip_passthrough(graph, input_value_id)) + if candidate is None: + continue + linear_match = match_linear(graph, candidate) + if linear_match is not None: + return linear_match.input_value_id + if len(node.inputs) != 1: + return None + node = producer(graph, strip_passthrough(graph, node.inputs[0])) + return None + + +def _find_qkv_linear( + graph: IRGraph, + normalized_input_value_id: str, +) -> tuple[str, str | None, int, int, int, int, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + split_node = _find_descendant_op(graph, linear_match.output_value_id, {"split_with_sizes"}, max_depth=8) + if split_node is None: + continue + split_sizes = tuple(int(v) for v in split_node.attrs.get("sizes", ())) + if len(split_sizes) != 3: + continue + + conv_node = _find_descendant_op(graph, linear_match.output_value_id, {"conv1d"}, max_depth=6) + conv_weight_value_id = conv_node.inputs[1] if conv_node is not None and len(conv_node.inputs) >= 2 else None + + q_shape = None + v_shape = None + node_ids = {node.id, split_node.id} + if conv_node is not None: + node_ids.add(conv_node.id) + + split_value = graph.values.get(split_node.outputs[0]) + if split_value is None: + continue + for getitem_node_id in split_value.users: + getitem_node = graph.nodes.get(getitem_node_id) + if getitem_node is None or getitem_node.op != "getitem" or not getitem_node.outputs: + continue + shape = _find_rank4_shape(graph, getitem_node.outputs[0], max_depth=5) + if shape is None: + continue + branch_has_sum = _find_descendant_op(graph, getitem_node.outputs[0], {"sum"}, max_depth=8) is not None + node_ids.add(getitem_node.id) + if branch_has_sum and q_shape is None: + q_shape = shape + elif not branch_has_sum and v_shape is None: + v_shape = shape + + if q_shape is None or v_shape is None: + continue + + num_k_heads = int(q_shape[2]) + key_dim = int(q_shape[3]) + num_v_heads = int(v_shape[2]) + value_dim = int(v_shape[3]) + if num_k_heads <= 0 or key_dim <= 0 or num_v_heads <= 0 or value_dim <= 0: + continue + + return ( + linear_match.weight_value_id, + conv_weight_value_id, + num_k_heads, + key_dim, + num_v_heads, + value_dim, + tuple(sorted(node_ids | set(linear_match.node_ids))), + ) + return None + + +def _find_a_linear( + graph: IRGraph, + normalized_input_value_id: str, +) -> tuple[str, str | None, str | None, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + softplus_node = _find_descendant_op(graph, linear_match.output_value_id, {"softplus"}, max_depth=6) + if softplus_node is None: + continue + dt_bias_value_id = None + softplus_input_node = producer(graph, strip_passthrough(graph, softplus_node.inputs[0])) + if softplus_input_node is not None and softplus_input_node.op == "add": + for input_value_id in softplus_input_node.inputs: + value = graph.values.get(input_value_id) + if value is not None and value.producer is None and input_value_id in graph.constants: + dt_bias_value_id = input_value_id + break + a_log_value_id = None + multiply_node = _find_descendant_op(graph, softplus_node.outputs[0], {"multiply"}, max_depth=6) + if multiply_node is not None: + for input_value_id in multiply_node.inputs: + a_log_value_id = _extract_exp_constant_input(graph, input_value_id) + if a_log_value_id is not None: + break + node_ids = {softplus_node.id, *linear_match.node_ids} + if softplus_input_node is not None: + node_ids.add(softplus_input_node.id) + if multiply_node is not None: + node_ids.add(multiply_node.id) + return linear_match.weight_value_id, dt_bias_value_id, a_log_value_id, tuple(sorted(node_ids)) + return None + + +def _find_b_linear(graph: IRGraph, normalized_input_value_id: str) -> tuple[str, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + sigmoid_node = _find_descendant_op(graph, linear_match.output_value_id, {"sigmoid"}, max_depth=4) + if sigmoid_node is None: + continue + return linear_match.weight_value_id, tuple(sorted({sigmoid_node.id, *linear_match.node_ids})) + return None + + +def _extract_exp_constant_input(graph: IRGraph, value_id: str) -> str | None: + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + value = graph.values.get(value_id) + if value is not None and value.producer is None and value_id in graph.constants: + return value_id + return None + if node.op == "scalar_multiply": + return _extract_exp_constant_input(graph, node.inputs[0]) + if node.op == "scalar_exp": + source = strip_passthrough(graph, node.inputs[0]) + value = graph.values.get(source) + if value is not None and value.producer is None and source in graph.constants: + return source + return None + + +def _find_rank4_shape(graph: IRGraph, value_id: str, *, max_depth: int) -> tuple[int, int, int, int] | None: + frontier = [(value_id, 0)] + seen: set[str] = set() + while frontier: + current, depth = frontier.pop(0) + if current in seen or depth > max_depth: + continue + seen.add(current) + value = graph.values.get(current) + if value is not None and value.shape is not None and len(value.shape) == 4: + shape = tuple(int(dim) for dim in value.shape) + return shape # type: ignore[return-value] + for user_id in graph.values.get(current, None).users if graph.values.get(current, None) is not None else (): + user = graph.nodes.get(user_id) + if user is None or not user.outputs: + continue + frontier.extend((output_id, depth + 1) for output_id in user.outputs) + return None + + +def _find_descendant_op(graph: IRGraph, value_id: str, targets: set[str], *, max_depth: int) -> IRNode | None: + frontier = [(value_id, 0)] + seen: set[str] = set() + while frontier: + current, depth = frontier.pop(0) + if current in seen or depth > max_depth: + continue + seen.add(current) + value = graph.values.get(current) + if value is None: + continue + for user_id in value.users: + user = graph.nodes.get(user_id) + if user is None or not user.outputs: + continue + if user.op in targets: + return user + frontier.extend((output_id, depth + 1) for output_id in user.outputs) + return None + + +def _classify_mode(graph: IRGraph, y2d_value_id: str, *, match_node_ids: set[str]) -> tuple[str, int]: + del y2d_value_id + chunk_size = _infer_prefill_chunk_size(graph, match_node_ids) + # Current exported graphs are uncached/full-sequence captures, so fuse the + # linear-attention block to the prefill kernel by default. Decode should be + # emitted only once the capture path models cache/state explicitly. + return "prefill", chunk_size + + +def _infer_prefill_chunk_size(graph: IRGraph, match_node_ids: set[str]) -> int: + for node_id in match_node_ids: + node = graph.nodes.get(node_id) + if node is None or node.op != "aten.triu.default" or not node.inputs: + continue + ones_value = graph.values.get(node.inputs[0]) + if ones_value is not None and ones_value.shape and len(ones_value.shape) >= 1: + return int(ones_value.shape[-1]) + return 64 diff --git a/python/cactus/transpile/fusion/linear.py b/python/cactus/transpile/fusion/linear.py new file mode 100644 index 000000000..4841019f3 --- /dev/null +++ b/python/cactus/transpile/fusion/linear.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class LinearMatch: + input_value_id: str + weight_value_id: str + bias_value_id: str | None + output_value_id: str + node_ids: tuple[str, ...] + kind: str + + +def match_linear(graph: IRGraph, node: IRNode) -> LinearMatch | None: + if node.op == "linear" and len(node.inputs) >= 2 and len(node.outputs) == 1: + return LinearMatch( + input_value_id=strip_passthrough(graph, node.inputs[0]), + weight_value_id=node.inputs[1], + bias_value_id=node.inputs[2] if bool(node.attrs.get("has_bias", False)) and len(node.inputs) > 2 else None, + output_value_id=node.outputs[0], + node_ids=(node.id,), + kind="linear", + ) + + if node.op == "addmm" and len(node.inputs) == 3 and len(node.outputs) == 1: + return LinearMatch( + input_value_id=strip_passthrough(graph, node.inputs[1]), + weight_value_id=node.inputs[2], + bias_value_id=node.inputs[0], + output_value_id=node.outputs[0], + node_ids=(node.id,), + kind="addmm", + ) + + if node.op != "add" or len(node.inputs) != 2 or len(node.outputs) != 1: + return None + + for matmul_value_id, bias_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + matmul_node = producer(graph, strip_passthrough(graph, matmul_value_id)) + if matmul_node is None or matmul_node.op != "matmul" or len(matmul_node.inputs) != 2: + continue + return LinearMatch( + input_value_id=strip_passthrough(graph, matmul_node.inputs[0]), + weight_value_id=matmul_node.inputs[1], + bias_value_id=bias_value_id, + output_value_id=node.outputs[0], + node_ids=tuple(sorted({node.id, matmul_node.id})), + kind="matmul_add", + ) + + return None diff --git a/python/cactus/transpile/fusion/lstm.py b/python/cactus/transpile/fusion/lstm.py new file mode 100644 index 000000000..64831240e --- /dev/null +++ b/python/cactus/transpile/fusion/lstm.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from cactus.transpile.fusion.common import collect_node_ids +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_passthrough +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class LSTMCellMatch: + anchor_node_id: str + node_ids: tuple[str, ...] + x_value_id: str + h_prev_value_id: str + c_prev_value_id: str + weight_ih_value_id: str + weight_hh_value_id: str + bias_ih_value_id: str + bias_hh_value_id: str | None + h_output_value_id: str + c_output_value_id: str + + +def match_lstm_cell(graph: IRGraph, node: IRNode) -> LSTMCellMatch | None: + if node.op != "multiply" or len(node.inputs) != 2 or len(node.outputs) != 1: + return None + + for output_gate_value_id, c_tanh_value_id in ( + (node.inputs[0], node.inputs[1]), + (node.inputs[1], node.inputs[0]), + ): + output_gate = _extract_gate_activation(graph, output_gate_value_id, expected_activation="sigmoid") + if output_gate is None or output_gate["index"] != 3: + continue + + c_tanh_node = producer(graph, strip_passthrough(graph, c_tanh_value_id)) + if c_tanh_node is None or c_tanh_node.op != "tanh" or len(c_tanh_node.inputs) != 1: + continue + + c_next_value_id = strip_passthrough(graph, c_tanh_node.inputs[0]) + c_next_node = producer(graph, c_next_value_id) + if c_next_node is None or c_next_node.op != "add" or len(c_next_node.inputs) != 2 or len(c_next_node.outputs) != 1: + continue + + for forget_term_value_id, input_term_value_id in ( + (c_next_node.inputs[0], c_next_node.inputs[1]), + (c_next_node.inputs[1], c_next_node.inputs[0]), + ): + forget_term = _extract_forget_term(graph, forget_term_value_id) + input_term = _extract_input_term(graph, input_term_value_id) + if forget_term is None or input_term is None: + continue + + input_gate = input_term["input_gate"] + cell_gate = input_term["cell_gate"] + forget_gate = forget_term["forget_gate"] + chunk_node_id = output_gate["chunk_node_id"] + if ( + input_gate["chunk_node_id"] != chunk_node_id + or forget_gate["chunk_node_id"] != chunk_node_id + or cell_gate["chunk_node_id"] != chunk_node_id + ): + continue + if input_gate["index"] != 0 or forget_gate["index"] != 1 or cell_gate["index"] != 2: + continue + + chunk_node = graph.nodes.get(chunk_node_id) + if chunk_node is None or len(chunk_node.inputs) != 1: + continue + + gates = _extract_lstm_gate_sum(graph, chunk_node.inputs[0]) + if gates is None: + continue + + return LSTMCellMatch( + anchor_node_id=node.id, + node_ids=collect_node_ids( + node, + c_tanh_node, + c_next_node, + output_gate["node_ids"], + forget_gate["node_ids"], + input_gate["node_ids"], + cell_gate["node_ids"], + forget_term["node_ids"], + input_term["node_ids"], + chunk_node, + gates["node_ids"], + ), + x_value_id=gates["x_value_id"], + h_prev_value_id=gates["h_prev_value_id"], + c_prev_value_id=forget_term["c_prev_value_id"], + weight_ih_value_id=gates["weight_ih_value_id"], + weight_hh_value_id=gates["weight_hh_value_id"], + bias_ih_value_id=gates["bias_value_ids"][0], + bias_hh_value_id=gates["bias_value_ids"][1] if len(gates["bias_value_ids"]) > 1 else None, + h_output_value_id=node.outputs[0], + c_output_value_id=c_next_node.outputs[0], + ) + + return None + + +def _extract_gate_activation( + graph: IRGraph, + value_id: str, + *, + expected_activation: str, +) -> dict[str, object] | None: + activation_node = producer(graph, strip_passthrough(graph, value_id)) + if activation_node is None or activation_node.op != expected_activation or len(activation_node.inputs) != 1: + return None + + getitem_node = producer(graph, strip_passthrough(graph, activation_node.inputs[0])) + if getitem_node is None or getitem_node.op != "getitem" or len(getitem_node.inputs) != 1: + return None + + chunk_node = producer(graph, strip_passthrough(graph, getitem_node.inputs[0])) + if chunk_node is None or chunk_node.op != "chunk": + return None + if int(chunk_node.attrs.get("chunks", 0)) != 4: + return None + + return { + "chunk_node_id": chunk_node.id, + "index": int(getitem_node.attrs.get("index", -1)), + "preactivation_value_id": getitem_node.inputs[0], + "node_ids": (activation_node.id, getitem_node.id), + } + + +def _extract_forget_term(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul_node = producer(graph, strip_passthrough(graph, value_id)) + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for gate_value_id, c_prev_value_id in ( + (mul_node.inputs[0], mul_node.inputs[1]), + (mul_node.inputs[1], mul_node.inputs[0]), + ): + forget_gate = _extract_gate_activation(graph, gate_value_id, expected_activation="sigmoid") + if forget_gate is None: + continue + return { + "forget_gate": forget_gate, + "c_prev_value_id": strip_passthrough(graph, c_prev_value_id), + "node_ids": (mul_node.id,), + } + return None + + +def _extract_input_term(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul_node = producer(graph, strip_passthrough(graph, value_id)) + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for input_gate_value_id, cell_gate_value_id in ( + (mul_node.inputs[0], mul_node.inputs[1]), + (mul_node.inputs[1], mul_node.inputs[0]), + ): + input_gate = _extract_gate_activation(graph, input_gate_value_id, expected_activation="sigmoid") + if input_gate is None: + continue + cell_tanh_node = producer(graph, strip_passthrough(graph, cell_gate_value_id)) + if cell_tanh_node is None or cell_tanh_node.op != "tanh" or len(cell_tanh_node.inputs) != 1: + continue + cell_gate = _extract_gate_activation(graph, cell_gate_value_id, expected_activation="tanh") + if cell_gate is None: + continue + return { + "input_gate": input_gate, + "cell_gate": cell_gate, + "node_ids": (mul_node.id, cell_tanh_node.id), + } + return None + + +def _extract_lstm_gate_sum(graph: IRGraph, value_id: str) -> dict[str, object] | None: + terms, node_ids = _flatten_add_terms(graph, strip_passthrough(graph, value_id)) + linear_nodes: list[IRNode] = [] + bias_value_ids: list[str] = [] + for term_value_id in terms: + term_value_id = strip_passthrough(graph, term_value_id) + term_node = producer(graph, term_value_id) + if term_node is not None and term_node.op == "linear" and len(term_node.inputs) >= 2: + linear_nodes.append(term_node) + continue + if term_value_id in graph.constants: + bias_value_ids.append(term_value_id) + continue + return None + + if len(linear_nodes) != 2 or len(bias_value_ids) not in {1, 2}: + return None + + first_linear, second_linear = linear_nodes + return { + "x_value_id": first_linear.inputs[0], + "weight_ih_value_id": first_linear.inputs[1], + "h_prev_value_id": second_linear.inputs[0], + "weight_hh_value_id": second_linear.inputs[1], + "bias_value_ids": tuple(bias_value_ids), + "node_ids": (*node_ids, first_linear.id, second_linear.id), + } + + +def _flatten_add_terms(graph: IRGraph, value_id: str) -> tuple[list[str], tuple[str, ...]]: + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return [value_id], () + left_terms, left_nodes = _flatten_add_terms(graph, node.inputs[0]) + right_terms, right_nodes = _flatten_add_terms(graph, node.inputs[1]) + return [*left_terms, *right_terms], (*left_nodes, *right_nodes, node.id) diff --git a/python/cactus/transpile/fusion/mlp.py b/python/cactus/transpile/fusion/mlp.py new file mode 100644 index 000000000..56a219514 --- /dev/null +++ b/python/cactus/transpile/fusion/mlp.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.fusion.common import strip_passthrough +from cactus.transpile.fusion.linear import match_linear + + +@dataclass(frozen=True) +class GatedMLPMatch: + input_value_id: str + gate_weight_value_id: str + up_weight_value_id: str + down_weight_value_id: str + activation: str + bias_value_ids: tuple[str | None, str | None, str | None] + node_ids: tuple[str, ...] + + +def match_gated_mlp(graph: IRGraph, node: IRNode) -> GatedMLPMatch | None: + down = match_linear(graph, node) + if down is None: + return None + + mul_node = graph.nodes.get(graph.values[down.input_value_id].producer) if graph.values.get(down.input_value_id) is not None else None + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for activated_value_id, up_value_id in ((mul_node.inputs[0], mul_node.inputs[1]), (mul_node.inputs[1], mul_node.inputs[0])): + activated_node = graph.nodes.get(graph.values[strip_passthrough(graph, activated_value_id)].producer) if graph.values.get(strip_passthrough(graph, activated_value_id)) is not None else None + if activated_node is None or activated_node.op not in {"gelu", "silu"}: + continue + + gate = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer]) if graph.values.get(strip_passthrough(graph, activated_node.inputs[0])) is not None and graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer in graph.nodes else None + up = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, up_value_id)].producer]) if graph.values.get(strip_passthrough(graph, up_value_id)) is not None and graph.values[strip_passthrough(graph, up_value_id)].producer in graph.nodes else None + if gate is None or up is None: + continue + if strip_passthrough(graph, gate.input_value_id) != strip_passthrough(graph, up.input_value_id): + continue + + return GatedMLPMatch( + input_value_id=strip_passthrough(graph, gate.input_value_id), + gate_weight_value_id=gate.weight_value_id, + up_weight_value_id=up.weight_value_id, + down_weight_value_id=down.weight_value_id, + activation=activated_node.op, + bias_value_ids=(gate.bias_value_id, up.bias_value_id, down.bias_value_id), + node_ids=tuple(sorted({*gate.node_ids, activated_node.id, *up.node_ids, mul_node.id, *down.node_ids})), + ) + + return None diff --git a/python/cactus/transpile/fusion/rel_pos_bias.py b/python/cactus/transpile/fusion/rel_pos_bias.py new file mode 100644 index 000000000..a0e709c9c --- /dev/null +++ b/python/cactus/transpile/fusion/rel_pos_bias.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_passthrough +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class RelPosBiasMatch: + query_value_id: str + relative_key_value_id: str + scale: float + node_ids: tuple[str, ...] + + +def match_rel_pos_bias(graph: IRGraph, node: IRNode) -> RelPosBiasMatch | None: + if node.op != "scalar_multiply" or len(node.inputs) != 1: + return None + + gather_match = _match_gather_rel_pos_bias(graph, node) + if gather_match is not None: + return gather_match + + final_slice = producer(graph, strip_passthrough(graph, node.inputs[0])) + if final_slice is None or final_slice.op != "slice": + return None + if int(final_slice.attrs.get("axis", -1)) != 3 or int(final_slice.attrs.get("start", -1)) != 0: + return None + + reshape_after_shift = producer(graph, strip_passthrough(graph, final_slice.inputs[0])) + if reshape_after_shift is None or reshape_after_shift.op not in {"reshape", "view"} or len(reshape_after_shift.inputs) != 1: + return None + + shift_slice = producer(graph, strip_passthrough(graph, reshape_after_shift.inputs[0])) + if shift_slice is None or shift_slice.op != "slice": + return None + if int(shift_slice.attrs.get("axis", -1)) != 2 or int(shift_slice.attrs.get("start", -1)) != 1: + return None + + reshape_for_shift = producer(graph, strip_passthrough(graph, shift_slice.inputs[0])) + if reshape_for_shift is None or reshape_for_shift.op not in {"reshape", "view"} or len(reshape_for_shift.inputs) != 1: + return None + + pad = producer(graph, strip_passthrough(graph, reshape_for_shift.inputs[0])) + if pad is None or pad.op != "pad" or len(pad.inputs) != 1: + return None + if tuple(int(v) for v in pad.attrs.get("pads", ())) != (1, 0): + return None + if str(pad.attrs.get("mode", "constant")) != "constant": + return None + if float(pad.attrs.get("value", 0.0)) != 0.0: + return None + + matmul = producer(graph, strip_passthrough(graph, pad.inputs[0])) + if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: + return None + + rhs_permute = producer(graph, strip_passthrough(graph, matmul.inputs[1])) + if rhs_permute is None or rhs_permute.op != "permute" or len(rhs_permute.inputs) != 1: + return None + if tuple(int(v) for v in rhs_permute.attrs.get("permutation", ())) != (0, 2, 3, 1): + return None + + query_value_id = strip_passthrough(graph, matmul.inputs[0]) + relative_key_value_id = strip_passthrough(graph, rhs_permute.inputs[0]) + + query_shape = graph.values.get(query_value_id).shape if query_value_id in graph.values else None + relative_key_shape = graph.values.get(relative_key_value_id).shape if relative_key_value_id in graph.values else None + output_shape = graph.values.get(node.outputs[0]).shape if node.outputs and node.outputs[0] in graph.values else None + if query_shape is None or relative_key_shape is None or output_shape is None: + return None + if len(query_shape) != 4 or len(relative_key_shape) != 4 or len(output_shape) != 4: + return None + + batch, heads, seq_len, head_dim = (int(v) for v in query_shape) + rel_batch, rel_len, rel_heads, rel_dim = (int(v) for v in relative_key_shape) + out_batch, out_heads, out_t, out_s = (int(v) for v in output_shape) + if rel_batch not in {1, batch}: + return None + if rel_heads != heads or rel_dim != head_dim: + return None + if rel_len < 2 * seq_len - 1: + return None + if (out_batch, out_heads, out_t, out_s) != (batch, heads, seq_len, seq_len): + return None + + return RelPosBiasMatch( + query_value_id=query_value_id, + relative_key_value_id=relative_key_value_id, + scale=float(node.attrs.get("value", 1.0)), + node_ids=tuple( + sorted( + { + node.id, + final_slice.id, + reshape_after_shift.id, + shift_slice.id, + reshape_for_shift.id, + pad.id, + matmul.id, + rhs_permute.id, + } + ) + ), + ) + + +def _match_gather_rel_pos_bias(graph: IRGraph, node: IRNode) -> RelPosBiasMatch | None: + gather = producer(graph, strip_passthrough(graph, node.inputs[0])) + if gather is None or gather.op != "gather" or len(gather.inputs) < 2: + return None + if int(gather.attrs.get("axis", -1)) not in {-1, 3}: + return None + + matmul = producer(graph, strip_passthrough(graph, gather.inputs[0])) + if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: + return None + + query_heads = producer(graph, strip_passthrough(graph, matmul.inputs[0])) + rhs_transpose = producer(graph, strip_passthrough(graph, matmul.inputs[1])) + if query_heads is None or query_heads.op != "permute" or len(query_heads.inputs) != 1: + return None + if rhs_transpose is None or rhs_transpose.op != "permute" or len(rhs_transpose.inputs) != 1: + return None + if tuple(int(v) for v in query_heads.attrs.get("permutation", ())) != (0, 2, 1, 3): + return None + if tuple(int(v) for v in rhs_transpose.attrs.get("permutation", ())) != (0, 1, 3, 2): + return None + + relative_key_heads = producer(graph, strip_passthrough(graph, rhs_transpose.inputs[0])) + if relative_key_heads is None or relative_key_heads.op != "permute" or len(relative_key_heads.inputs) != 1: + return None + if tuple(int(v) for v in relative_key_heads.attrs.get("permutation", ())) != (0, 2, 1, 3): + return None + + query_value_id = strip_passthrough(graph, query_heads.inputs[0]) + relative_key_value_id = strip_passthrough(graph, relative_key_heads.inputs[0]) + output_value_id = node.outputs[0] if node.outputs else "" + + query_shape = graph.values.get(query_value_id).shape if query_value_id in graph.values else None + relative_key_shape = graph.values.get(relative_key_value_id).shape if relative_key_value_id in graph.values else None + output_shape = graph.values.get(output_value_id).shape if output_value_id in graph.values else None + if query_shape is None or relative_key_shape is None or output_shape is None: + return None + if len(query_shape) != 4 or len(relative_key_shape) != 4 or len(output_shape) != 4: + return None + + batch, seq_len, heads, head_dim = (int(v) for v in query_shape) + rel_batch, rel_len, rel_heads, rel_dim = (int(v) for v in relative_key_shape) + out_batch, out_heads, out_t, out_s = (int(v) for v in output_shape) + if rel_batch not in {1, batch}: + return None + if rel_heads != heads or rel_dim != head_dim: + return None + if rel_len < 2 * seq_len - 1: + return None + if (out_batch, out_heads, out_t, out_s) != (batch, heads, seq_len, seq_len): + return None + + return RelPosBiasMatch( + query_value_id=query_value_id, + relative_key_value_id=relative_key_value_id, + scale=float(node.attrs.get("value", 1.0)), + node_ids=tuple( + sorted( + { + node.id, + gather.id, + matmul.id, + query_heads.id, + rhs_transpose.id, + relative_key_heads.id, + } + ) + ), + ) diff --git a/python/cactus/transpile/fusion/rms_norm.py b/python/cactus/transpile/fusion/rms_norm.py new file mode 100644 index 000000000..8951e3c42 --- /dev/null +++ b/python/cactus/transpile/fusion/rms_norm.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class RMSNormMatch: + input_value_id: str + weight_value_id: str | None + weight_offset: float + eps: float + node_ids: tuple[str, ...] + + +def match_rms_norm(graph: IRGraph, node: IRNode) -> RMSNormMatch | None: + anchor_value_id = node.inputs[0] if node.op == "type_as" and node.inputs else node.outputs[0] + anchor_value_id = strip_passthrough(graph, anchor_value_id) + anchor_node = producer(graph, anchor_value_id) + if anchor_node is None or anchor_node.op != "multiply" or len(anchor_node.inputs) != 2: + return None + + for normed_value_id, scale_value_id in ( + (anchor_node.inputs[0], anchor_node.inputs[1]), + (anchor_node.inputs[1], anchor_node.inputs[0]), + ): + scale = _extract_scale(graph, scale_value_id) + norm = _extract_normed_value(graph, normed_value_id) + if scale is None or norm is None: + continue + if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): + continue + return RMSNormMatch( + input_value_id=norm["input_value_id"], + weight_value_id=scale["weight_value_id"], + weight_offset=float(scale["offset"]), + eps=float(norm["eps"]), + node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"], *scale["node_ids"]})), + ) + + norm = _extract_normed_value(graph, anchor_node.outputs[0]) + if norm is None: + return None + if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): + return None + return RMSNormMatch( + input_value_id=norm["input_value_id"], + weight_value_id=None, + weight_offset=0.0, + eps=float(norm["eps"]), + node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"]})), + ) + + +def _extract_scale(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + offset = 0.0 + node_ids: set[str] = set() + producer_node = producer(graph, current) + if producer_node is not None and producer_node.op == "scalar_add": + add_value = float(producer_node.attrs.get("value", 0.0)) + if add_value in (0.0, 1.0): + offset = add_value + node_ids.add(producer_node.id) + current = producer_node.inputs[0] + current = strip_passthrough(graph, current) + if current not in graph.constants or not isinstance(graph.constants[current], torch.Tensor): + return None + return {"weight_value_id": current, "offset": offset, "node_ids": tuple(sorted(node_ids))} + + +def _extract_normed_value(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul = producer(graph, strip_passthrough(graph, value_id)) + if mul is None or mul.op != "multiply" or len(mul.inputs) != 2: + return None + for x_candidate, rsqrt_candidate in ((mul.inputs[0], mul.inputs[1]), (mul.inputs[1], mul.inputs[0])): + rsqrt = _extract_rsqrt_chain(graph, rsqrt_candidate) + if rsqrt is None: + continue + if strip_passthrough(graph, x_candidate) != strip_passthrough(graph, rsqrt["pow_input_value_id"]): + continue + return { + "input_value_id": strip_passthrough(graph, x_candidate), + "pow_input_value_id": rsqrt["pow_input_value_id"], + "eps": rsqrt["eps"], + "node_ids": tuple(sorted({mul.id, *rsqrt["node_ids"]})), + } + return None + + +def _extract_rsqrt_chain(graph: IRGraph, value_id: str) -> dict[str, object] | None: + pow_node = producer(graph, strip_passthrough(graph, value_id)) + if pow_node is None or pow_node.op != "pow" or float(pow_node.attrs.get("exponent", 0.0)) != -0.5: + return None + add_node = producer(graph, pow_node.inputs[0]) + if add_node is None or add_node.op != "scalar_add": + return None + mean_node = producer(graph, add_node.inputs[0]) + if mean_node is None or mean_node.op != "mean" or not bool(mean_node.attrs.get("keepdim", False)): + return None + pow2_node = producer(graph, mean_node.inputs[0]) + if pow2_node is None or pow2_node.op != "pow" or float(pow2_node.attrs.get("exponent", 0.0)) != 2.0: + return None + return { + "pow_input_value_id": pow2_node.inputs[0], + "eps": float(add_node.attrs.get("value", 0.0)), + "node_ids": (pow_node.id, add_node.id, mean_node.id, pow2_node.id), + } diff --git a/python/cactus/transpile/fusion/rope.py b/python/cactus/transpile/fusion/rope.py new file mode 100644 index 000000000..25118b868 --- /dev/null +++ b/python/cactus/transpile/fusion/rope.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_layout_passthrough +from cactus.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class RoPEMatch: + input_value_id: str + theta: float + position_offset: int + partial: bool + node_ids: tuple[str, ...] + + +def match_rope(graph: IRGraph, value_id: str) -> RoPEMatch | None: + desc = _extract_rope_descriptor(graph, value_id) + if desc is None: + return None + return RoPEMatch( + input_value_id=strip_passthrough(graph, desc["input_value_id"]), + theta=float(desc["theta"]), + position_offset=int(desc["position_offset"]), + partial=bool(desc.get("partial_rope", False)), + node_ids=tuple(sorted(set(desc.get("node_ids", ())))), + ) + + +def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = strip_layout_passthrough(graph, value_id) + node = producer(graph, current) + if node is None: + return None + if node.op == "rope": + return { + "input_value_id": node.inputs[0], + "theta": float(node.attrs.get("theta", 0.0)), + "position_offset": int(node.attrs.get("position_offset", 0)), + "partial_rope": False, + "node_ids": (node.id,), + } + classic = _extract_classic_rope_descriptor(graph, current) + if classic is not None: + classic["partial_rope"] = False + return classic + if node.op == "cat" and len(node.inputs) == 2: + for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) + if rope_branch is None: + continue + passthrough_node = producer(graph, strip_layout_passthrough(graph, passthrough_value_id)) + if passthrough_node is None or passthrough_node.op != "slice": + continue + if strip_passthrough(graph, passthrough_node.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): + continue + rope_branch["partial_rope"] = True + rope_branch["node_ids"] = tuple(sorted({node.id, passthrough_node.id, *rope_branch["node_ids"]})) + return rope_branch + return None + + +def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node = producer(graph, strip_layout_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return None + for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + direct_mult = producer(graph, direct_mult_id) + rotated_mult = producer(graph, rotated_mult_id) + if direct_mult is None or direct_mult.op != "multiply" or rotated_mult is None or rotated_mult.op != "multiply": + continue + direct_match = _extract_direct_rope_branch(graph, direct_mult) + rotated_match = _extract_rotated_rope_branch(graph, rotated_mult) + if direct_match is None or rotated_match is None: + continue + input_value_id = strip_passthrough(graph, direct_match["input_value_id"]) + if input_value_id != strip_passthrough(graph, rotated_match["input_value_id"]): + continue + cos_info = _extract_rope_trig(graph, direct_match["trig_value_id"], expected="scalar_cos") + sin_info = _extract_rope_trig(graph, rotated_match["trig_value_id"], expected="scalar_sin") + if cos_info is None or sin_info is None: + continue + if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: + continue + if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): + continue + return { + "input_value_id": input_value_id, + "theta": cos_info["theta"], + "position_offset": cos_info["position_offset"], + "node_ids": tuple(sorted({node.id, *direct_match["node_ids"], *rotated_match["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), + } + return None + + +def _extract_direct_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + if _unwrap_trig_value(graph, trig_value_id, expected="scalar_cos") is not None: + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotated_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + input_value_id = _extract_rotate_half_source(graph, rotated_value_id) + if input_value_id is None: + continue + if _unwrap_trig_value(graph, trig_value_id, expected="scalar_sin") is None: + continue + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: + cat_node = producer(graph, strip_passthrough(graph, value_id)) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: + return None + neg_node = producer(graph, cat_node.inputs[0]) + right_slice = producer(graph, cat_node.inputs[1]) + if neg_node is None or neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: + return None + left_slice = producer(graph, neg_node.inputs[0]) + if left_slice is None or right_slice is None or left_slice.op != "slice" or right_slice.op != "slice": + return None + if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): + return None + return left_slice.inputs[0] + + +def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: + trig_info = _unwrap_trig_value(graph, value_id, expected=expected) + if trig_info is None: + return None + trig_node = trig_info["trig_node"] + cat_node = producer(graph, trig_node.inputs[0]) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: + return None + angle_info = _extract_rope_angle_source(graph, cat_node.inputs[0]) + if angle_info is None: + return None + matmul_node = angle_info["matmul_node"] + inv_freq_const_id = None + arange_node = None + for input_id in matmul_node.inputs: + if inv_freq_const_id is None: + inv_freq_const_id = _find_constant_ancestor(graph, input_id) + if arange_node is None: + arange_node = _find_arange_ancestor(graph, input_id) + if inv_freq_const_id is None or arange_node is None: + return None + theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) + if theta is None: + return None + return { + "theta": theta, + "position_offset": int(arange_node.attrs.get("start", 0)), + "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), + } + + +def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"unsqueeze", "precision_cast", "reshape", "view", "expand"}: + current = node.inputs[0] + continue + if node.op == "type_as": + current = node.inputs[0] + continue + if node.op == "scalar_multiply": + current = node.inputs[0] + continue + if node.op != expected: + return None + return {"trig_node": node} + + +def _extract_rope_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + while True: + current = strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "permute": + matmul_node = producer(graph, node.inputs[0]) + if matmul_node is None or matmul_node.op != "matmul": + return None + return {"matmul_node": matmul_node} + if node.op == "matmul": + return {"matmul_node": node} + if node.op in {"index", "slice"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + return None + + +def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_layout_passthrough(graph, current) + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return current + node = producer(graph, current) + if node is None or len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "arange": + return node + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return None + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _infer_rope_theta(value: Any) -> float | None: + if not isinstance(value, torch.Tensor): + return None + flat = value.detach().cpu().float().reshape(-1) + if flat.numel() < 2: + return None + second = float(flat[1].item()) + if second <= 0.0: + return None + inv_count = flat.numel() + return float((1.0 / second) ** inv_count) diff --git a/python/cactus/transpile/capture/graph_ir.py b/python/cactus/transpile/graph_ir.py similarity index 65% rename from python/cactus/transpile/capture/graph_ir.py rename to python/cactus/transpile/graph_ir.py index 56d9dcd18..632585c57 100644 --- a/python/cactus/transpile/capture/graph_ir.py +++ b/python/cactus/transpile/graph_ir.py @@ -1,6 +1,32 @@ +import copy from dataclasses import dataclass from dataclasses import field +import torch + + +def _clone_ir_payload(value, memo): + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, torch.Tensor): + # IR constants may be real tensors or export-time tensor wrappers. + # They are treated as immutable graph payloads, so preserve them by + # reference instead of triggering Tensor.__deepcopy__. + return value + if isinstance(value, tuple): + return tuple(_clone_ir_payload(item, memo) for item in value) + if isinstance(value, list): + return [_clone_ir_payload(item, memo) for item in value] + if isinstance(value, dict): + return { + copy.deepcopy(key, memo): _clone_ir_payload(inner, memo) + for key, inner in value.items() + } + try: + return copy.deepcopy(value, memo) + except Exception: + return value + @dataclass class IRValue: @@ -11,6 +37,21 @@ class IRValue: users: list[str] = field(default_factory=list) meta: dict[str, object] = field(default_factory=dict) + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRValue( + id=self.id, + shape=_clone_ir_payload(self.shape, memo), + dtype=self.dtype, + producer=self.producer, + users=list(self.users), + meta=_clone_ir_payload(self.meta, memo), + ) + memo[id(self)] = cloned + return cloned + @dataclass class IRNode: id: str @@ -21,6 +62,22 @@ class IRNode: meta: dict[str, object] = field(default_factory=dict) kind: str = "generic" + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRNode( + id=self.id, + op=self.op, + inputs=list(self.inputs), + outputs=list(self.outputs), + attrs=_clone_ir_payload(self.attrs, memo), + meta=_clone_ir_payload(self.meta, memo), + kind=self.kind, + ) + memo[id(self)] = cloned + return cloned + @dataclass class IRGraph: values: dict[str, IRValue] @@ -31,6 +88,25 @@ class IRGraph: constants: dict[str, object] = field(default_factory=dict) meta: dict[str, object] = field(default_factory=dict) + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRGraph( + values={value_id: copy.deepcopy(value, memo) for value_id, value in self.values.items()}, + nodes={node_id: copy.deepcopy(node, memo) for node_id, node in self.nodes.items()}, + order=list(self.order), + inputs=list(self.inputs), + outputs=list(self.outputs), + constants={ + value_id: _clone_ir_payload(constant, memo) + for value_id, constant in self.constants.items() + }, + meta=_clone_ir_payload(self.meta, memo), + ) + memo[id(self)] = cloned + return cloned + def add_node(self, node: IRNode) -> None: if node.id in self.nodes: raise ValueError(f"duplicate IR node id: {node.id}") diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py new file mode 100644 index 000000000..84c113c32 --- /dev/null +++ b/python/cactus/transpile/hf_model.py @@ -0,0 +1,3330 @@ +from __future__ import annotations + +import argparse +import copy +import gc +import json +import os +import re +import struct +import sys +from collections.abc import Mapping +from collections import Counter +from dataclasses import dataclass +from dataclasses import fields +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +PYTHON_ROOT = Path(__file__).resolve().parents[2] +if str(PYTHON_ROOT) not in sys.path: + sys.path.insert(0, str(PYTHON_ROOT)) + +from cactus.transpile.runtime_compat import Graph +from cactus.convert.cactus_adapters.tensor_io import CACTUS_ALIGNMENT +from cactus.convert.cactus_adapters.tensor_io import CACTUS_MAGIC +from cactus.convert.cactus_adapters.tensor_io import compute_padding +from cactus.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features +from cactus.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform +from cactus.transpile.audio_preprocess import prepare_cactus_audio_features +from cactus.transpile.audio_preprocess import prepare_native_gemma4_audio_features +from cactus.transpile.capture_pytorch import capture_model +from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph +from cactus.transpile.component_partition import extract_component_subgraphs +from cactus.transpile.component_partition import summarize_ir_components +from cactus.transpile.component_plan import infer_component_plan_from_config +from cactus.transpile.component_pipeline import capture_component_spec +from cactus.transpile.component_pipeline import execute_component_pipeline +from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs as _shared_prepare_gemma4_multimodal_inputs +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import verify_ir +from cactus.transpile.lower import TranspiledGraph +from cactus.transpile.lower import _lower_constant_value +from cactus.transpile.lower import _lower_input_value +from cactus.transpile.lower import _lower_ir_node +from cactus.transpile.lower import _lookup_weight_binding +from cactus.transpile.media_limits import resize_static_image +from cactus.transpile.model_adapters import build_component_module_specs +from cactus.transpile.model_adapters import canonicalize_model_interface +from cactus.transpile.model_profiles import multimodal_context_tokens_for_model_type +from cactus.transpile.optimize_graph import FusionConfig +from cactus.transpile.optimize_graph import optimize_graph +from cactus.transpile.runtime_support import ensure_transformers_supports_model_type as _ensure_transformers_supports_profiled_model_type +from cactus.transpile.runtime_support import patch_torch_flex_attention_compat as _patch_torch_flex_attention_compat +from cactus.transpile.runtime_support import patch_transformers_torchvision_probe as _patch_transformers_torchvision_probe +from cactus.transpile.tdt_runtime import greedy_decode_parakeet_tdt_token_ids +from cactus.transpile.tdt_runtime import load_tdt_local_model +from cactus.transpile.tdt_runtime import prepare_parakeet_tdt_audio_features +from cactus.transpile.weight_compat import ensure_binding_compatible + +_DEFAULT_CAUSAL_PROMPT = "The capital of France is" +_DEFAULT_MULTIMODAL_CONTEXT_TOKENS = 2048 +_CACTUS_FLAG_EXTENDED_SHAPE = 1 << 4 +_CACTUS_BASE_HEADER_SIZE = 84 +_CACTUS_EXTENDED_SHAPE_DIMS = 8 + + +@dataclass +class PreparedInputs: + names: tuple[str, ...] + tensors: tuple[torch.Tensor, ...] + metadata: dict[str, object] + + +def _ensure_transformers_supports_model_type(model_type: str) -> str | None: + return _ensure_transformers_supports_profiled_model_type(model_type) + + +def _resolve_local_snapshot(model_id_or_path: str) -> str | None: + explicit = Path(model_id_or_path) + if explicit.exists(): + return str(explicit) + + snapshots_dir = ( + Path.home() + / ".cache" + / "huggingface" + / "hub" + / ("models--" + model_id_or_path.replace("/", "--")) + / "snapshots" + ) + if not snapshots_dir.exists(): + return None + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + return None + return str(snapshots[-1]) + + +def _snapshot_has_model_weights(path: str | Path) -> bool: + root = Path(path) + if not root.exists() or not root.is_dir(): + return False + candidates = ( + "model.safetensors", + "model.safetensors.index.json", + "pytorch_model.bin", + "pytorch_model.bin.index.json", + ) + return any((root / name).exists() for name in candidates) + + +def _validate_weights_dir(weights_dir: str | None, *, model_id: str) -> Path | None: + if not weights_dir: + return None + + root = Path(weights_dir).resolve() + if not root.exists(): + raise RuntimeError( + f"weights_dir does not exist: {root}\n" + "\n" + f"Create the folder first with:\n" + f" cactus convert {model_id} {root}\n" + ) + + manifest_path = root / "weights_manifest.json" + if not manifest_path.exists(): + raise RuntimeError( + f"weights_dir is missing weights_manifest.json: {manifest_path}\n" + "\n" + "The transpiler binds weights only through the converted CQ/Cactus " + "manifest; it no longer guesses filenames from model-specific layer names.\n" + "\n" + f"Re-convert with the current converter:\n" + f" cactus convert {model_id} {root}\n" + ) + + return root + + +def _serialize_json_compatible(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): _serialize_json_compatible(inner) for key, inner in value.items()} + if isinstance(value, (list, tuple)): + return [_serialize_json_compatible(inner) for inner in value] + if isinstance(value, Path): + return str(value) + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, torch.dtype): + return str(value) + if isinstance(value, torch.Tensor): + return { + "type": "torch.Tensor", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if isinstance(value, np.ndarray): + return { + "type": "numpy.ndarray", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if hasattr(value, "__dataclass_fields__"): + return { + field.name: _serialize_json_compatible(getattr(value, field.name)) + for field in fields(value) + } + try: + return repr(value) + except Exception: + return f"<{type(value).__module__}.{type(value).__name__}>" + + +def _slugify_model_artifact_name(value: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("._-") or "model" + + +def _default_artifact_dir_for_model(model_id: str) -> Path: + return (Path.cwd() / "transpiled" / _slugify_model_artifact_name(model_id)).resolve() + + +def _graph_to_dict(graph) -> dict[str, object]: + return { + "meta": _serialize_json_compatible(graph.meta), + "inputs": list(graph.inputs), + "outputs": list(graph.outputs), + "constants": { + value_id: _serialize_json_compatible(constant) + for value_id, constant in graph.constants.items() + }, + "values": { + value_id: _serialize_json_compatible(value) + for value_id, value in graph.values.items() + }, + "nodes": [ + _serialize_json_compatible(graph.nodes[node_id]) + for node_id in graph.order + ], + } + + +def _write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def _component_artifact_name(prefix: str, component: str) -> str: + return f"{prefix}_{component}.json" + + +def _component_graphs_to_payload(component_graphs: dict[str, IRGraph]) -> dict[str, object]: + return { + component: _graph_to_dict(graph) + for component, graph in component_graphs.items() + } + + +def _binding_entries_by_node_id( + bindings: list[dict[str, object]], +) -> dict[int, dict[str, object]]: + result: dict[int, dict[str, object]] = {} + for binding in bindings: + try: + result[int(binding["node_id"])] = binding + except Exception: + continue + return result + + +def _binding_entries_by_value_id( + bindings: list[dict[str, object]], +) -> dict[str, dict[str, object]]: + result: dict[str, dict[str, object]] = {} + for binding in bindings: + value_id = binding.get("value_id") + if isinstance(value_id, str) and value_id: + result[value_id] = binding + return result + + +def _constant_precision_to_numpy_dtype(precision: int): + if int(precision) == int(Graph.FP16): + return np.float16 + if int(precision) == int(Graph.FP32): + return np.float32 + if int(precision) == int(Graph.INT8): + return np.int8 + raise ValueError(f"materialized graph constants must be FP16/FP32/INT8, got precision={precision}") + + +def _write_cactus_constant_tensor( + *, + output_path: Path, + value: object, + precision: int, +) -> None: + if isinstance(value, torch.Tensor): + array = value.detach().cpu().numpy() + elif isinstance(value, np.ndarray): + array = value + else: + raise TypeError(f"unsupported materialized constant type: {type(value).__name__}") + + dtype = _constant_precision_to_numpy_dtype(int(precision)) + array = np.ascontiguousarray(array.astype(dtype, copy=False)) + shape = list(array.shape) + if len(shape) > _CACTUS_EXTENDED_SHAPE_DIMS: + raise ValueError( + f"Cactus tensor files support at most rank {_CACTUS_EXTENDED_SHAPE_DIMS} constants, got shape={shape}" + ) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("wb") as handle: + data = array.reshape(-1) + data_bytes = int(data.nbytes) + flags = _CACTUS_FLAG_EXTENDED_SHAPE if len(shape) > 4 else 0 + header_size = _CACTUS_BASE_HEADER_SIZE + (32 if flags else 0) + handle.write(CACTUS_MAGIC) + handle.write(struct.pack(" int: + if array.dtype == np.float16: + return int(Graph.FP16) + if array.dtype in (np.float32, np.float64, np.bool_, np.int16, np.int32, np.int64, np.uint8): + return int(Graph.FP32) + if array.dtype == np.int8: + return int(Graph.INT8) + return int(Graph.FP32) + + +def _write_graph_binding_manifest( + *, + artifact_dir: Path, + filename: str, + model_id: str, + model_source: str, + task: str, + family: str, + inputs_metadata: dict[str, object], + transpiled_graph: TranspiledGraph, +) -> Path: + manifest_path = artifact_dir / filename + _write_json( + manifest_path, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(inputs_metadata), + "runtime_input_node_ids": [int(tensor.id) for tensor in transpiled_graph.runtime_inputs], + "output_node_ids": [int(tensor.id) for tensor in transpiled_graph.outputs], + "bound_constant_bindings": _serialize_json_compatible(transpiled_graph.bound_constant_bindings), + }, + ) + return manifest_path + + +def _write_component_bundle( + *, + artifact_dir: Path, + model_id: str, + model_source: str, + task: str, + family: str, + inputs_metadata: dict[str, object], + raw_component_graphs: dict[str, IRGraph], + optimized_component_graphs: dict[str, IRGraph], + transpiled_component_graphs: dict[str, TranspiledGraph] | None = None, + component_io_signatures: dict[str, dict[str, tuple[str, ...]]] | None = None, + graph_filename: str = "graph.cactus", +) -> Path: + bundle_dir = artifact_dir / "components" + component_order = [ + component + for component in ( + "audio_encoder", + "vision_encoder", + "lm_encoder", + "decoder_prefill_chunk", + "decoder", + "lm_encoder_step", + "decoder_step", + "unspecified", + ) + if component in raw_component_graphs or component in optimized_component_graphs + ] + extra_components = sorted( + component + for component in set(raw_component_graphs) | set(optimized_component_graphs) + if component not in component_order + ) + component_order.extend(extra_components) + + manifest_components: list[dict[str, object]] = [] + for component in component_order: + raw_graph = raw_component_graphs.get(component) + optimized_graph = optimized_component_graphs.get(component) + transpiled_graph = None if transpiled_component_graphs is None else transpiled_component_graphs.get(component) + component_dir = bundle_dir / component + raw_relpath = None + optimized_relpath = None + graph_relpath = None + if raw_graph is not None: + raw_relpath = Path(component) / "raw_ir.json" + _write_json( + bundle_dir / raw_relpath, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(inputs_metadata), + "graph": _graph_to_dict(raw_graph), + }, + ) + if optimized_graph is not None: + optimized_relpath = Path(component) / "optimized_ir.json" + _write_json( + bundle_dir / optimized_relpath, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(inputs_metadata), + "graph": _graph_to_dict(optimized_graph), + }, + ) + if transpiled_graph is not None: + graph_relpath = Path(component) / graph_filename + component_dir.mkdir(parents=True, exist_ok=True) + transpiled_graph.graph.save(bundle_dir / graph_relpath) + + materialized_constant_bindings: list[dict[str, object]] = [] + if transpiled_graph is not None: + existing_bindings = list(_serialize_json_compatible(transpiled_graph.bound_constant_bindings)) + existing_by_node_id = _binding_entries_by_node_id(existing_bindings) + existing_by_value_id = _binding_entries_by_value_id(existing_bindings) + constant_value_ids = dict(getattr(transpiled_graph, "bound_constant_value_ids", {})) + value_to_node_id = {value_id: node_id for node_id, value_id in constant_value_ids.items()} + constant_dir = component_dir / "bound_constants" + for constant_tensor in transpiled_graph.bound_constants: + node_id = int(constant_tensor.id) + if node_id in existing_by_node_id: + continue + constant_dir.mkdir(parents=True, exist_ok=True) + constant_filename = f"node_{node_id}.weights" + constant_relpath = Path(component) / "bound_constants" / constant_filename + _write_cactus_constant_tensor( + output_path=bundle_dir / constant_relpath, + value=constant_tensor.numpy(), + precision=int(constant_tensor.dtype), + ) + materialized_constant_bindings.append( + { + "node_id": node_id, + "value_id": str(constant_value_ids.get(node_id, f"materialized_constant_{node_id}")), + "path": str((bundle_dir / constant_relpath).relative_to(artifact_dir)), + "kind": "saved_constant", + "source_name": str(constant_value_ids.get(node_id, f"materialized_constant_{node_id}")), + "format": "tensor_io", + "precision": int(constant_tensor.dtype), + } + ) + + graph_constants = (optimized_graph or raw_graph).constants if (optimized_graph or raw_graph) is not None else {} + for value_id, constant_value in graph_constants.items(): + if value_id in existing_by_value_id: + continue + if any(binding.get("value_id") == value_id for binding in materialized_constant_bindings): + continue + if isinstance(constant_value, torch.Tensor): + constant_array = constant_value.detach().cpu().numpy().copy() + elif isinstance(constant_value, np.ndarray): + constant_array = np.ascontiguousarray(constant_value) + else: + continue + if value_id not in value_to_node_id: + continue + constant_dir.mkdir(parents=True, exist_ok=True) + safe_value_id = re.sub(r"[^A-Za-z0-9._-]+", "_", value_id).strip("._-") or "constant" + constant_filename = f"{safe_value_id}.weights" + constant_relpath = Path(component) / "bound_constants" / constant_filename + precision = _constant_precision_from_array(constant_array) + _write_cactus_constant_tensor( + output_path=bundle_dir / constant_relpath, + value=constant_array, + precision=precision, + ) + materialized_constant_bindings.append( + { + "node_id": int(value_to_node_id.get(value_id, -1)), + "value_id": str(value_id), + "path": str((bundle_dir / constant_relpath).relative_to(artifact_dir)), + "kind": "saved_constant", + "source_name": str(value_id), + "format": "tensor_io", + "precision": int(precision), + } + ) + + graph_for_signature = optimized_graph or raw_graph + if graph_for_signature is None: + continue + manifest_components.append( + { + "component": component, + "directory": str(component_dir.relative_to(artifact_dir)), + "raw_ir": None if raw_relpath is None else str((bundle_dir / raw_relpath).relative_to(artifact_dir)), + "optimized_ir": None if optimized_relpath is None else str((bundle_dir / optimized_relpath).relative_to(artifact_dir)), + "graph": None if graph_relpath is None else str((bundle_dir / graph_relpath).relative_to(artifact_dir)), + "inputs": list(graph_for_signature.inputs), + "outputs": list(graph_for_signature.outputs), + "logical_inputs": list((component_io_signatures or {}).get(component, {}).get("input_keys", ())), + "logical_outputs": list((component_io_signatures or {}).get(component, {}).get("output_keys", ())), + "node_count": len(graph_for_signature.order), + "weight_binding_count": _count_weight_bindings(graph_for_signature), + "runtime_input_node_ids": [] if transpiled_graph is None else [int(tensor.id) for tensor in transpiled_graph.runtime_inputs], + "output_node_ids": [] if transpiled_graph is None else [int(tensor.id) for tensor in transpiled_graph.outputs], + "cache_state_node_ids": [] if transpiled_graph is None else [ + { + "layer_key": str(layer_key), + "key": int(key_tensor.id), + "value": int(value_tensor.id), + } + for layer_key, key_tensor, value_tensor in getattr(transpiled_graph, "cache_state_tensors", []) + ], + "bound_constant_bindings": [] if transpiled_graph is None else ( + list(_serialize_json_compatible(transpiled_graph.bound_constant_bindings)) + + materialized_constant_bindings + ), + } + ) + + manifest_path = bundle_dir / "manifest.json" + _write_json( + manifest_path, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component_order": component_order, + "inputs": _serialize_json_compatible(inputs_metadata), + "components": manifest_components, + }, + ) + return manifest_path + + +def _named_tensor_store(prepared: PreparedInputs) -> dict[str, torch.Tensor]: + return { + name: tensor + for name, tensor in zip(prepared.names, prepared.tensors, strict=True) + } + + +def _aggregate_component_counts(component_graphs: dict[str, IRGraph]) -> dict[str, int]: + return { + component: len(graph.order) + for component, graph in sorted(component_graphs.items()) + } + + +def _aggregate_component_op_counts(component_graphs: dict[str, IRGraph]) -> Counter: + counter: Counter[str] = Counter() + for graph in component_graphs.values(): + counter.update(graph.nodes[node_id].op for node_id in graph.order) + return counter + + +def _count_component_weight_bindings(component_graphs: dict[str, IRGraph]) -> int: + return sum(_count_weight_bindings(graph) for graph in component_graphs.values()) + + +def _execute_gemma4_component_pipeline( + *, + component_graphs, + prepared: PreparedInputs, +) -> np.ndarray: + store, _ = execute_component_pipeline( + [component_graphs[name] for name in ("vision_encoder", "audio_encoder", "lm_encoder", "decoder")], + initial_store=_named_tensor_store(prepared), + ) + logits = store["logits"] + return np.asarray(logits, dtype=np.float32) + + +def _run_parakeet_tdt_component_decode( + *, + component_graphs, + model, + prepared: PreparedInputs, +) -> dict[str, object]: + store, _ = execute_component_pipeline( + [component_graphs["audio_encoder"]], + initial_store=_named_tensor_store(prepared), + ) + encoder_hidden_states = np.asarray(store["encoder_hidden_states"]) + batch_size = int(encoder_hidden_states.shape[0]) + if batch_size != 1: + raise ValueError("Parakeet TDT component decode currently expects batch size 1") + + hidden_dtype = prepared.tensors[0].dtype + initial_states = model.initial_decoder_state( + batch_size=batch_size, + device=torch.device("cpu"), + dtype=hidden_dtype, + ) + initial_state_arrays = tuple(state.detach().cpu().numpy() for state in initial_states) + decoder_component = component_graphs["decoder"] + + def _step( + frame: np.ndarray, + token_id: int, + state_values: tuple[np.ndarray, ...], + ) -> tuple[np.ndarray, tuple[np.ndarray, ...]]: + input_store: dict[str, object] = { + "encoder_frame": np.ascontiguousarray(frame), + "token_ids": np.full((batch_size,), token_id, dtype=np.int64), + } + for index in range(model.config.predictor_num_layers): + input_store[f"state_h_{index}"] = np.ascontiguousarray(state_values[index * 2]) + input_store[f"state_c_{index}"] = np.ascontiguousarray(state_values[index * 2 + 1]) + + runtime_inputs = [input_store[key] for key in decoder_component.input_keys] + decoder_component.transpiled_graph.set_inputs(runtime_inputs) + outputs = decoder_component.transpiled_graph.execute() + logits = outputs[0].numpy().astype(np.float32, copy=False) + next_states = tuple(output.numpy() for output in outputs[1:]) + return logits, next_states + + emitted = greedy_decode_parakeet_tdt_token_ids( + config=model.config, + encoder_hidden_states=encoder_hidden_states, + initial_states=initial_state_arrays, + step=_step, + ) + + return { + "token_ids": emitted, + "transcript": model.decode_token_ids(emitted), + "encoder_hidden_shape": list(encoder_hidden_states.shape), + } + + +def _run_component_pipeline_transpile( + *, + args, + task: str, + family: str, + model_source: str, + model, + prepared: PreparedInputs, + component_specs, + fusion_config: FusionConfig, + weights_dir: str | None, + artifact_dir: Path | None, + processor_or_tokenizer, + canonical, +) -> int: + print(f"model_id={args.model_id}") + print(f"model_source={model_source}") + print(f"task={task}") + print(f"adapter_family={family}") + print("adapter_module=component_pipeline") + print(f"input_names={','.join(prepared.names)}") + for name, tensor in zip(prepared.names, prepared.tensors, strict=True): + print(f"input_{name}_shape={list(tensor.shape)}") + if weights_dir: + print(f"weights_dir={weights_dir}") + + print("capture_begin=true", flush=True) + captured_components = {} + for spec in component_specs: + print(f"capture_component_begin={spec.component}", flush=True) + captured_components[spec.component] = capture_component_spec(spec, fusion_config=fusion_config) + print(f"capture_component_done={spec.component}", flush=True) + print("capture_done=true", flush=True) + + raw_component_graphs = { + name: captured.raw_ir_graph + for name, captured in captured_components.items() + } + optimized_component_graphs = { + name: captured.optimized_ir_graph + for name, captured in captured_components.items() + } + transpiled_component_graphs = { + name: captured.transpiled_graph + for name, captured in captured_components.items() + } + component_io_signatures = { + name: { + "input_keys": tuple(captured.input_keys), + "output_keys": tuple(captured.output_keys), + } + for name, captured in captured_components.items() + } + + raw_ir_nodes = sum(len(graph.order) for graph in raw_component_graphs.values()) + optimized_ir_nodes = sum(len(graph.order) for graph in optimized_component_graphs.values()) + binding_count = _count_component_weight_bindings(optimized_component_graphs) + raw_component_counts = _aggregate_component_counts(raw_component_graphs) + optimized_component_counts = _aggregate_component_counts(optimized_component_graphs) + op_counts = _aggregate_component_op_counts(optimized_component_graphs) + + print(f"raw_ir_nodes={raw_ir_nodes}") + print(f"optimized_ir_nodes={optimized_ir_nodes}") + print(f"weight_bindings={binding_count}") + if raw_component_counts: + print( + "raw_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(raw_component_counts.items())) + ) + if optimized_component_counts: + print( + "optimized_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(optimized_component_counts.items())) + ) + print( + "ops=" + f"attention:{op_counts.get('attention', 0)} " + f"conv1d:{op_counts.get('conv1d', 0)} " + f"conv2d:{op_counts.get('conv2d', 0)} " + f"batch_norm:{op_counts.get('batch_norm', 0)} " + f"layer_norm:{op_counts.get('layer_norm', 0)} " + f"rms_norm:{op_counts.get('rms_norm', 0)} " + f"rope:{op_counts.get('rope', 0)} " + f"linear:{op_counts.get('linear', 0)}" + ) + + if weights_dir and binding_count == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir}\n" + "\n" + "The weights folder exists, but none of the component graphs matched entries in weights_manifest.json.\n" + "\n" + f"Recommended fix:\n" + f" cactus convert {args.model_id} {weights_dir}\n" + ) + + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "component_order": [spec.component for spec in component_specs], + "components": _component_graphs_to_payload(raw_component_graphs), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "component_order": [spec.component for spec in component_specs], + "components": _component_graphs_to_payload(optimized_component_graphs), + }, + ) + for component, component_graph in raw_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("raw_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + for component, component_graph in optimized_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("optimized_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + for component in raw_component_graphs: + print(f"saved_raw_component_ir_{component}={artifact_dir / _component_artifact_name('raw_ir', component)}") + for component in optimized_component_graphs: + print( + f"saved_optimized_component_ir_{component}=" + f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" + ) + component_manifest_path = _write_component_bundle( + artifact_dir=artifact_dir, + model_id=args.model_id, + model_source=model_source, + task=task, + family=family, + inputs_metadata=prepared.metadata, + raw_component_graphs=raw_component_graphs, + optimized_component_graphs=optimized_component_graphs, + transpiled_component_graphs=transpiled_component_graphs, + component_io_signatures=component_io_signatures, + graph_filename=args.graph_filename, + ) + print(f"saved_component_bundle_manifest={component_manifest_path}") + for component in transpiled_component_graphs: + print( + f"saved_component_graph_{component}=" + f"{artifact_dir / 'components' / component / args.graph_filename}" + ) + print("note=saved split component graphs; use the component bundle manifest to rebind mmap weights/embeddings when loading") + + if args.skip_execute: + return 0 + + if task == "multimodal_causal_lm_logits": + print("execute_begin=true") + transpiled_output = _execute_gemma4_component_pipeline( + component_graphs=captured_components, + prepared=prepared, + ) + print("execute_done=true") + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + if args.skip_reference_compare: + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(transpiled_output.shape), + "raw_ir_nodes": raw_ir_nodes, + "optimized_ir_nodes": optimized_ir_nodes, + "weight_bindings": binding_count, + "reference_compare_skipped": True, + } + print(f"output_shape={list(transpiled_output.shape)}") + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"transpiled_next_token_id={transpiled_next}") + result_payload["transpiled_next_token_id"] = transpiled_next + if hasattr(tokenizer_like, "decode"): + transpiled_token = tokenizer_like.decode([transpiled_next]) + print(f"transpiled_next_token={transpiled_token!r}") + result_payload["transpiled_next_token"] = transpiled_token + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + print("reference_begin=true") + with torch.no_grad(): + reference_output = canonical.module(*prepared.tensors).detach().float().cpu().numpy() + print("reference_done=true") + max_abs_diff = float(np.max(np.abs(reference_output - transpiled_output))) + mean_abs_diff = float(np.mean(np.abs(reference_output - transpiled_output))) + hf_next = int(np.argmax(reference_output[0, -1])) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"hf_next_token_id={hf_next}") + print(f"transpiled_next_token_id={transpiled_next}") + print(f"logits_max_abs_diff={max_abs_diff:.6f}") + print(f"logits_mean_abs_diff={mean_abs_diff:.6f}") + if hasattr(tokenizer_like, "decode"): + print(f"hf_next_token={tokenizer_like.decode([hf_next])!r}") + print(f"transpiled_next_token={tokenizer_like.decode([transpiled_next])!r}") + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(reference_output.shape), + "raw_ir_nodes": raw_ir_nodes, + "optimized_ir_nodes": optimized_ir_nodes, + "weight_bindings": binding_count, + "max_abs_diff": max_abs_diff, + "mean_abs_diff": mean_abs_diff, + "hf_next_token_id": hf_next, + "transpiled_next_token_id": transpiled_next, + } + if hasattr(tokenizer_like, "decode"): + result_payload["hf_next_token"] = tokenizer_like.decode([hf_next]) + result_payload["transpiled_next_token"] = tokenizer_like.decode([transpiled_next]) + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + if task == "tdt_transcription": + print("execute_begin=true") + transpiled_decode = _run_parakeet_tdt_component_decode( + component_graphs=captured_components, + model=model, + prepared=prepared, + ) + print("execute_done=true") + print(f"transpiled_transcript={transpiled_decode['transcript']!r}") + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "raw_ir_nodes": raw_ir_nodes, + "optimized_ir_nodes": optimized_ir_nodes, + "weight_bindings": binding_count, + "transpiled_token_ids": transpiled_decode["token_ids"], + "transpiled_transcript": transpiled_decode["transcript"], + "encoder_hidden_shape": transpiled_decode["encoder_hidden_shape"], + } + if args.skip_reference_compare: + result_payload["reference_compare_skipped"] = True + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + print("reference_begin=true") + reference_token_ids = model.greedy_decode_token_ids(prepared.tensors[0]) + reference_transcript = model.decode_token_ids(reference_token_ids) + print("reference_done=true") + print(f"reference_transcript={reference_transcript!r}") + print(f"transcript_match={reference_transcript == transpiled_decode['transcript']}") + result_payload["reference_token_ids"] = reference_token_ids + result_payload["reference_transcript"] = reference_transcript + result_payload["transcript_match"] = bool(reference_transcript == transpiled_decode["transcript"]) + result_payload["token_id_match"] = bool(reference_token_ids == transpiled_decode["token_ids"]) + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + raise NotImplementedError(f"component pipeline execution is not implemented for task={task}") + + +def _parse_dtype(name: str) -> torch.dtype: + normalized = name.strip().lower() + mapping = { + "float16": torch.float16, + "fp16": torch.float16, + "float32": torch.float32, + "fp32": torch.float32, + "bfloat16": torch.bfloat16, + "bf16": torch.bfloat16, + } + if normalized not in mapping: + raise ValueError(f"unsupported torch dtype: {name}") + return mapping[normalized] + + +class TranspileWrapper(torch.nn.Module): + def __init__(self, adapter_module: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__() + self.adapter = adapter_module + self.weights_dir = weights_dir + + def forward(self, *bound_inputs: torch.Tensor) -> torch.Tensor: + return self.adapter(*bound_inputs) + + def get_transpile_metadata(self) -> dict[str, object]: + metadata: dict[str, object] = {} + provider = getattr(self.adapter, "get_transpile_metadata", None) + if callable(provider): + provided = provider() + if isinstance(provided, dict): + metadata.update(provided) + graph_meta: dict[str, object] = {} + base_graph = metadata.get("graph", {}) + if isinstance(base_graph, dict): + graph_meta.update(base_graph) + if self.weights_dir: + graph_meta["weights_dir"] = self.weights_dir + metadata["graph"] = graph_meta + return metadata + + +def _tie_lfm2_vl_lm_head_if_needed(model: torch.nn.Module) -> str | None: + if str(getattr(getattr(model, "config", None), "model_type", "") or "").lower() != "lfm2_vl": + return None + lm_head = getattr(model, "lm_head", None) + model_root = getattr(model, "model", None) + language_model = getattr(model_root, "language_model", None) + embed_tokens = getattr(language_model, "embed_tokens", None) + if not isinstance(lm_head, torch.nn.Linear) or not isinstance(embed_tokens, torch.nn.Embedding): + return None + if tuple(lm_head.weight.shape) != tuple(embed_tokens.weight.shape): + return None + if lm_head.weight.data_ptr() == embed_tokens.weight.data_ptr(): + return None + lm_head.weight = embed_tokens.weight + return "tied LFM2-VL lm_head.weight to language_model.embed_tokens.weight" + + +def _load_local_torch_state_dict(model_source: str) -> dict[str, torch.Tensor] | None: + root = Path(model_source) + if not root.exists() or not root.is_dir(): + return None + + safetensors_path = root / "model.safetensors" + if safetensors_path.exists(): + try: + from safetensors.torch import load_file # type: ignore + + return load_file(str(safetensors_path)) + except Exception: + return None + + pytorch_path = root / "pytorch_model.bin" + if pytorch_path.exists(): + try: + loaded = torch.load(str(pytorch_path), map_location="cpu", weights_only=True) + except TypeError: + loaded = torch.load(str(pytorch_path), map_location="cpu") + return loaded if isinstance(loaded, dict) else None + + return None + + +def _remap_gemma4_checkpoint_state_dict(state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + remapped: dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + if "audio_tower" in new_key: + new_key = re.sub(r"subsample_conv_projection\.layer(\d+)\.", r"subsample_conv_projection.conv_\1.", new_key) + new_key = new_key.replace("audio_tower.layers.", "audio_tower.conformer.") + new_key = new_key.replace(".feed_forward1.", ".ffw_layer_start.") + new_key = new_key.replace(".feed_forward2.", ".ffw_layer_end.") + new_key = re.sub(r"\.self_attn\.(q_proj|k_proj|v_proj)\.", r".attention.attn.\1.", new_key) + new_key = new_key.replace(".self_attn.per_dim_scale", ".attention.attn.per_dim_scale") + new_key = new_key.replace(".self_attn.relative_k_proj.", ".attention.attn.relative_position_embedding.pos_proj.") + new_key = new_key.replace(".self_attn.post.", ".attention.post.") + new_key = new_key.replace(".norm_pre_attn.", ".attention.pre_attn_norm.") + new_key = new_key.replace(".norm_post_attn.", ".attention.post_norm.") + new_key = new_key.replace(".norm_out.", ".norm.") + new_key = new_key.replace(".linear.weight", ".weight") + remapped[new_key] = value + if new_key.endswith(".attention.attn.per_dim_scale"): + remapped[new_key.replace(".per_dim_scale", ".per_dim_key_scale")] = value + if "lm_head.weight" not in remapped: + tied_embedding = remapped.get("model.language_model.embed_tokens.weight") + if tied_embedding is not None: + remapped["lm_head.weight"] = tied_embedding + return remapped + + +def _repair_gemma4_checkpoint_weights(model: torch.nn.Module, model_source: str) -> dict[str, object]: + multimodal_backbone = getattr(model, "model", model) + audio_tower = getattr(multimodal_backbone, "audio_tower", None) + # Older/local Gemma4 variants used Cactus-style module names such as + # audio_tower.conformer and bare clippable-linear weights. Current HF + # Gemma4 classes use audio_tower.layers and *.linear.weight, and + # from_pretrained already loads them correctly. Do not reload a legacy + # remap into the current layout, because that leaves real weights missing. + if not hasattr(audio_tower, "conformer"): + return {"applied": False, "reason": "current HF Gemma4 layout does not need legacy key remap"} + + raw_state_dict = _load_local_torch_state_dict(model_source) + if raw_state_dict is None: + return {"applied": False, "reason": "no local checkpoint state_dict"} + + remapped_state_dict = _remap_gemma4_checkpoint_state_dict(raw_state_dict) + load_result = model.load_state_dict(remapped_state_dict, strict=False) + return { + "applied": True, + "missing_keys": list(load_result.missing_keys), + "unexpected_keys": list(load_result.unexpected_keys), + } + + +def _load_config_json(model_id_or_path: str) -> dict[str, object]: + local_snapshot = _resolve_local_snapshot(model_id_or_path) + config_source = Path(local_snapshot) / "config.json" if local_snapshot else None + if config_source is not None and config_source.exists(): + return json.loads(config_source.read_text()) + explicit = Path(model_id_or_path) / "config.json" + if explicit.exists(): + return json.loads(explicit.read_text()) + return {} + + +def _load_optional_json(model_id_or_path: str, filename: str) -> dict[str, object]: + local_snapshot = _resolve_local_snapshot(model_id_or_path) + candidate = Path(local_snapshot) / filename if local_snapshot else None + if candidate is not None and candidate.exists(): + return json.loads(candidate.read_text()) + explicit = Path(model_id_or_path) / filename + if explicit.exists(): + return json.loads(explicit.read_text()) + return {} + + +def _load_gemma4_tokenizer_fallback(source_candidates: list[str]) -> object | None: + try: + from transformers import PreTrainedTokenizerFast # type: ignore + except Exception: + return None + + for source in source_candidates: + root = Path(source) + if not root.exists() or not root.is_dir(): + continue + tokenizer_json = root / "tokenizer.json" + tokenizer_config_path = root / "tokenizer_config.json" + if not tokenizer_json.exists() or not tokenizer_config_path.exists(): + continue + + tokenizer_config = json.loads(tokenizer_config_path.read_text()) + tokenizer = PreTrainedTokenizerFast( + tokenizer_file=str(tokenizer_json), + bos_token=tokenizer_config.get("bos_token"), + eos_token=tokenizer_config.get("eos_token"), + unk_token=tokenizer_config.get("unk_token"), + pad_token=tokenizer_config.get("pad_token"), + mask_token=tokenizer_config.get("mask_token"), + padding_side=str(tokenizer_config.get("padding_side", "right")), + additional_special_tokens=list(tokenizer_config.get("extra_special_tokens", []) or []), + ) + model_max_length = tokenizer_config.get("model_max_length") + if isinstance(model_max_length, int): + tokenizer.model_max_length = model_max_length + + chat_template_path = root / "chat_template.jinja" + if chat_template_path.exists(): + tokenizer.chat_template = chat_template_path.read_text() + + for token_attr in ( + "image_token", + "audio_token", + "boi_token", + "eoi_token", + "boa_token", + "eoa_token", + ): + token_value = tokenizer_config.get(token_attr) + if isinstance(token_value, str): + setattr(tokenizer, token_attr, token_value) + setattr(tokenizer, f"{token_attr}_id", tokenizer.convert_tokens_to_ids(token_value)) + return tokenizer + + return None + + +def _load_gemma4_processor_fallback( + *, + source_candidates: list[str], + common_kwargs: dict[str, object], +) -> object | None: + try: + from transformers.models.gemma4.feature_extraction_gemma4 import Gemma4AudioFeatureExtractor # type: ignore + from transformers.models.gemma4.image_processing_gemma4 import Gemma4ImageProcessor # type: ignore + from transformers.models.gemma4.processing_gemma4 import Gemma4Processor # type: ignore + except Exception: + return None + + tokenizer = None + processor_config: dict[str, object] = {} + for source in source_candidates: + try: + try: + from transformers import AutoTokenizer # type: ignore + + tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) + except Exception: + tokenizer = _load_gemma4_tokenizer_fallback([source]) + processor_config = _load_optional_json(source, "processor_config.json") + if tokenizer is not None and processor_config: + break + except Exception: + continue + if tokenizer is None or not processor_config: + return None + + feature_config = dict(processor_config.get("feature_extractor", {}) or {}) + image_config = dict(processor_config.get("image_processor", {}) or {}) + feature_config.pop("feature_extractor_type", None) + image_config.pop("image_processor_type", None) + + feature_extractor = Gemma4AudioFeatureExtractor(**feature_config) + image_processor = Gemma4ImageProcessor(**image_config) + + processor_kwargs = {} + for key in ("image_seq_length", "audio_seq_length", "audio_ms_per_token"): + if key in processor_config: + processor_kwargs[key] = processor_config[key] + + return Gemma4Processor( + feature_extractor=feature_extractor, + image_processor=image_processor, + tokenizer=tokenizer, + **processor_kwargs, + ) + + +def _infer_task_from_config(model_id_or_path: str) -> str: + config = _load_config_json(model_id_or_path) + plan = infer_component_plan_from_config(config, model_id=model_id_or_path) + if plan is not None: + return plan.task + architectures = [str(value) for value in config.get("architectures", []) if isinstance(value, str)] + model_type = str(config.get("model_type", "") or "").lower() + decoding_cfg = config.get("decoding") + if isinstance(decoding_cfg, dict) and str(decoding_cfg.get("model_type", "") or "").lower() == "tdt": + return "tdt_transcription" + loss_cfg = config.get("loss") + if isinstance(loss_cfg, dict) and str(loss_cfg.get("loss_name", "") or "").lower() == "tdt": + return "tdt_transcription" + + if any("CausalLM" in value for value in architectures): + return "causal_lm_logits" + if any("CTC" in value for value in architectures): + return "ctc_logits" + if model_type == "whisper": + return "seq2seq_transcription" + if any("ConditionalGeneration" in value and "Whisper" in value for value in architectures): + return "seq2seq_transcription" + + lowered_id = model_id_or_path.lower() + if "parakeet-tdt" in lowered_id: + return "tdt_transcription" + if "whisper" in lowered_id: + return "seq2seq_transcription" + if "ctc" in lowered_id: + return "ctc_logits" + if any(token in lowered_id for token in ("qwen", "gemma", "llama", "mistral", "lfm")): + return "causal_lm_logits" + + raise RuntimeError( + f"Could not infer transpile task for {model_id_or_path}.\n" + "\n" + "Pass one explicitly with --task, for example:\n" + " --task causal_lm_logits\n" + " --task tdt_transcription\n" + " --task ctc_logits\n" + " --task encoder_hidden_states\n" + " --task seq2seq_transcription\n" + ) + + +def _resolve_audio_sample_rate(processor: object) -> int: + for attr_name in ("feature_extractor", "tokenizer"): + child = getattr(processor, attr_name, None) + sample_rate = getattr(child, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + sample_rate = getattr(processor, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + return 16000 + + +def _infer_fallback_audio_input_names(config: dict[str, object], task: str) -> tuple[str, ...]: + model_type = str(config.get("model_type", "") or "").lower() + if model_type == "whisper": + return ("input_features",) + if task == "seq2seq_transcription": + return ("input_features",) + if task == "encoder_hidden_states": + return ("input_features",) + if task == "tdt_transcription": + return ("input_features",) + audio_cfg = config.get("audio_config") + if isinstance(audio_cfg, dict) and any(key in audio_cfg for key in ("features", "input_feat_size", "num_mel_bins")): + return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) + encoder_cfg = config.get("encoder") + if isinstance(encoder_cfg, dict) and any(key in encoder_cfg for key in ("feat_in", "num_mel_bins")): + return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) + return ("input_values", "attention_mask") if task == "ctc_logits" else ("input_values",) + + +def _resolve_encoder_module(model: torch.nn.Module) -> torch.nn.Module | None: + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder = get_encoder() + if isinstance(encoder, torch.nn.Module): + return encoder + encoder = getattr(model, "encoder", None) + if isinstance(encoder, torch.nn.Module): + return encoder + model_attr = getattr(model, "model", None) + if model_attr is not None: + encoder = getattr(model_attr, "encoder", None) + if isinstance(encoder, torch.nn.Module): + return encoder + return None + + +def _infer_expected_input_feature_frames(model: torch.nn.Module) -> int | None: + config = getattr(model, "config", None) + max_source_positions = getattr(config, "max_source_positions", None) + if not isinstance(max_source_positions, int) or max_source_positions <= 0: + return None + + encoder = _resolve_encoder_module(model) + if encoder is None: + return None + + stride_product = 1 + found_conv = False + for child in encoder.children(): + if isinstance(child, torch.nn.Conv1d): + stride = child.stride[0] if isinstance(child.stride, tuple) else child.stride + stride_product *= int(stride) + found_conv = True + + if not found_conv: + return None + return int(max_source_positions) * stride_product + + +def _prepare_fallback_audio_inputs( + *, + input_names: tuple[str, ...], + config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + task: str, + audio_file: str, + torch_dtype: torch.dtype, +) -> PreparedInputs: + if not input_names: + input_names = _infer_fallback_audio_input_names(config, task) + preprocessor_root = config.get("preprocessor") + preprocessor_root = preprocessor_root if isinstance(preprocessor_root, dict) else {} + encoder_root = config.get("encoder") + encoder_root = encoder_root if isinstance(encoder_root, dict) else {} + + sample_rate = int( + preprocessor_config.get( + "sampling_rate", + preprocessor_root.get("sample_rate", config.get("sampling_rate", 16000)), + ) + or 16000 + ) + num_mels = int( + preprocessor_config.get( + "feature_size", + preprocessor_root.get( + "features", + encoder_root.get("feat_in", config.get("num_mel_bins", config.get("feature_size", 80))), + ), + ) + or 80 + ) + encoder_cfg = config.get("encoder_config") + if isinstance(encoder_cfg, dict): + num_mels = int(encoder_cfg.get("num_mel_bins", encoder_cfg.get("feat_in", num_mels)) or num_mels) + raw_hop_length = preprocessor_config.get( + "hop_length", + preprocessor_root.get("window_stride", config.get("hop_length", 160)), + ) + if raw_hop_length is None: + raw_hop_length = 160 + if isinstance(raw_hop_length, float) and raw_hop_length > 0.0 and raw_hop_length < 1.0: + hop_length = max(1, int(round(float(sample_rate) * raw_hop_length))) + else: + hop_length = int(raw_hop_length) + + n_fft = int(preprocessor_config.get("n_fft", config.get("n_fft", preprocessor_root.get("n_fft", 400))) or 400) + raw_frame_length = preprocessor_config.get( + "win_length", + preprocessor_config.get( + "frame_length", + preprocessor_root.get("window_size", preprocessor_root.get("n_window_size", config.get("frame_length", n_fft))), + ), + ) + if raw_frame_length is None: + raw_frame_length = n_fft + if isinstance(raw_frame_length, float) and raw_frame_length > 0.0 and raw_frame_length < 1.0: + frame_length = max(1, int(round(float(sample_rate) * raw_frame_length))) + else: + frame_length = int(raw_frame_length) + preemphasis = preprocessor_config.get("preemphasis", preprocessor_root.get("preemph")) + if preemphasis is not None: + preemphasis = float(preemphasis) + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + + tensors: list[torch.Tensor] = [] + if input_names and input_names[0] == "input_features": + model_type = str(config.get("model_type", "") or "").lower() + is_parakeet_like = ( + task == "tdt_transcription" + or "parakeet" in model_type + or "parakeet" in type(model).__module__.lower() + ) + if is_parakeet_like: + expected_frames = _infer_expected_input_feature_frames(model) + parakeet_features, _ = prepare_parakeet_tdt_audio_features( + audio_file, + expected_frames=expected_frames, + expected_mels=num_mels, + torch_dtype=torch_dtype, + ) + tensors.append(parakeet_features) + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "native_parakeet_audio_preprocessor": True, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + is_whisper_like = ( + str(config.get("model_type", "") or "").lower() == "whisper" + or "transformers.models.whisper." in type(model).__module__ + ) + if is_whisper_like: + expected_frames = _infer_expected_input_feature_frames(model) or 3000 + try: + whisper_features, active_frames = prepare_cactus_audio_features( + audio_file, + model_type="whisper", + expected_frames=expected_frames, + expected_mels=num_mels, + torch_dtype=torch_dtype, + layout="mels_frames", + ) + tensors.append(whisper_features) + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "native_cactus_audio_preprocessor": True, + "active_feature_frames": active_frames, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + except Exception: + pass + features, feature_length = _generic_log_mel_features( + waveform, + sample_rate=sample_rate, + num_mels=num_mels, + n_fft=n_fft, + hop_length=hop_length, + frame_length=frame_length, + preemphasis=preemphasis, + ) + expected_frames = _infer_expected_input_feature_frames(model) + attention_mask: np.ndarray | None = None + target_frames = feature_length + if isinstance(expected_frames, int) and expected_frames > 0: + target_frames = expected_frames + + active_frames = min(feature_length, target_frames) + features = features[:active_frames, :] + if target_frames > active_frames: + pad_width = target_frames - active_frames + features = np.pad(features, ((0, pad_width), (0, 0)), mode="constant") + attention_mask = np.zeros((target_frames,), dtype=np.bool_) + attention_mask[:active_frames] = True + + if is_whisper_like: + features = np.ascontiguousarray(features.T) + + tensors.append(torch.from_numpy(features).unsqueeze(0).to(dtype=torch_dtype)) + if attention_mask is not None and len(input_names) > 1 and input_names[1] == "attention_mask": + tensors.append(torch.from_numpy(attention_mask).unsqueeze(0)) + else: + input_values = torch.from_numpy(waveform).unsqueeze(0).to(dtype=torch_dtype) + tensors.append(input_values) + if len(input_names) > 1 and input_names[1] == "attention_mask": + tensors.append(torch.ones_like(input_values, dtype=torch.float32)) + + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + + +def _prepare_audio_inputs( + processor: object | None, + *, + input_names: tuple[str, ...], + config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + task: str, + audio_file: str, + torch_dtype: torch.dtype, +) -> PreparedInputs: + if processor is None: + return _prepare_fallback_audio_inputs( + input_names=input_names, + config=config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + + model_type = str(config.get("model_type", "") or "").lower() + if model_type == "whisper" or "transformers.models.whisper." in type(model).__module__: + return _prepare_fallback_audio_inputs( + input_names=input_names, + config=config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + + sample_rate = _resolve_audio_sample_rate(processor) + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + batch = processor( + waveform, + sampling_rate=sample_rate, + return_tensors="pt", + ) + + preferred_keys = tuple(input_names) + tuple( + key for key in ("input_features", "input_values", "attention_mask") if key not in input_names + ) + tensor_keys = [key for key, value in batch.items() if isinstance(value, torch.Tensor)] + ordered_keys = [key for key in preferred_keys if key in tensor_keys] + ordered_keys.extend(key for key in tensor_keys if key not in ordered_keys) + if not ordered_keys: + raise RuntimeError(f"processor did not return tensor inputs for audio file: {audio_file}") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": False, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + }, + ) + + +def _contains_token_subsequence(token_ids: list[int], subsequence: list[int]) -> bool: + if not subsequence: + return False + width = len(subsequence) + return any(token_ids[index : index + width] == subsequence for index in range(len(token_ids) - width + 1)) + + +def _resolve_whisper_decoder_prompt_token_ids( + tokenizer_or_processor: object, + *, + prompt: str | None, + decoder_start_token_id: int | None, + forced_decoder_ids: list[list[int]] | tuple[tuple[int, int], ...] | None = None, +) -> list[int]: + tokenizer = getattr(tokenizer_or_processor, "tokenizer", tokenizer_or_processor) + encode = getattr(tokenizer, "encode", None) + if not callable(encode): + encode = None + + normalized_prompt = (prompt or "").strip() + if normalized_prompt == _DEFAULT_CAUSAL_PROMPT: + normalized_prompt = "" + + prompt_token_ids: list[int] = [] + if decoder_start_token_id is not None: + prompt_token_ids.append(int(decoder_start_token_id)) + + normalized_forced_ids: list[tuple[int, int]] = [] + if forced_decoder_ids is not None: + for item in forced_decoder_ids: + if ( + isinstance(item, (list, tuple)) + and len(item) == 2 + and isinstance(item[0], int) + and isinstance(item[1], int) + ): + normalized_forced_ids.append((int(item[0]), int(item[1]))) + normalized_forced_ids.sort(key=lambda pair: pair[0]) + + next_position = 1 + for position, token_id in normalized_forced_ids: + if position != next_position: + break + prompt_token_ids.append(int(token_id)) + next_position += 1 + + if normalized_prompt: + if encode is None: + raise RuntimeError("Whisper transpile requires a tokenizer to encode a non-empty prompt") + try: + encoded_prompt = encode(normalized_prompt, add_special_tokens=False) + except TypeError: + encoded_prompt = encode(normalized_prompt) + prompt_token_ids.extend(int(value) for value in encoded_prompt) + + if not prompt_token_ids: + if encode is None: + raise RuntimeError("Whisper transpile requires a tokenizer or decoder_start_token_id") + try: + prompt_token_ids = [int(value) for value in encode("<|startoftranscript|>", add_special_tokens=False)] + except TypeError: + prompt_token_ids = [int(value) for value in encode("<|startoftranscript|>")] + if not prompt_token_ids and decoder_start_token_id is not None: + prompt_token_ids = [int(decoder_start_token_id)] + + return prompt_token_ids + + +def _augment_whisper_seq2seq_metadata( + prepared: PreparedInputs, + *, + tokenizer_or_processor: object, + model: torch.nn.Module, + prompt: str | None, + max_new_tokens: int, +) -> PreparedInputs: + config = getattr(model, "config", None) + decoder_start_token_id = getattr(config, "decoder_start_token_id", None) + eos_token_id = getattr(config, "eos_token_id", None) + pad_token_id = getattr(config, "pad_token_id", None) + max_target_positions = int(getattr(config, "max_target_positions", 0) or 0) + suppress_tokens = [int(value) for value in (getattr(config, "suppress_tokens", None) or [])] + begin_suppress_tokens = [int(value) for value in (getattr(config, "begin_suppress_tokens", None) or [])] + + decoder_input_ids = _resolve_whisper_decoder_prompt_token_ids( + tokenizer_or_processor, + prompt=prompt, + decoder_start_token_id=int(decoder_start_token_id) if isinstance(decoder_start_token_id, int) else None, + forced_decoder_ids=getattr(config, "forced_decoder_ids", None), + ) + if not decoder_input_ids: + raise RuntimeError("Whisper transpile could not resolve decoder prompt token ids") + + target_token_count = len(decoder_input_ids) + max(1, int(max_new_tokens)) + if max_target_positions > 0: + target_token_count = min(target_token_count, max_target_positions) + target_token_count = max(target_token_count, len(decoder_input_ids)) + + metadata = dict(prepared.metadata) + metadata.update( + { + "decoder_input_ids": [int(value) for value in decoder_input_ids], + "decoder_start_token_id": None if decoder_start_token_id is None else int(decoder_start_token_id), + "eos_token_id": None if eos_token_id is None else int(eos_token_id), + "pad_token_id": None if pad_token_id is None else int(pad_token_id), + "target_token_count": int(target_token_count), + "max_target_positions": int(max_target_positions), + "suppress_tokens": suppress_tokens, + "begin_suppress_tokens": begin_suppress_tokens, + } + ) + return PreparedInputs( + names=prepared.names, + tensors=prepared.tensors, + metadata=metadata, + ) + + +def _tokenize_text_prompt( + tokenizer: object, + prompt: str, + *, + enable_thinking_if_supported: bool = False, +) -> torch.Tensor: + apply_chat_template = getattr(tokenizer, "apply_chat_template", None) + if callable(apply_chat_template): + try: + encoded = apply_chat_template( + [{"role": "user", "content": prompt}], + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + enable_thinking=bool(enable_thinking_if_supported), + ) + if isinstance(encoded, torch.Tensor) and encoded.ndim == 2: + return encoded.to(dtype=torch.long) + if isinstance(encoded, Mapping): + input_ids = encoded.get("input_ids") + if isinstance(input_ids, torch.Tensor) and input_ids.ndim == 2: + return input_ids.to(dtype=torch.long) + except Exception: + pass + encoded = tokenizer(prompt, return_tensors="pt") + return encoded["input_ids"].to(dtype=torch.long) + + +def _resolve_text_padding_token_id(tokenizer: object | None) -> int: + for attr_name in ("pad_token_id", "eos_token_id", "bos_token_id"): + token_id = getattr(tokenizer, attr_name, None) if tokenizer is not None else None + if isinstance(token_id, int) and token_id >= 0: + return int(token_id) + return 0 + + +def _resolve_graph_safe_text_padding_token_id( + tokenizer: object | None, + prompt_input_ids: torch.Tensor, +) -> int: + padding_token_id = _resolve_text_padding_token_id(tokenizer) + if padding_token_id <= 60000: + return padding_token_id + + used_token_ids = {int(value) for value in prompt_input_ids.detach().cpu().reshape(-1).tolist()} + vocab_size = getattr(tokenizer, "vocab_size", None) + upper_bound = int(vocab_size) if isinstance(vocab_size, int) and vocab_size > 0 else 2048 + # v2 compare fallbacks currently legalize scalar comparisons through FP16. + # A large HF pad token such as Qwen's 151643 overflows that path, so choose + # a small valid token ID that is absent from the prompt and mask it out. + for candidate in range(min(upper_bound, 2048)): + if candidate not in used_token_ids: + return int(candidate) + return 0 + + +def _prepare_text_inputs( + tokenizer: object, + *, + prompt: str, + input_ids_text: str | None, + max_new_tokens: int, + enable_thinking_if_supported: bool = False, +) -> PreparedInputs: + if input_ids_text: + token_ids = [int(part.strip()) for part in input_ids_text.split(",") if part.strip()] + if not token_ids: + raise ValueError("--input-ids was provided but no ids were parsed") + prompt_input_ids = torch.tensor([token_ids], dtype=torch.long) + else: + prompt_input_ids = _tokenize_text_prompt( + tokenizer, + prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + if prompt_input_ids.ndim != 2 or int(prompt_input_ids.shape[0]) != 1: + raise ValueError( + "causal_lm_logits transpile currently expects prompt input ids with shape [1, T], " + f"got {tuple(int(dim) for dim in prompt_input_ids.shape)}" + ) + if int(max_new_tokens) < 0: + raise ValueError("--max-new-tokens must be non-negative") + + prompt_token_count = int(prompt_input_ids.shape[1]) + target_token_count = prompt_token_count + int(max_new_tokens) + padding_token_id = _resolve_graph_safe_text_padding_token_id(tokenizer, prompt_input_ids) + if target_token_count > prompt_token_count: + input_ids = torch.full((1, target_token_count), padding_token_id, dtype=torch.long) + input_ids[:, :prompt_token_count] = prompt_input_ids + else: + input_ids = prompt_input_ids + return PreparedInputs( + names=("input_ids",), + tensors=(input_ids,), + metadata={ + "prompt": prompt, + "prompt_input_ids": prompt_input_ids.tolist(), + "input_ids": input_ids.tolist(), + "prompt_token_count": prompt_token_count, + "target_token_count": target_token_count, + "max_new_tokens": int(max_new_tokens), + "padding_token_id": int(padding_token_id), + "enable_thinking": bool(enable_thinking_if_supported), + }, + ) + + +def _add_multimodal_generation_headroom( + prepared: PreparedInputs, + *, + tokenizer: object | None, + max_new_tokens: int, + min_context_tokens: int | None = None, +) -> PreparedInputs: + requested = max(0, int(max_new_tokens)) + if "input_ids" not in prepared.names: + return prepared + + tensor_by_name = dict(zip(prepared.names, prepared.tensors, strict=True)) + input_ids = tensor_by_name.get("input_ids") + if not isinstance(input_ids, torch.Tensor) or input_ids.ndim != 2: + return prepared + + prompt_token_count = int(input_ids.shape[1]) + min_context = max(0, int(min_context_tokens or 0)) + target_token_count = max(prompt_token_count + requested, min_context) + if target_token_count <= prompt_token_count: + return prepared + padding_token_id = _resolve_graph_safe_text_padding_token_id(tokenizer, input_ids) + + padded_tensors: list[torch.Tensor] = [] + for name, tensor in zip(prepared.names, prepared.tensors, strict=True): + if ( + name not in {"input_ids", "attention_mask", "token_type_ids"} + or tensor.ndim != 2 + or int(tensor.shape[0]) != 1 + or int(tensor.shape[1]) != prompt_token_count + ): + padded_tensors.append(tensor) + continue + pad_value = int(padding_token_id) if name == "input_ids" else 0 + padded = torch.full( + (1, target_token_count), + pad_value, + dtype=tensor.dtype, + device=tensor.device, + ) + padded[:, :prompt_token_count] = tensor + padded_tensors.append(padded) + + metadata = dict(prepared.metadata) + metadata["prompt_token_count"] = prompt_token_count + metadata["target_token_count"] = target_token_count + metadata["max_new_tokens"] = max(requested, target_token_count - prompt_token_count) + metadata["padding_token_id"] = int(padding_token_id) + input_shapes = dict(metadata.get("input_shapes") or {}) + for name, tensor in zip(prepared.names, padded_tensors, strict=True): + input_shapes[name] = [int(dim) for dim in tensor.shape] + metadata["input_shapes"] = input_shapes + return PreparedInputs( + names=prepared.names, + tensors=tuple(padded_tensors), + metadata=metadata, + ) + + +def _multimodal_context_token_floor(model_type: str = "") -> int: + default_context = multimodal_context_tokens_for_model_type( + model_type, + _DEFAULT_MULTIMODAL_CONTEXT_TOKENS, + ) + raw = os.environ.get( + "CACTUS_TRANSPILER_MULTIMODAL_CONTEXT_TOKENS", + str(default_context), + ) + try: + return max(0, int(raw)) + except (TypeError, ValueError): + return default_context + + +_GEMMA4_MULTIMODAL_INPUT_ORDER = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", +) + +_LFM2_VL_MULTIMODAL_INPUT_ORDER = ( + "input_ids", + "attention_mask", + "pixel_values", + "spatial_shapes", + "pixel_attention_mask", +) + + +def _normalize_multimodal_prompt( + prompt: str, + *, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, +) -> str: + normalized = prompt.strip() + prefixes: list[str] = [] + + if image_token and num_images > 0: + image_count = normalized.count(image_token) + if image_count < num_images: + prefixes.append(" ".join(image_token for _ in range(num_images - image_count))) + if audio_token and num_audio_segments > 0: + audio_count = normalized.count(audio_token) + if audio_count < num_audio_segments: + prefixes.append(" ".join(audio_token for _ in range(num_audio_segments - audio_count))) + + if prefixes: + prefix = "\n".join(part for part in prefixes if part) + if normalized: + return f"{prefix}\n{normalized}" + return prefix + return normalized + + +def _build_gemma4_chat_prompt( + *, + prompt: str, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> str: + result = "" + normalized_system = system_prompt.strip() + if enable_thinking_if_supported or normalized_system: + result += "<|turn>system\n" + if enable_thinking_if_supported: + result += "<|think|>" + result += normalized_system + result += "\n" + + result += "<|turn>user\n" + if image_token and num_images > 0: + for _ in range(num_images): + # Gemma4Processor expands the raw placeholder into the native + # <|image> ... soft-token region. Supplying wrappers here + # double-wraps the prompt and shifts the multimodal prefill. + result += f"\n\n{image_token}\n\n" + result += prompt + if audio_token and num_audio_segments > 0: + result += "".join( + audio_token + for _ in range(num_audio_segments) + ) + result += "\n" + result += "<|turn>model\n" + return result + + +def _gemma4_split_cactus_newline_token_merges(batch: object) -> None: + """Match native Cactus Gemma4 tokenization for newline runs. + + The HF fast tokenizer merges ``\n\n`` and ``\n\n\n`` into dedicated tokens + (108 and 109). Cactus' converted BPE tokenizer emits repeated newline token + 107 instead. Gemma4's multimodal prefill is sensitive enough that this one + token of drift changes the first generated token, so normalize only this + Gemma4-specific quirk after the processor has expanded image/audio regions. + """ + + input_ids = batch.get("input_ids") if hasattr(batch, "get") else None + if not isinstance(input_ids, torch.Tensor) or input_ids.ndim != 2: + return + + expansions = { + 108: (107, 107), + 109: (107, 107, 107), + } + if not any(int(token) in expansions for token in input_ids.reshape(-1).tolist()): + return + + lengths: list[list[int]] = [] + max_len = 0 + for row in input_ids.detach().cpu().tolist(): + row_lengths = [len(expansions.get(int(token), (int(token),))) for token in row] + lengths.append(row_lengths) + max_len = max(max_len, sum(row_lengths)) + + for key in ("input_ids", "attention_mask", "token_type_ids"): + value = batch.get(key) if hasattr(batch, "get") else None + if not isinstance(value, torch.Tensor) or value.ndim != 2: + continue + pad_value = 0 + expanded = torch.full( + (value.shape[0], max_len), + pad_value, + dtype=value.dtype, + device=value.device, + ) + for row_idx, row in enumerate(value.detach().cpu().tolist()): + out: list[int] = [] + for token_idx, item in enumerate(row): + if key == "input_ids": + out.extend(expansions.get(int(item), (int(item),))) + else: + out.extend([int(item)] * lengths[row_idx][token_idx]) + expanded[row_idx, : len(out)] = torch.tensor(out, dtype=value.dtype, device=value.device) + batch[key] = expanded + + +def _build_gemma4_chat_messages( + *, + prompt: str, + num_images: int, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> list[dict[str, object]]: + messages: list[dict[str, object]] = [] + normalized_system = system_prompt.strip() + if normalized_system or enable_thinking_if_supported: + system_text = normalized_system + if enable_thinking_if_supported: + system_text = f"<|think|>{system_text}" + messages.append({"role": "system", "content": system_text}) + + user_content: list[dict[str, str]] = [] + for _ in range(num_images): + user_content.append({"type": "image"}) + user_content.append({"type": "text", "text": prompt.strip()}) + for _ in range(num_audio_segments): + user_content.append({"type": "audio"}) + messages.append({"role": "user", "content": user_content}) + return messages + + +def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: + if not image_files: + return [] + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for --image-file: {exc}") from exc + + images: list[object] = [] + for image_file in image_files: + path = Path(image_file).resolve() + if not path.exists(): + raise RuntimeError(f"image_file does not exist: {path}") + with Image.open(path) as image: + images.append(resize_static_image(image.convert("RGB")).copy()) + return images + + +def _get_processor_image_attr(processor: object, name: str, default: object) -> object: + image_processor = getattr(processor, "image_processor", None) + if image_processor is not None and hasattr(image_processor, name): + return getattr(image_processor, name) + if isinstance(image_processor, dict) and name in image_processor: + return image_processor[name] + return default + + +def _prepare_gemma4_native_image_tensors( + processor: object, + image_files: tuple[str, ...], +) -> tuple[torch.Tensor, torch.Tensor] | None: + if not image_files: + return None + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for Gemma4 native image preprocessing: {exc}") from exc + + patch_size = int(_get_processor_image_attr(processor, "patch_size", 16)) + pooling_kernel_size = int(_get_processor_image_attr(processor, "pooling_kernel_size", 3)) + max_soft_tokens = int(_get_processor_image_attr(processor, "max_soft_tokens", 280)) + rescale_factor = float(_get_processor_image_attr(processor, "rescale_factor", 1.0 / 255.0)) + max_patches = max_soft_tokens * pooling_kernel_size * pooling_kernel_size + side_multiple = pooling_kernel_size * patch_size + patch_dim = 3 * patch_size * patch_size + if patch_size <= 0 or pooling_kernel_size <= 0 or max_patches <= 0: + return None + + try: + resample_bilinear = Image.Resampling.BILINEAR + except AttributeError: # pragma: no cover + resample_bilinear = Image.BILINEAR + + pixel_batches: list[np.ndarray] = [] + position_batches: list[np.ndarray] = [] + for image_file in image_files: + path = Path(image_file).resolve() + with Image.open(path) as image: + rgb = image.convert("RGB") + width, height = rgb.size + target_pixels = float(max_patches * patch_size * patch_size) + factor = float(np.sqrt(target_pixels / max(1.0, float(width * height)))) + target_h = int(np.floor(factor * height / side_multiple)) * side_multiple + target_w = int(np.floor(factor * width / side_multiple)) * side_multiple + if target_h == 0: + target_h = side_multiple + if target_w == 0: + target_w = side_multiple + if (target_w, target_h) != rgb.size: + rgb = rgb.resize((target_w, target_h), resample=resample_bilinear) + array = np.asarray(rgb, dtype=np.float32) * rescale_factor + + patch_h = target_h // patch_size + patch_w = target_w // patch_size + num_patches = patch_h * patch_w + if num_patches > max_patches: + raise RuntimeError( + f"Gemma4 native image preprocessing produced {num_patches} patches, " + f"but max_patches={max_patches}" + ) + chw = np.transpose(array, (2, 0, 1)) + patches = ( + chw.reshape(3, patch_h, patch_size, patch_w, patch_size) + .transpose(1, 3, 2, 4, 0) + .reshape(num_patches, patch_dim) + ) + + padded_patches = np.zeros((max_patches, patch_dim), dtype=np.float32) + padded_patches[:num_patches] = patches + positions = np.full((max_patches, 2), -1, dtype=np.int64) + valid_positions = np.zeros((num_patches, 2), dtype=np.int64) + for patch_y in range(patch_h): + row_start = patch_y * patch_w + valid_positions[row_start: row_start + patch_w, 0] = np.arange(patch_w, dtype=np.int64) + valid_positions[row_start: row_start + patch_w, 1] = patch_y + positions[:num_patches] = valid_positions + + pixel_batches.append(padded_patches) + position_batches.append(positions) + + return ( + torch.from_numpy(np.stack(pixel_batches, axis=0)), + torch.from_numpy(np.stack(position_batches, axis=0)), + ) + + +def _prepare_gemma4_multimodal_inputs( + processor: object | None, + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, + use_gemma4_chat_template: bool = False, +) -> PreparedInputs: + if processor is None: + raise RuntimeError("multimodal Gemma4 transpile requires an AutoProcessor with image and audio support") + shared = _shared_prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + use_gemma4_chat_template=use_gemma4_chat_template, + ) + return PreparedInputs( + names=shared.names, + tensors=shared.tensors, + metadata=shared.metadata, + ) + + images = _load_image_inputs(image_files) + audio_waveforms: list[np.ndarray] = [] + sample_rate: int | None = None + if audio_file: + sample_rate = _resolve_audio_sample_rate(processor) + audio_waveforms.append(_load_audio_waveform(audio_file, target_sample_rate=sample_rate)) + + image_token = getattr(processor, "image_token", None) + audio_token = getattr(processor, "audio_token", None) + processor_prompt = _normalize_multimodal_prompt( + prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + ) + normalized_prompt = processor_prompt + if use_gemma4_chat_template: + normalized_prompt = prompt.strip() + # Match the native Cactus Gemma4 prompt builder instead of the HF chat + # template. The HF template omits the begin/end multimodal wrapper tokens, + # which shifts the multimodal prefill by four tokens for image+audio prompts. + processor_prompt = _build_gemma4_chat_prompt( + prompt=normalized_prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + + batch = processor( + text=processor_prompt, + images=images or None, + audio=audio_waveforms or None, + return_tensors="pt", + ) + if use_gemma4_chat_template: + _gemma4_split_cactus_newline_token_merges(batch) + native_image_tensors = _prepare_gemma4_native_image_tensors(processor, image_files) + if native_image_tensors is not None: + batch["pixel_values"], batch["pixel_position_ids"] = native_image_tensors + if audio_file and isinstance(batch.get("input_features"), torch.Tensor): + feature_tensor = batch["input_features"] + expected_mels = int(feature_tensor.shape[-1]) + try: + native_audio, native_audio_mask, native_audio_frames = prepare_native_gemma4_audio_features( + audio_file, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + except Exception as exc: + print(f"note=falling back to processor gemma4 audio features: {exc}") + batch["input_features"] = feature_tensor.to(dtype=torch_dtype) + fallback_mask = batch.get("input_features_mask") + if isinstance(fallback_mask, torch.Tensor): + batch["input_features_mask"] = fallback_mask.to(dtype=torch.bool) + native_audio_frames = int(fallback_mask.to(dtype=torch.int32).sum().item()) + else: + native_audio_frames = int(feature_tensor.shape[1]) + batch["input_features_mask"] = torch.ones( + (int(feature_tensor.shape[0]), native_audio_frames), + dtype=torch.bool, + ) + batch["native_audio_frames"] = native_audio_frames + else: + batch["input_features"] = native_audio + batch["input_features_mask"] = native_audio_mask + batch["native_audio_frames"] = native_audio_frames + + ordered_keys = [ + key + for key in _GEMMA4_MULTIMODAL_INPUT_ORDER + if isinstance(batch.get(key), torch.Tensor) + ] + if not ordered_keys: + raise RuntimeError("Gemma4 multimodal processor did not return any tensor inputs") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + metadata: dict[str, object] = { + "prompt": normalized_prompt, + "processor_prompt": processor_prompt, + "image_files": [str(Path(path).resolve()) for path in image_files], + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + } + if audio_file: + metadata["audio_file"] = str(Path(audio_file).resolve()) + if sample_rate is not None: + metadata["sample_rate"] = sample_rate + native_audio_frames = batch.get("native_audio_frames") + if isinstance(native_audio_frames, int): + metadata["native_audio_frames"] = native_audio_frames + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata=metadata, + ) + + +def _prepare_lfm2_vl_multimodal_inputs( + processor: object | None, + *, + prompt: str, + image_files: tuple[str, ...], + torch_dtype: torch.dtype, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> PreparedInputs: + if processor is None: + raise RuntimeError("LFM2-VL multimodal transpile requires an AutoProcessor with image support") + images = _load_image_inputs(image_files) + if not images: + raise RuntimeError("LFM2-VL multimodal transpile requires at least one --image-file") + + user_content: list[dict[str, object]] = [{"type": "image", "image": image} for image in images] + user_content.append({"type": "text", "text": prompt.strip()}) + + messages: list[dict[str, object]] = [] + normalized_system = system_prompt.strip() + if normalized_system: + messages.append({"role": "system", "content": normalized_system}) + messages.append({"role": "user", "content": user_content}) + + batch: Mapping[str, object] + apply_chat_template = getattr(processor, "apply_chat_template", None) + if callable(apply_chat_template): + batch = apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ) + else: + batch = processor(text=prompt, images=images, return_tensors="pt") + + tensors: list[torch.Tensor] = [] + names: list[str] = [] + input_shapes: dict[str, list[int]] = {} + for key in _LFM2_VL_MULTIMODAL_INPUT_ORDER: + value = batch.get(key) if hasattr(batch, "get") else None + if not isinstance(value, torch.Tensor): + raise RuntimeError(f"LFM2-VL processor did not return required tensor input: {key}") + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + elif key == "pixel_attention_mask": + value = value.to(dtype=torch.int64) + else: + value = value.to(dtype=torch.long) + names.append(key) + tensors.append(value) + input_shapes[key] = [int(dim) for dim in value.shape] + + return PreparedInputs( + names=tuple(names), + tensors=tuple(tensors), + metadata={ + "prompt": prompt, + "system_prompt": system_prompt, + "image_files": [str(Path(path).resolve()) for path in image_files], + "input_shapes": input_shapes, + "enable_thinking": bool(enable_thinking_if_supported), + }, + ) + + +def _load_model_source(model_id: str, *, local_files_only: bool) -> str: + local_snapshot = _resolve_local_snapshot(model_id) + if local_snapshot and _snapshot_has_model_weights(local_snapshot): + return local_snapshot + if local_snapshot and local_files_only: + raise RuntimeError( + f"Found local snapshot for {model_id}, but it is incomplete and has no model weights:\n" + f" {local_snapshot}\n" + "\n" + "Re-run without --local-files-only to let transformers download the missing weights." + ) + return model_id + + +def _load_transformers_bundle( + *, + model_id: str, + task: str, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, +): + config = _load_config_json(model_id) + config_model_type = str(config.get("model_type", "") or "").lower() + external_transformers_site_packages = _ensure_transformers_supports_model_type(config_model_type) + patch_note = _patch_transformers_torchvision_probe() + if patch_note: + print(f"note={patch_note}") + flex_patch_note = _patch_torch_flex_attention_compat() + if flex_patch_note: + print(f"note={flex_patch_note}") + if external_transformers_site_packages: + print(f"note=using external transformers install for {config_model_type}: {external_transformers_site_packages}") + + try: + from transformers import AutoFeatureExtractor # type: ignore + from transformers import AutoModel # type: ignore + from transformers import AutoModelForCTC # type: ignore + from transformers import AutoModelForCausalLM # type: ignore + from transformers import AutoModelForImageTextToText # type: ignore + from transformers import AutoModelForSeq2SeqLM # type: ignore + from transformers import AutoModelForSpeechSeq2Seq # type: ignore + from transformers import AutoProcessor # type: ignore + from transformers import AutoTokenizer # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"transformers is not available: {exc}") from exc + + model_source = _load_model_source(model_id, local_files_only=local_files_only) + source_candidates = [] + for candidate in (model_source, model_id): + if candidate not in source_candidates: + source_candidates.append(candidate) + common_kwargs: dict[str, object] = { + "trust_remote_code": trust_remote_code, + "local_files_only": local_files_only, + } + if token: + common_kwargs["token"] = token + + if task == "tdt_transcription": + model = load_tdt_local_model(model_source, torch_dtype=torch_dtype).eval() + return model_source, None, model, config + + if task == "causal_lm_logits": + tokenizer = None + tokenizer_errors: list[str] = [] + for source in source_candidates: + try: + tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + tokenizer_errors.append(f"{source}: {exc}") + if tokenizer is None: + raise RuntimeError( + f"Could not load tokenizer for {model_id}:\n" + + "\n".join(tokenizer_errors) + ) + if config_model_type == "lfm2_vl": + model = AutoModelForImageTextToText.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + tie_note = _tie_lfm2_vl_lm_head_if_needed(model) + if tie_note: + print(f"note={tie_note}") + else: + model = AutoModelForCausalLM.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model_source, tokenizer, model, config + if task == "multimodal_causal_lm_logits": + processor = None + processor_errors: list[str] = [] + for source in source_candidates: + try: + processor = AutoProcessor.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + processor_errors.append(f"{source}: {exc}") + if processor is None and config_model_type == "gemma4": + processor = _load_gemma4_processor_fallback( + source_candidates=source_candidates, + common_kwargs=common_kwargs, + ) + if processor is not None: + print("note=using manual gemma4 processor fallback") + if processor is None: + processor_config_hint = "" + if config_model_type == "gemma4": + processor_config_hint = ( + "\n" + "Gemma4 multimodal transpile needs a processor bundle, not just tokenizer/model weights.\n" + "Your local snapshot may be missing files such as `processor_config.json` or modality-specific\n" + "preprocessor configs. Use an official Gemma4 snapshot that includes the processor, or let\n" + "transformers download one by re-running without `--local-files-only`.\n" + ) + raise RuntimeError( + f"Could not load processor for {model_id}:\n" + + "\n".join(processor_errors) + + processor_config_hint + ) + + if config_model_type == "lfm2_vl": + model = AutoModelForImageTextToText.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + tie_note = _tie_lfm2_vl_lm_head_if_needed(model) + if tie_note: + print(f"note={tie_note}") + else: + model = AutoModelForCausalLM.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + if config_model_type == "gemma4": + repair_result = _repair_gemma4_checkpoint_weights(model, model_source) + if repair_result.get("applied"): + missing = repair_result.get("missing_keys", []) + unexpected = repair_result.get("unexpected_keys", []) + print( + "note=applied gemma4 checkpoint key remap" + f" missing_after={len(missing)} unexpected_after={len(unexpected)}" + ) + return model_source, processor, model, config + + processor = None + processor_errors: list[str] = [] + missing_optional_audio_dep: str | None = None + for source in source_candidates: + for loader in (AutoProcessor, AutoFeatureExtractor): + try: + processor = loader.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + processor_errors.append(f"{loader.__name__}@{source}: {exc}") + if isinstance(exc, ImportError) and "requires the librosa library" in str(exc): + missing_optional_audio_dep = "librosa" + break + if processor is not None: + break + if missing_optional_audio_dep is not None: + break + if processor is None: + if missing_optional_audio_dep == "librosa": + print("note=falling back to built-in audio preprocessing because the HF feature extractor requires librosa") + else: + print("note=falling back to built-in audio preprocessing because no HF processor/feature extractor was available") + + if task == "ctc_logits": + model_loaders = (AutoModelForCTC, AutoModel) + elif task in {"encoder_hidden_states", "seq2seq_transcription"}: + model_loaders = (AutoModelForSpeechSeq2Seq, AutoModelForSeq2SeqLM, AutoModel) + else: + raise NotImplementedError(f"unsupported generic HF task: {task}") + + load_errors: list[str] = [] + for loader in model_loaders: + try: + model = loader.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model_source, processor, model, config + except Exception as exc: + load_errors.append(f"{loader.__name__}: {exc}") + + raise RuntimeError( + f"Could not load model for task={task} from {model_source}.\n" + "\n".join(load_errors) + ) + + +def _load_optional_tokenizer( + *, + model_id: str, + model_source: str, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, +): + try: + from transformers import AutoTokenizer # type: ignore + except Exception: + return None + + source_candidates = [] + for candidate in (model_source, model_id): + if candidate not in source_candidates: + source_candidates.append(candidate) + + common_kwargs: dict[str, object] = { + "trust_remote_code": trust_remote_code, + "local_files_only": local_files_only, + } + if token: + common_kwargs["token"] = token + + for source in source_candidates: + try: + return AutoTokenizer.from_pretrained(source, **common_kwargs) + except Exception: + continue + return None + + +def _ctc_greedy_decode_token_ids(logits: np.ndarray, *, blank_token_id: int | None) -> list[int]: + if logits.ndim != 3 or logits.shape[0] < 1: + raise ValueError(f"expected CTC logits with shape [batch, time, vocab], got {list(logits.shape)}") + + raw_ids = np.argmax(logits[0], axis=-1).tolist() + collapsed: list[int] = [] + previous: int | None = None + for token_id in raw_ids: + if token_id != previous: + collapsed.append(int(token_id)) + previous = int(token_id) + + if blank_token_id is None: + return collapsed + return [token_id for token_id in collapsed if int(token_id) != int(blank_token_id)] + + +def _decode_token_ids(tokenizer: object, token_ids: list[int]) -> str: + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + raise TypeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") + try: + return str(decode(token_ids, skip_special_tokens=True)) + except TypeError: + return str(decode(token_ids)) + + +def _count_weight_bindings(ir_graph: IRGraph) -> int: + count = 0 + for value in ir_graph.values.values(): + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str): + count += 1 + return count + + +def _lower_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + graph = Graph() + graph._transpile_materialized_constants = [] # type: ignore[attr-defined] + env: dict[str, Any] = {} + runtime_inputs = [] + bound_constants = [] + bound_constant_bindings = [] + bound_constant_value_ids: dict[int, str] = {} + + for value_id in ir.inputs: + value = ir.values[value_id] + tensor = _lower_input_value(graph, value) + env[value_id] = tensor + runtime_inputs.append(tensor) + + for value_id, const in ir.constants.items(): + value = ir.values[value_id] + binding = _lookup_weight_binding(value) + if binding is not None: + binding = ensure_binding_compatible(binding, source_tensor=const) + lowered_const = _lower_constant_value(graph, value, const, binding=binding) + env[value_id] = lowered_const + if hasattr(lowered_const, "g") and hasattr(lowered_const, "id"): + bound_constants.append(lowered_const) + bound_constant_value_ids[int(lowered_const.id)] = str(value_id) + if binding is not None: + bound_constant_bindings.append( + { + "node_id": int(lowered_const.id), + "value_id": str(value_id), + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + ) + + for node_id in ir.order: + node = ir.nodes[node_id] + outputs = _lower_ir_node(graph, node, env, ir) + if len(outputs) != len(node.outputs): + raise ValueError( + f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" + ) + for output_id, tensor in zip(node.outputs, outputs): + env[output_id] = tensor + + outputs = [env[value_id] for value_id in ir.outputs] + seen_bound_constant_ids = {int(tensor.id) for tensor in bound_constants} + for tensor in getattr(graph, "_transpile_materialized_constants", []): + tensor_id = int(tensor.id) + if tensor_id in seen_bound_constant_ids: + continue + bound_constants.append(tensor) + seen_bound_constant_ids.add(tensor_id) + return TranspiledGraph( + graph=graph, + runtime_inputs=runtime_inputs, + bound_constants=bound_constants, + bound_constant_bindings=bound_constant_bindings, + bound_constant_value_ids=bound_constant_value_ids, + outputs=outputs, + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Load a Hugging Face model, canonicalize it into a generic transpile task, " + "capture it with the Cactus transpiler, lower it to a Cactus Graph, and " + "optionally save artifacts or run the lowered graph." + ) + ) + parser.add_argument( + "--model-id", + required=True, + help="Hugging Face model id or local snapshot path.", + ) + parser.add_argument( + "--task", + default="auto", + choices=( + "auto", + "causal_lm_logits", + "multimodal_causal_lm_logits", + "ctc_logits", + "encoder_hidden_states", + "seq2seq_transcription", + "tdt_transcription", + ), + help="Transpile task. Use auto to infer from config/model id.", + ) + parser.add_argument( + "--prompt", + default=_DEFAULT_CAUSAL_PROMPT, + help="Prompt used for causal_lm_logits or multimodal_causal_lm_logits when --input-ids is not set.", + ) + parser.add_argument( + "--system-prompt", + default="", + help="Optional system prompt used for Gemma4 multimodal chat-style prompt construction.", + ) + parser.add_argument( + "--enable-thinking", + action="store_true", + help="Enable model-specific thinking markers when the multimodal prompt format supports them.", + ) + parser.add_argument( + "--input-ids", + default="", + help="Optional comma-separated token ids for causal_lm_logits.", + ) + parser.add_argument( + "--max-new-tokens", + type=int, + default=32, + help=( + "For causal_lm_logits bundles, preallocate prompt-plus-generation context " + "so run-transpiled can do greedy autoregressive decoding." + ), + ) + parser.add_argument( + "--audio-file", + default="", + help="Path to a WAV file for audio or multimodal tasks.", + ) + parser.add_argument( + "--image-file", + action="append", + default=[], + help="Path to an image file for multimodal tasks. Repeat to pass multiple images.", + ) + parser.add_argument( + "--torch-dtype", + default="float16", + help="Torch dtype for model loading: float16, float32, or bfloat16.", + ) + parser.add_argument( + "--token", + default=os.environ.get("HF_TOKEN"), + help="Optional Hugging Face token. Defaults to HF_TOKEN.", + ) + parser.add_argument( + "--trust-remote-code", + action="store_true", + help="Pass trust_remote_code=True to transformers loaders.", + ) + parser.add_argument( + "--local-files-only", + action="store_true", + help="Require the model/processor to already exist locally.", + ) + parser.add_argument( + "--weights-dir", + default="", + help="Converted Cactus CQ weights directory for mmap weight binding.", + ) + parser.add_argument( + "--allow-unconverted-weights", + action="store_true", + help="Debug only: allow transpiling without converted Cactus CQ weights.", + ) + parser.add_argument( + "--artifact-dir", + default="", + help="Directory where raw_ir.json, optimized_ir.json, graph.cactus, component graphs, and result.json are saved. Defaults to ./transpiled//.", + ) + parser.add_argument( + "--graph-filename", + default="graph.cactus", + help="Filename to use for Graph.save() inside --artifact-dir.", + ) + parser.add_argument( + "--component-pipeline", + default="auto", + choices=("auto", "on", "off"), + help="Use a split component pipeline when the model adapter supports it.", + ) + parser.add_argument( + "--components", + default=None, + help=( + "Optional comma-separated component subset for component-pipeline models " + "(for example: vision_encoder,audio_encoder,lm_encoder,decoder)." + ), + ) + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + parser.add_argument( + "--skip-execute", + action="store_true", + help="Stop after lowering instead of running the transpiled graph.", + ) + parser.add_argument( + "--skip-reference-compare", + action="store_true", + help="Run the transpiled graph but skip the follow-up PyTorch reference pass.", + ) + args = parser.parse_args() + + validated_weights_dir = _validate_weights_dir(args.weights_dir.strip() or None, model_id=args.model_id) + if validated_weights_dir is None and not args.allow_unconverted_weights: + raise RuntimeError( + "Transpilation requires converted Cactus CQ weights.\n" + "\n" + "Create them first with:\n" + f" cactus convert {args.model_id} --bits 4\n" + "\n" + "Then retry with:\n" + f" cactus transpile {args.model_id} --weights-dir \n" + "\n" + "For compiler-only debugging, pass --allow-unconverted-weights." + ) + + image_files = tuple(str(path) for path in args.image_file if str(path).strip()) + if args.task == "auto": + inferred_task = _infer_task_from_config(args.model_id) + config_for_auto = _load_config_json(args.model_id) + plan_for_auto = infer_component_plan_from_config(config_for_auto, model_id=args.model_id) + has_multimodal_config = ( + plan_for_auto is not None + and plan_for_auto.task == "multimodal_causal_lm_logits" + ) + if image_files or (args.audio_file and has_multimodal_config and inferred_task == "causal_lm_logits"): + task = plan_for_auto.task if plan_for_auto is not None else "multimodal_causal_lm_logits" + else: + task = inferred_task + else: + task = args.task + torch_dtype = _parse_dtype(args.torch_dtype) + weights_dir = str(validated_weights_dir) if validated_weights_dir is not None else None + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else _default_artifact_dir_for_model(args.model_id) + + model_source, processor_or_tokenizer, model, model_config = _load_transformers_bundle( + model_id=args.model_id, + task=task, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") + if not preprocessor_config: + preprocessor_config = _load_optional_json(args.model_id, "preprocessor_config.json") + auxiliary_tokenizer = None + if task == "ctc_logits": + auxiliary_tokenizer = _load_optional_tokenizer( + model_id=args.model_id, + model_source=model_source, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + + canonical = None + if task == "causal_lm_logits": + prepared = _prepare_text_inputs( + processor_or_tokenizer, + prompt=args.prompt, + input_ids_text=args.input_ids.strip() or None, + max_new_tokens=int(args.max_new_tokens), + enable_thinking_if_supported=args.enable_thinking, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + weights_dir=weights_dir, + inputs_metadata=prepared.metadata, + ) + elif task == "multimodal_causal_lm_logits": + config_model_type = str(model_config.get("model_type", "") or "").lower() + if config_model_type == "lfm2_vl": + prepared = _prepare_lfm2_vl_multimodal_inputs( + processor_or_tokenizer, + prompt=args.prompt, + image_files=image_files, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + ) + else: + prepared = _prepare_gemma4_multimodal_inputs( + processor_or_tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=args.audio_file.strip() or None, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + use_gemma4_chat_template=True, + ) + prepared = _add_multimodal_generation_headroom( + prepared, + tokenizer=getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer), + max_new_tokens=int(args.max_new_tokens), + min_context_tokens=_multimodal_context_token_floor(config_model_type), + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + weights_dir=weights_dir, + ) + prime_static_features = getattr(canonical.module, "prime_static_multimodal_features", None) + if callable(prime_static_features): + prime_static_features(*prepared.tensors) + elif task == "tdt_transcription": + if not args.audio_file: + raise RuntimeError(f"--audio-file is required for task={task}") + prepared = _prepare_audio_inputs( + processor_or_tokenizer, + input_names=("input_features",), + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=args.audio_file, + torch_dtype=torch_dtype, + ) + elif task == "seq2seq_transcription": + if not args.audio_file: + raise RuntimeError(f"--audio-file is required for task={task}") + prepared = _prepare_audio_inputs( + processor_or_tokenizer, + input_names=("input_features",), + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=args.audio_file, + torch_dtype=torch_dtype, + ) + if getattr(model_config, "get", None) is not None or getattr(model, "config", None) is not None: + prepared = _augment_whisper_seq2seq_metadata( + prepared, + tokenizer_or_processor=processor_or_tokenizer, + model=model, + prompt=args.prompt, + max_new_tokens=int(args.max_new_tokens), + ) + else: + if not args.audio_file: + raise RuntimeError(f"--audio-file is required for task={task}") + canonical = canonicalize_model_interface(model, task=task) + prepared = _prepare_audio_inputs( + processor_or_tokenizer, + input_names=canonical.input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=args.audio_file, + torch_dtype=torch_dtype, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + ) + requested_components = None + if args.components: + requested_components = tuple( + component.strip() + for component in str(args.components).split(",") + if component.strip() + ) + component_specs = build_component_module_specs( + model, + task=task, + named_tensors=_named_tensor_store(prepared), + weights_dir=weights_dir, + inputs_metadata=prepared.metadata, + components=requested_components, + ) + use_component_pipeline = False + if args.component_pipeline == "on": + if component_specs is None: + raise RuntimeError( + f"--component-pipeline=on was requested, but {type(model).__name__} does not expose component specs for task={task}" + ) + use_component_pipeline = True + elif args.component_pipeline == "auto" and component_specs is not None: + use_component_pipeline = True + + if use_component_pipeline: + family = canonical.family if canonical is not None else getattr(model, "family", type(model).__name__) + return _run_component_pipeline_transpile( + args=args, + task=task, + family=str(family), + model_source=model_source, + model=model, + prepared=prepared, + component_specs=component_specs, + fusion_config=FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ), + weights_dir=weights_dir, + artifact_dir=artifact_dir, + processor_or_tokenizer=processor_or_tokenizer, + canonical=canonical, + ) + + if canonical is None: + raise RuntimeError(f"task={task} requires a component pipeline but none was selected") + wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + + print(f"model_id={args.model_id}") + print(f"model_source={model_source}") + print(f"task={task}") + print(f"adapter_family={canonical.family}") + print(f"adapter_module={type(canonical.module).__name__}") + print(f"input_names={','.join(prepared.names)}") + for name, tensor in zip(prepared.names, prepared.tensors): + print(f"input_{name}_shape={list(tensor.shape)}") + if weights_dir: + print(f"weights_dir={weights_dir}") + + print("capture_begin=true", flush=True) + prepare_cpu_float32_capture = getattr(canonical.module, "prepare_cpu_float32_capture", None) + restore_cpu_float32_capture = getattr(canonical.module, "restore_cpu_float32_capture", None) + if callable(prepare_cpu_float32_capture): + prepare_cpu_float32_capture() + try: + captured = capture_model(wrapper, prepared.tensors) + finally: + if callable(restore_cpu_float32_capture): + restore_cpu_float32_capture() + print("capture_done=true", flush=True) + captured.ir_graph.meta.setdefault("task", task) + captured.ir_graph.meta.setdefault("adapter_family", canonical.family) + raw_ir_graph = copy.deepcopy(captured.ir_graph) + + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + print("canonicalize_begin=true", flush=True) + canonicalize_exported_graph(captured.ir_graph) + print("canonicalize_done=true", flush=True) + print("optimize_begin=true", flush=True) + optimize_graph(captured.ir_graph, config=fusion_config) + print("optimize_done=true", flush=True) + print("lower_begin=true", flush=True) + tg = _lower_preoptimized_ir(captured.ir_graph) + print("lower_done=true", flush=True) + + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + binding_count = _count_weight_bindings(optimized_ir_graph) + op_counts = Counter(optimized_ir_graph.nodes[node_id].op for node_id in optimized_ir_graph.order) + raw_component_counts = summarize_ir_components(raw_ir_graph) + optimized_component_counts = summarize_ir_components(optimized_ir_graph) + raw_component_graphs = extract_component_subgraphs(raw_ir_graph) + optimized_component_graphs = extract_component_subgraphs(optimized_ir_graph) + transpiled_component_graphs: dict[str, TranspiledGraph] | None = None + + print(f"raw_ir_nodes={len(raw_ir_graph.order)}") + print(f"optimized_ir_nodes={len(optimized_ir_graph.order)}") + print(f"weight_bindings={binding_count}") + if raw_component_counts: + print( + "raw_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(raw_component_counts.items())) + ) + if optimized_component_counts: + print( + "optimized_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(optimized_component_counts.items())) + ) + print( + "ops=" + f"attention:{op_counts.get('attention', 0)} " + f"conv1d:{op_counts.get('conv1d', 0)} " + f"conv2d:{op_counts.get('conv2d', 0)} " + f"batch_norm:{op_counts.get('batch_norm', 0)} " + f"layer_norm:{op_counts.get('layer_norm', 0)} " + f"rms_norm:{op_counts.get('rms_norm', 0)} " + f"rope:{op_counts.get('rope', 0)} " + f"linear:{op_counts.get('linear', 0)}" + ) + + if weights_dir and binding_count == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir}\n" + "\n" + "The weights folder exists, but none of the captured constants matched entries in weights_manifest.json.\n" + "\n" + f"Recommended fix:\n" + f" cactus convert {args.model_id} {weights_dir}\n" + ) + + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + transpiled_component_graphs = { + component: _lower_preoptimized_ir(copy.deepcopy(component_graph)) + for component, component_graph in optimized_component_graphs.items() + } + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(raw_ir_graph), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(optimized_ir_graph), + }, + ) + for component, component_graph in raw_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("raw_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + for component, component_graph in optimized_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("optimized_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + for component in raw_component_graphs: + print(f"saved_raw_component_ir_{component}={artifact_dir / _component_artifact_name('raw_ir', component)}") + for component in optimized_component_graphs: + print( + f"saved_optimized_component_ir_{component}=" + f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" + ) + component_manifest_path = _write_component_bundle( + artifact_dir=artifact_dir, + model_id=args.model_id, + model_source=model_source, + task=task, + family=canonical.family, + inputs_metadata=prepared.metadata, + raw_component_graphs=raw_component_graphs, + optimized_component_graphs=optimized_component_graphs, + transpiled_component_graphs=transpiled_component_graphs, + graph_filename=args.graph_filename, + ) + print(f"saved_component_bundle_manifest={component_manifest_path}") + for component in optimized_component_graphs: + print( + f"saved_component_graph_{component}=" + f"{artifact_dir / 'components' / component / args.graph_filename}" + ) + graph_path = artifact_dir / args.graph_filename + tg.graph.save(graph_path) + print(f"saved_graph={graph_path}") + graph_binding_manifest_path = _write_graph_binding_manifest( + artifact_dir=artifact_dir, + filename="graph_bindings.json", + model_id=args.model_id, + model_source=model_source, + task=task, + family=canonical.family, + inputs_metadata=prepared.metadata, + transpiled_graph=tg, + ) + print(f"saved_graph_bindings={graph_binding_manifest_path}") + if binding_count > 0: + print( + "note=saved graph structure without embedded mmap bindings; use the saved component/full-graph manifests " + "to rebind weights/embeddings from the weights folder when loading" + ) + + if args.skip_execute: + return 0 + + tg.set_inputs([tensor.cpu().numpy() for tensor in prepared.tensors]) + print("execute_begin=true") + transpiled_output = tg.execute()[0].numpy().astype(np.float32) + print("execute_done=true") + + if args.skip_reference_compare: + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(transpiled_output.shape), + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": binding_count, + "reference_compare_skipped": True, + } + print(f"output_shape={list(transpiled_output.shape)}") + if task in {"causal_lm_logits", "multimodal_causal_lm_logits"}: + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"transpiled_next_token_id={transpiled_next}") + result_payload["transpiled_next_token_id"] = transpiled_next + if hasattr(tokenizer_like, "decode"): + transpiled_token = tokenizer_like.decode([transpiled_next]) + print(f"transpiled_next_token={transpiled_token!r}") + result_payload["transpiled_next_token"] = transpiled_token + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + del model + del wrapper + gc.collect() + return 0 + + print("reference_begin=true") + with torch.no_grad(): + reference_output = wrapper(*prepared.tensors).detach().float().cpu().numpy() + print("reference_done=true") + + max_abs_diff = float(np.max(np.abs(reference_output - transpiled_output))) + mean_abs_diff = float(np.mean(np.abs(reference_output - transpiled_output))) + result_payload: dict[str, object] = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(reference_output.shape), + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": binding_count, + "max_abs_diff": max_abs_diff, + "mean_abs_diff": mean_abs_diff, + } + + if task == "causal_lm_logits": + tokenizer = processor_or_tokenizer + hf_next = int(np.argmax(reference_output[0, -1])) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"hf_next_token_id={hf_next}") + print(f"transpiled_next_token_id={transpiled_next}") + print(f"logits_max_abs_diff={max_abs_diff:.6f}") + print(f"logits_mean_abs_diff={mean_abs_diff:.6f}") + print(f"hf_next_token={tokenizer.decode([hf_next])!r}") + print(f"transpiled_next_token={tokenizer.decode([transpiled_next])!r}") + result_payload.update( + { + "hf_next_token_id": hf_next, + "transpiled_next_token_id": transpiled_next, + "hf_next_token": tokenizer.decode([hf_next]), + "transpiled_next_token": tokenizer.decode([transpiled_next]), + } + ) + elif task == "ctc_logits": + blank_token_id = getattr(model.config, "pad_token_id", None) + if blank_token_id is None and auxiliary_tokenizer is not None: + blank_token_id = getattr(auxiliary_tokenizer, "pad_token_id", None) + print(f"output_shape={list(reference_output.shape)}") + print(f"output_max_abs_diff={max_abs_diff:.6f}") + print(f"output_mean_abs_diff={mean_abs_diff:.6f}") + if auxiliary_tokenizer is not None: + hf_token_ids = _ctc_greedy_decode_token_ids(reference_output, blank_token_id=blank_token_id) + transpiled_token_ids = _ctc_greedy_decode_token_ids(transpiled_output, blank_token_id=blank_token_id) + hf_transcript = _decode_token_ids(auxiliary_tokenizer, hf_token_ids) + transpiled_transcript = _decode_token_ids(auxiliary_tokenizer, transpiled_token_ids) + print(f"hf_transcript={hf_transcript!r}") + print(f"transpiled_transcript={transpiled_transcript!r}") + result_payload.update( + { + "blank_token_id": None if blank_token_id is None else int(blank_token_id), + "hf_transcript_token_ids": hf_token_ids, + "transpiled_transcript_token_ids": transpiled_token_ids, + "hf_transcript": hf_transcript, + "transpiled_transcript": transpiled_transcript, + } + ) + else: + print(f"output_shape={list(reference_output.shape)}") + print(f"output_max_abs_diff={max_abs_diff:.6f}") + print(f"output_mean_abs_diff={mean_abs_diff:.6f}") + + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/cactus/transpile/capture/import_ir.py b/python/cactus/transpile/import_ir.py similarity index 89% rename from python/cactus/transpile/capture/import_ir.py rename to python/cactus/transpile/import_ir.py index a8d125148..c3dc44bf6 100644 --- a/python/cactus/transpile/capture/import_ir.py +++ b/python/cactus/transpile/import_ir.py @@ -7,23 +7,24 @@ from torch.export.graph_signature import ConstantArgument from torch.export.graph_signature import InputKind -from cactus.transpile.capture.capture_pytorch import format_target -from cactus.transpile.capture.capture_pytorch import get_dtype -from cactus.transpile.capture.capture_pytorch import get_shape -from cactus.transpile.capture.capture_pytorch import resolve_attr -from cactus.transpile.capture.graph_ir import IRGraph -from cactus.transpile.capture.graph_ir import verify_ir -from cactus.transpile.capture.importers import extract_literals -from cactus.transpile.capture.importers import ImportContext -from cactus.transpile.capture.importers import import_call_function -from cactus.transpile.capture.importers import import_get_attr -from cactus.transpile.capture.importers import import_output -from cactus.transpile.capture.importers import import_placeholder -from cactus.transpile.capture.importers import value_id -from cactus.transpile.capture.normalize import dtype_to_ir -from cactus.transpile.capture.import_semantics import apply_import_semantics -from cactus.transpile.capture.weight_binding import resolve_transpile_weights_dir -from cactus.transpile.capture.weight_binding import resolve_weight_binding +from cactus.transpile.capture_pytorch import format_target +from cactus.transpile.capture_pytorch import get_dtype +from cactus.transpile.capture_pytorch import get_shape +from cactus.transpile.capture_pytorch import resolve_attr +from cactus.transpile.component_partition import annotate_ir_components +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import verify_ir +from cactus.transpile.importers import extract_literals +from cactus.transpile.importers import ImportContext +from cactus.transpile.importers import import_call_function +from cactus.transpile.importers import import_get_attr +from cactus.transpile.importers import import_output +from cactus.transpile.importers import import_placeholder +from cactus.transpile.importers import value_id +from cactus.transpile.normalize import dtype_to_ir +from cactus.transpile.import_semantics import apply_import_semantics +from cactus.transpile.weight_binding import resolve_transpile_weights_dir +from cactus.transpile.weight_binding import resolve_weight_binding def import_captured_to_ir(captured: Any, *, strict: bool = True) -> IRGraph: @@ -253,6 +254,9 @@ def _inline_wrap_with_set_grad_enabled(node: Any, shape: Any, dtype: Any) -> Non import_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) continue + if node.op == "call_module" and str(getattr(node, "target", "")) == "_guards_fn" and not node.users: + continue + if node.op == "output": import_output(ir, node, ctx) continue @@ -262,5 +266,6 @@ def _inline_wrap_with_set_grad_enabled(node: Any, shape: Any, dtype: Any) -> Non ctx.pop_node() apply_import_semantics(ir) + annotate_ir_components(ir) verify_ir(ir) return ir diff --git a/python/cactus/transpile/import_semantics.py b/python/cactus/transpile/import_semantics.py new file mode 100644 index 000000000..0bf84d3e7 --- /dev/null +++ b/python/cactus/transpile/import_semantics.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_passthrough +from cactus.transpile.canonicalize.utils import rebuild_graph +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import verify_ir + + +@dataclass(frozen=True) +class EarlyRoPEMatch: + anchor_node_id: str + input_value_id: str + theta: float + position_offset: int + partial: bool + node_ids: tuple[str, ...] + + +def apply_import_semantics(graph: IRGraph) -> IRGraph: + _rewrite_explicit_attention(graph) + _tag_explicit_linear(graph) + _rewrite_early_rope(graph) + rebuild_graph(graph) + verify_ir(graph) + return graph + + +def _rewrite_explicit_attention(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "scaled_dot_product_attention": + continue + additive_mask = bool(node.attrs.get("additive_mask", False)) + if not additive_mask and len(node.inputs) > 3: + mask_value = graph.values.get(node.inputs[3]) + if mask_value is not None and mask_value.dtype not in (None, "bool"): + additive_mask = True + node.op = "attention" + node.kind = "semantic" + node.meta["semantic_source"] = "import" + node.attrs = { + "scale": float(node.attrs.get("scale", 0.0)), + "is_causal": bool(node.attrs.get("is_causal", False)), + "window_size": int(node.attrs.get("window_size", 0)), + **({"enable_gqa": bool(node.attrs.get("enable_gqa", False))} if "enable_gqa" in node.attrs else {}), + **({"additive_mask": True} if additive_mask else {}), + } + + +def _tag_explicit_linear(graph: IRGraph) -> None: + for node in graph.nodes.values(): + if node.op not in {"linear", "addmm"}: + continue + node.meta.setdefault("semantic_source", "import") + + +def _rewrite_early_rope(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or not node.outputs: + continue + match = match_early_rope(graph, node.outputs[0]) + if match is None or match.anchor_node_id != node.id: + continue + if match.partial: + node.meta["semantic_hint"] = "partial_rope" + node.meta["rope_theta_hint"] = float(match.theta) + node.meta["rope_position_offset_hint"] = int(match.position_offset) + continue + node.op = "rope" + node.inputs = [match.input_value_id] + node.attrs = { + "theta": float(match.theta), + "position_offset": int(match.position_offset), + } + node.kind = "semantic" + node.meta["semantic_source"] = "import" + + +def match_early_rope(graph: IRGraph, value_id: str) -> EarlyRoPEMatch | None: + desc = _extract_rope_descriptor(graph, value_id) + if desc is None: + return None + return EarlyRoPEMatch( + anchor_node_id=str(desc["anchor_node_id"]), + input_value_id=str(desc["input_value_id"]), + theta=float(desc["theta"]), + position_offset=int(desc["position_offset"]), + partial=bool(desc.get("partial_rope", False)), + node_ids=tuple(sorted(set(desc.get("node_ids", ())))), + ) + + +def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = _strip_layout_passthrough(graph, value_id) + node = producer(graph, current) + if node is None: + return None + if node.op == "rope": + return { + "anchor_node_id": node.id, + "input_value_id": node.inputs[0], + "theta": float(node.attrs.get("theta", 0.0)), + "position_offset": int(node.attrs.get("position_offset", 0)), + "partial_rope": False, + "node_ids": (node.id,), + } + classic = _extract_classic_rope_descriptor(graph, current) + if classic is not None: + classic["partial_rope"] = False + return classic + if node.op == "cat" and len(node.inputs) == 2: + for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) + if rope_branch is None: + continue + passthrough = producer(graph, _strip_layout_passthrough(graph, passthrough_value_id)) + if passthrough is None or passthrough.op != "slice": + continue + if strip_passthrough(graph, passthrough.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): + continue + rope_branch["anchor_node_id"] = node.id + rope_branch["partial_rope"] = True + rope_branch["node_ids"] = tuple(sorted({node.id, passthrough.id, *rope_branch["node_ids"]})) + return rope_branch + return None + + +def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node = producer(graph, _strip_layout_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return None + for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + direct_mult = producer(graph, direct_mult_id) + rotated_mult = producer(graph, rotated_mult_id) + if direct_mult is None or rotated_mult is None: + continue + if direct_mult.op != "multiply" or rotated_mult.op != "multiply": + continue + direct = _extract_direct_branch(graph, direct_mult) + rotated = _extract_rotated_branch(graph, rotated_mult) + if direct is None or rotated is None: + continue + input_value_id = strip_passthrough(graph, direct["input_value_id"]) + if input_value_id != strip_passthrough(graph, rotated["input_value_id"]): + continue + cos_info = _extract_rope_trig(graph, direct["trig_value_id"], expected=("cos", "scalar_cos")) + sin_info = _extract_rope_trig(graph, rotated["trig_value_id"], expected=("sin", "scalar_sin")) + if cos_info is None or sin_info is None: + continue + if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: + continue + if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): + continue + return { + "anchor_node_id": node.id, + "input_value_id": input_value_id, + "theta": float(cos_info["theta"]), + "position_offset": int(cos_info["position_offset"]), + "node_ids": tuple(sorted({node.id, *direct["node_ids"], *rotated["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), + } + return None + + +def _extract_direct_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + if _unwrap_trig_value(graph, trig_value_id, expected=("cos", "scalar_cos")) is not None: + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotated_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + input_value_id = _extract_rotate_half_source(graph, rotated_value_id) + if input_value_id is None: + continue + if _unwrap_trig_value(graph, trig_value_id, expected=("sin", "scalar_sin")) is None: + continue + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: + cat_node = producer(graph, strip_passthrough(graph, value_id)) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: + return None + neg_node = producer(graph, cat_node.inputs[0]) + right_slice = producer(graph, cat_node.inputs[1]) + if neg_node is None or right_slice is None: + return None + if neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: + return None + left_slice = producer(graph, neg_node.inputs[0]) + if left_slice is None or left_slice.op != "slice" or right_slice.op != "slice": + return None + if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): + return None + return left_slice.inputs[0] + + +def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: + trig = _unwrap_trig_value(graph, value_id, expected=expected) + if trig is None: + return None + trig_node = trig["trig_node"] + cat_node = producer(graph, trig_node.inputs[0]) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: + return None + angle_info = _extract_angle_source(graph, cat_node.inputs[0]) + if angle_info is None: + return None + matmul_node = angle_info["matmul_node"] + inv_freq_const_id = None + arange_node = None + for input_id in matmul_node.inputs: + if inv_freq_const_id is None: + inv_freq_const_id = _find_constant_ancestor(graph, input_id) + if arange_node is None: + arange_node = _find_arange_ancestor(graph, input_id) + if inv_freq_const_id is None or arange_node is None: + return None + theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) + if theta is None: + return None + return { + "theta": theta, + "position_offset": int(arange_node.attrs.get("start", 0)), + "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), + } + + +def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"unsqueeze", "reshape", "view", "expand", "precision_cast"}: + current = node.inputs[0] + continue + if node.op == "type_as": + current = node.inputs[0] + continue + if node.op == "scalar_multiply": + current = node.inputs[0] + continue + if node.op not in expected: + return None + return {"trig_node": node} + + +def _extract_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + while True: + current = _strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op in {"transpose", "permute"}: + matmul_node = producer(graph, node.inputs[0]) + if matmul_node is None or matmul_node.op != "matmul": + return None + return {"matmul_node": matmul_node} + if node.op == "matmul": + return {"matmul_node": node} + if node.op in {"index", "slice"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + return None + + +def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = _strip_layout_passthrough(graph, current) + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return current + node = producer(graph, current) + if node is None or len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = _strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "arange": + return node + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return None + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def _infer_rope_theta(value: Any) -> float | None: + if not isinstance(value, torch.Tensor): + return None + flat = value.detach().cpu().float().reshape(-1) + if flat.numel() < 2: + return None + second = float(flat[1].item()) + if second <= 0.0: + return None + return float((1.0 / second) ** flat.numel()) diff --git a/python/cactus/transpile/capture/importers.py b/python/cactus/transpile/importers.py similarity index 60% rename from python/cactus/transpile/capture/importers.py rename to python/cactus/transpile/importers.py index 3dcc2ee30..ceaaec9bc 100644 --- a/python/cactus/transpile/capture/importers.py +++ b/python/cactus/transpile/importers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import math from dataclasses import dataclass from dataclasses import field from types import SimpleNamespace @@ -7,11 +8,12 @@ import torch -from cactus.transpile.capture.graph_ir import IRGraph -from cactus.transpile.capture.graph_ir import IRNode -from cactus.transpile.capture.graph_ir import IRValue -from cactus.transpile.capture.normalize import dtype_to_ir -from cactus.transpile.capture.normalize import normalize_target +from cactus.transpile.aten_ops import canonical_torch_op +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.normalize import dtype_to_ir +from cactus.transpile.normalize import normalize_target class UnsupportedImportError(NotImplementedError): @@ -61,73 +63,6 @@ def pop_node(self) -> None: if self.node_stack: self.node_stack.pop() - def _node_module_paths(self, node: Any) -> tuple[str, ...]: - meta = getattr(node, "meta", {}) or {} - raw_stack = meta.get("nn_module_stack") - if not isinstance(raw_stack, dict): - return () - paths: list[str] = [] - for entry in raw_stack.values(): - if not isinstance(entry, (tuple, list)) or not entry: - continue - module_path = entry[0] - if isinstance(module_path, str) and module_path: - paths.append(module_path) - # Prefer the deepest module path first. - paths.reverse() - deduped: list[str] = [] - for path in paths: - if path not in deduped: - deduped.append(path) - return tuple(deduped) - - def lookup_import_hint(self, node: Any, *, torch_op: str, op_name: str) -> dict[str, object]: - hints = self.transpile_metadata.get("import_hints", ()) - if not isinstance(hints, list): - return {} - - module_paths = self._node_module_paths(node) - matched_attrs: dict[str, object] = {} - matched_meta: dict[str, object] = {} - matched_provider: list[str] = [] - - for hint in hints: - if not isinstance(hint, dict): - continue - hint_op = hint.get("op") - if hint_op is not None and hint_op != op_name: - continue - hint_torch_op = hint.get("torch_op") - if hint_torch_op is not None and hint_torch_op != torch_op: - continue - - module_path = hint.get("module_path") - if module_path is not None and module_path not in module_paths: - continue - module_path_suffix = hint.get("module_path_suffix") - if module_path_suffix is not None and not any( - path == module_path_suffix or path.endswith(f".{module_path_suffix}") for path in module_paths - ): - continue - - attrs = hint.get("attrs", {}) - if isinstance(attrs, dict): - matched_attrs.update(attrs) - meta = hint.get("meta", {}) - if isinstance(meta, dict): - matched_meta.update(meta) - provider = hint.get("provider") - if isinstance(provider, str) and provider not in matched_provider: - matched_provider.append(provider) - - if not matched_attrs and not matched_meta and not matched_provider: - return {} - - if matched_provider: - matched_meta.setdefault("import_hint_providers", tuple(matched_provider)) - matched_meta.setdefault("import_hint_source", "capture_metadata") - return {"attrs": matched_attrs, "meta": matched_meta} - def value_id(node: Any, ctx: ImportContext) -> str: return ctx.resolve_value_id(node) @@ -159,7 +94,13 @@ def extract_input_ids(arg: Any, ctx: ImportContext) -> list[str]: def extract_literals(value: Any) -> Any: if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return extract_literals(meta_value) return None + if isinstance(value, torch.Size): + return tuple(extract_literals(v) for v in value) if isinstance(value, (tuple, list)): return type(value)(extract_literals(v) for v in value) if isinstance(value, dict): @@ -213,15 +154,106 @@ def _extract_numeric_literal(value: Any) -> float | int | None: return None +def _extract_static_int(value: Any) -> int | None: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return _extract_static_int(meta_value) + return None + + literal = extract_literals(value) + if isinstance(literal, bool): + return int(literal) + if isinstance(literal, int): + return int(literal) + if isinstance(literal, float) and float(literal).is_integer(): + return int(literal) + + try: + return int(value) + except Exception: + return None + + +def _extract_int_sequence_literal(value: Any) -> tuple[int, ...] | None: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return _extract_int_sequence_literal(meta_value) + return None + + if isinstance(value, torch.Size): + value = tuple(value) + + if isinstance(value, (tuple, list)): + result: list[int] = [] + for item in value: + item_value = _extract_static_int(item) + if item_value is None: + return None + result.append(item_value) + return tuple(result) + + scalar_value = _extract_static_int(value) + if scalar_value is not None: + return (scalar_value,) + return None + + +def _try_materialize_shape(shape: tuple[Any, ...] | None) -> tuple[int, ...] | None: + if shape is None: + return None + result: list[int] = [] + for dim in shape: + dim_value = _extract_static_int(dim) + if dim_value is None: + return None + result.append(dim_value) + return tuple(result) + + +def _extract_fx_tensor_shape(value: Any) -> tuple[Any, ...] | None: + if not is_fx_node(value): + return None + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if isinstance(meta_value, torch.Tensor): + return tuple(meta_value.shape) + tensor_meta = meta.get("tensor_meta") + tensor_shape = getattr(tensor_meta, "shape", None) + if tensor_shape is not None: + return tuple(tensor_shape) + return None + + def _base_meta(shape: tuple[int, ...] | None, dtype: str | None, torch_op: str, node: Any) -> dict[str, object]: + aten_op = canonical_torch_op(torch_op) return { "shape": shape, "dtype": dtype, - "torch_op": torch_op, + "torch_op": aten_op, + "aten_op": aten_op, "torch_name": node.name, } +def _extract_fx_torch_dtype(node: Any) -> torch.dtype | None: + if not is_fx_node(node): + return None + meta = getattr(node, "meta", {}) or {} + value = meta.get("val") + dtype = getattr(value, "dtype", None) + if isinstance(dtype, torch.dtype): + return dtype + tensor_meta = meta.get("tensor_meta") + dtype = getattr(tensor_meta, "dtype", None) + if isinstance(dtype, torch.dtype): + return dtype + return None + + def _import_scalar_binary( ir: IRGraph, node: Any, @@ -355,27 +387,11 @@ def import_call_function( torch_op: str, ) -> None: op = normalize_target(torch_op) - import_hint = ctx.lookup_import_hint(node, torch_op=torch_op, op_name=op) importer = OP_IMPORTERS.get(op) if importer is None: import_opaque_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name=op) else: importer(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) - _apply_import_hint(ir, node, import_hint) - - -def _apply_import_hint(ir: IRGraph, node: Any, hint: dict[str, object]) -> None: - if not hint: - return - ir_node = ir.nodes.get(node_id(node)) - if ir_node is None: - return - attrs = hint.get("attrs", {}) - if isinstance(attrs, dict): - ir_node.attrs.update(attrs) - meta = hint.get("meta", {}) - if isinstance(meta, dict): - ir_node.meta.update(meta) def import_opaque_call_function( @@ -418,7 +434,7 @@ def import_multiply(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[ def import_multiply_inplace(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply_inplace") + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply") def import_divide(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: @@ -463,6 +479,159 @@ def import_not_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + lhs, rhs = node.args[0], node.args[1] + lhs_is_node = is_fx_node(lhs) + rhs_is_node = is_fx_node(rhs) + lhs_literal = _extract_numeric_literal(lhs) + rhs_literal = _extract_numeric_literal(rhs) + + if lhs_is_node and rhs_is_node: + ir_node = IRNode( + id=node_id(node), + op="equal", + inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return + + if lhs_is_node and rhs_literal is not None: + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") + + ir_node = IRNode( + id=node_id(node), + op="scalar_equal", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_compare(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str, op_name: str) -> None: + lhs, rhs = node.args[0], node.args[1] + lhs_is_node = is_fx_node(lhs) + rhs_is_node = is_fx_node(rhs) + lhs_literal = _extract_numeric_literal(lhs) + rhs_literal = _extract_numeric_literal(rhs) + + if lhs_is_node and rhs_is_node: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return + + if lhs_is_node and rhs_literal is not None: + inputs = [value_id(lhs, ctx)] + scalar_op = f"scalar_{op_name}" + scalar_value = rhs_literal + elif rhs_is_node and lhs_literal is not None: + reverse_ops = { + "greater": "less", + "greater_equal": "less_equal", + "less": "greater", + "less_equal": "greater_equal", + } + inputs = [value_id(rhs, ctx)] + scalar_op = f"scalar_{reverse_ops[op_name]}" + scalar_value = lhs_literal + else: + ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") + + ir_node = IRNode( + id=node_id(node), + op=scalar_op, + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs={"value": scalar_value}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_binary(op_name: str): + def _importer( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, + ) -> None: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + return _importer + + +def import_where( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + condition = node.args[0] + true_value = node.args[1] + false_value = node.args[2] + + inputs = [value_id(condition, ctx)] + attrs: dict[str, object] = {} + + if is_fx_node(true_value): + inputs.append(value_id(true_value, ctx)) + attrs["true_is_scalar"] = False + else: + literal = _extract_numeric_literal(true_value) + if literal is None: + ctx.fail(f"unsupported true branch for {torch_op}: {node.args!r}") + attrs["true_is_scalar"] = True + attrs["true_value"] = float(literal) + + if is_fx_node(false_value): + inputs.append(value_id(false_value, ctx)) + attrs["false_is_scalar"] = False + else: + literal = _extract_numeric_literal(false_value) + if literal is None: + ctx.fail(f"unsupported false branch for {torch_op}: {node.args!r}") + attrs["false_is_scalar"] = True + attrs["false_value"] = float(literal) + + ir_node = IRNode( + id=node_id(node), + op="where", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_negate(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: ir_node = IRNode( id=node_id(node), @@ -544,7 +713,9 @@ def import_identity(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[ def import_reshape(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_shape = tuple(int(v) for v in node.args[1]) + target_shape = _extract_int_sequence_literal(node.args[1]) + if target_shape is None: + ctx.fail(f"unsupported reshape shape for {torch_op}: {node.args!r}") ir_node = IRNode( id=node_id(node), op="reshape", @@ -571,8 +742,9 @@ def import_flatten(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i def import_unsqueeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: dim = int(node.args[1]) attrs: dict[str, object] = {"dim": dim} - if shape is not None: - attrs["shape"] = tuple(int(v) for v in shape) + materialized_shape = _try_materialize_shape(shape) + if materialized_shape is not None: + attrs["shape"] = materialized_shape ir_node = IRNode( id=node_id(node), op="unsqueeze", @@ -584,8 +756,71 @@ def import_unsqueeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_squeeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + dim = int(node.args[1]) if len(node.args) > 1 and node.args[1] is not None else 0 + attrs: dict[str, object] = {"dim": dim} + materialized_shape = _try_materialize_shape(shape) + if materialized_shape is not None: + attrs["shape"] = materialized_shape + ir_node = IRNode( + id=node_id(node), + op="squeeze", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_masked_scatter( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + ir_node = IRNode( + id=node_id(node), + op="masked_scatter", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_masked_fill( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + value = _extract_numeric_literal(node.args[2]) + if value is None: + ctx.fail(f"unsupported masked_fill value for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="masked_fill", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": float(value)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_expand(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_shape = tuple(int(v) for v in node.args[1]) + target_shape = _extract_int_sequence_literal(node.args[1]) + if target_shape is None and shape is not None: + target_shape = _extract_int_sequence_literal(shape) + if target_shape is None: + ctx.fail(f"unsupported expand shape for {torch_op}: {node.args!r}") ir_node = IRNode( id=node_id(node), op="expand", @@ -597,6 +832,70 @@ def import_expand(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[in register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_repeat(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + repeats = _extract_int_sequence_literal(node.args[1]) if len(node.args) > 1 else None + if repeats is None: + ctx.fail(f"unsupported repeat factors for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="repeat", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"repeats": repeats}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_one_hot(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + num_classes = _extract_static_int(node.args[1]) if len(node.args) > 1 else None + if (num_classes is None or num_classes <= 0) and shape is not None and shape: + num_classes = int(shape[-1]) + if num_classes is None or num_classes <= 0: + ctx.fail(f"unsupported one_hot num_classes for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="one_hot", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"num_classes": int(num_classes)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_tril(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + diagonal = int(extract_literals(node.args[1])) if len(node.args) > 1 and extract_literals(node.args[1]) is not None else 0 + ir_node = IRNode( + id=node_id(node), + op="tril", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"diagonal": diagonal}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_unfold(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 4: + ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") + dimension = extract_literals(node.args[1]) + size = extract_literals(node.args[2]) + step = extract_literals(node.args[3]) + if not isinstance(dimension, int) or not isinstance(size, int) or not isinstance(step, int): + ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="unfold", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"dimension": dimension, "size": size, "step": step}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_transpose(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: attrs = {"dim0": 0, "dim1": 1} if torch_op == "aten.t.default" else {"dim0": int(node.args[1]), "dim1": int(node.args[2])} ir_node = IRNode( @@ -611,7 +910,26 @@ def import_transpose(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple def import_permute(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - permutation = tuple(int(v) for v in node.args[1]) + permutation = _extract_int_sequence_literal(node.args[1]) + if permutation is None: + ctx.fail(f"unsupported permute dims for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="permute", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"permutation": permutation}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_numpy_t(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + input_shape = _extract_fx_tensor_shape(node.args[0]) + rank_source = input_shape if input_shape is not None else shape + if rank_source is None: + ctx.fail(f"unsupported numpy_T rank inference for {torch_op}: {node.args!r}") + permutation = tuple(range(len(rank_source) - 1, -1, -1)) ir_node = IRNode( id=node_id(node), op="permute", @@ -623,6 +941,22 @@ def import_permute(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_movedim(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + source = _extract_int_sequence_literal(node.args[1]) + destination = _extract_int_sequence_literal(node.args[2]) + if source is None or destination is None: + ctx.fail(f"unsupported movedim dims for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="movedim", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"source": source, "destination": destination}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_matmul(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: ir_node = IRNode( id=node_id(node), @@ -675,6 +1009,25 @@ def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, . return _importer +def import_clamp(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + min_value = extract_literals(node.args[1]) if len(node.args) > 1 else None + max_value = extract_literals(node.args[2]) if len(node.args) > 2 else None + attrs: dict[str, object] = {} + if isinstance(min_value, (int, float)): + attrs["min"] = float(min_value) + if isinstance(max_value, (int, float)): + attrs["max"] = float(max_value) + ir_node = IRNode( + id=node_id(node), + op="clamp", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_pow_rsqrt(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: ir_node = IRNode( id=node_id(node), @@ -687,6 +1040,43 @@ def import_pow_rsqrt(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_reciprocal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="pow", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"exponent": -1.0}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_floor_divide( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported floor_divide signature for {torch_op}: {node.args!r}") + divisor = _extract_numeric_literal(node.args[1]) + if divisor is None: + ctx.fail(f"unsupported floor_divide divisor for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="scalar_floor_divide", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": float(divisor)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_softmax(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: ir_node = IRNode( id=node_id(node), @@ -698,6 +1088,22 @@ def import_softmax(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i ) register_node(ir, ir_node, shape=shape, dtype=dtype) + +def import_glu(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + axis = -1 + if len(node.args) > 1 and extract_literals(node.args[1]) is not None: + axis = int(node.args[1]) + ir_node = IRNode( + id=node_id(node), + op="glu", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_scaled_dot_product_attention( ir: IRGraph, node: Any, @@ -715,6 +1121,9 @@ def import_scaled_dot_product_attention( attrs["mask"] = mask_literal elif is_fx_node(node.args[3]): inputs.append(value_id(node.args[3], ctx)) + mask_dtype = _extract_fx_torch_dtype(node.args[3]) + if mask_dtype is not None and mask_dtype != torch.bool: + attrs["additive_mask"] = True if len(node.args) > 4 and extract_literals(node.args[4]) is not None: attrs["dropout_p"] = float(node.args[4]) if len(node.args) > 5 and extract_literals(node.args[5]) is not None: @@ -775,8 +1184,8 @@ def import_cat(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, def import_split_with_sizes(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: if len(node.args) < 2: ctx.fail(f"unsupported split_with_sizes signature for {torch_op}: {node.args!r}") - sizes = extract_literals(node.args[1]) - if not isinstance(sizes, (tuple, list)) or not all(isinstance(v, int) for v in sizes): + sizes = _extract_int_sequence_literal(node.args[1]) + if sizes is None: ctx.fail(f"unsupported split sizes for {torch_op}: {node.args!r}") axis = -1 if len(node.args) > 2 and extract_literals(node.args[2]) is not None: @@ -786,7 +1195,7 @@ def import_split_with_sizes(ir: IRGraph, node: Any, ctx: ImportContext, *, shape op="split_with_sizes", inputs=[value_id(node.args[0], ctx)], outputs=[value_id(node, ctx)], - attrs={"sizes": tuple(int(v) for v in sizes), "axis": axis}, + attrs={"sizes": sizes, "axis": axis}, meta=_base_meta(shape, dtype, torch_op, node), ) register_node(ir, ir_node, shape=shape, dtype=dtype) @@ -813,9 +1222,44 @@ def import_chunk(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int def import_pow(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx)] + base_literal = _extract_numeric_literal(node.args[0]) if len(node.args) > 0 else None attrs: dict[str, object] = {} exponent_literal = extract_literals(node.args[1]) if len(node.args) > 1 else None + + if base_literal is not None and isinstance(exponent_literal, (int, float)): + add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) + ir.constants[value_id(node, ctx)] = base_literal**exponent_literal + return + + if base_literal is not None: + if not (len(node.args) > 1 and is_fx_node(node.args[1])): + ctx.fail(f"unsupported pow signature for {torch_op}: scalar base with non-literal exponent {node.args!r}") + if float(base_literal) <= 0.0: + ctx.fail(f"unsupported pow signature for {torch_op}: non-positive scalar base {base_literal!r}") + + exponent_value_id = value_id(node.args[1], ctx) + mul_output_id = f"{value_id(node, ctx)}__pow_scalar_base_log_mul" + mul_node = IRNode( + id=f"{node_id(node)}__pow_scalar_base_log_mul", + op="scalar_multiply", + inputs=[exponent_value_id], + outputs=[mul_output_id], + attrs={"value": float(math.log(float(base_literal)))}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, mul_node, shape=shape, dtype=dtype) + + exp_node = IRNode( + id=node_id(node), + op="scalar_exp", + inputs=[mul_output_id], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, exp_node, shape=shape, dtype=dtype) + return + + inputs = [value_id(node.args[0], ctx)] if exponent_literal is None and len(node.args) > 1 and is_fx_node(node.args[1]): inputs.append(value_id(node.args[1], ctx)) else: @@ -873,16 +1317,34 @@ def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, prepend_value = None append_value = None + prepend_node = None + append_node = None if len(inputs) > 3: - prepend_value = extract_literals(inputs[3]) + if is_fx_node(inputs[3]): + prepend_node = inputs[3] + else: + prepend_value = extract_literals(inputs[3]) if len(inputs) > 4: - append_value = extract_literals(inputs[4]) + if is_fx_node(inputs[4]): + append_node = inputs[4] + else: + append_value = extract_literals(inputs[4]) if "prepend" in getattr(node, "kwargs", {}): - prepend_value = extract_literals(node.kwargs["prepend"]) + prepend_arg = node.kwargs["prepend"] + if is_fx_node(prepend_arg): + prepend_node = prepend_arg + prepend_value = None + else: + prepend_value = extract_literals(prepend_arg) if "append" in getattr(node, "kwargs", {}): - append_value = extract_literals(node.kwargs["append"]) + append_arg = node.kwargs["append"] + if is_fx_node(append_arg): + append_node = append_arg + append_value = None + else: + append_value = extract_literals(append_arg) if prepend_value is not None or append_value is not None: - ctx.fail(f"unsupported diff signature for {torch_op}: prepend/append are not supported") + ctx.fail(f"unsupported diff signature for {torch_op}: literal prepend/append are not supported") source_value_id = value_id(source, ctx) source_shape = None @@ -902,16 +1364,75 @@ def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, if dim_extent < 1: ctx.fail(f"unsupported diff input extent for {torch_op}: dim size {dim_extent}") - left_shape = list(source_shape) - left_shape[normalized_dim] = max(0, dim_extent - 1) + concat_inputs = [source_value_id] + concat_shapes = [tuple(int(v) for v in source_shape)] + + def _append_concat_input(arg: Any, label: str) -> None: + nonlocal concat_inputs, concat_shapes + if arg is None: + return + arg_value_id = value_id(arg, ctx) + arg_value = ir.values.get(arg_value_id) + arg_shape = arg_value.shape if arg_value is not None else None + if arg_shape is None: + ctx.fail(f"unsupported diff import for {torch_op}: missing {label} shape") + arg_shape = tuple(int(v) for v in arg_shape) + if len(arg_shape) != rank: + ctx.fail( + f"unsupported diff import for {torch_op}: {label} rank {len(arg_shape)} does not match source rank {rank}" + ) + for axis, (arg_dim, src_dim) in enumerate(zip(arg_shape, source_shape, strict=True)): + if axis == normalized_dim: + continue + if int(arg_dim) != int(src_dim): + ctx.fail( + f"unsupported diff import for {torch_op}: {label} shape {arg_shape} " + f"is incompatible with source shape {source_shape}" + ) + concat_inputs.append(arg_value_id) + concat_shapes.append(arg_shape) + + if prepend_node is not None: + concat_inputs = [] + concat_shapes = [] + _append_concat_input(prepend_node, "prepend") + concat_inputs.append(source_value_id) + concat_shapes.append(tuple(int(v) for v in source_shape)) + if append_node is not None: + _append_concat_input(append_node, "append") + elif append_node is not None: + _append_concat_input(append_node, "append") + + augmented_shape = list(source_shape) + augmented_shape[normalized_dim] = sum(int(shape_item[normalized_dim]) for shape_item in concat_shapes) + augmented_shape_tuple = tuple(int(v) for v in augmented_shape) + diff_shape = list(augmented_shape) + diff_shape[normalized_dim] = max(0, int(augmented_shape[normalized_dim]) - 1) + diff_shape_tuple = tuple(int(v) for v in diff_shape) + + diff_source_value_id = source_value_id + if len(concat_inputs) > 1: + concat_output_id = f"{value_id(node, ctx)}__diff_source" + concat_node = IRNode( + id=f"{node_id(node)}__diff_source", + op="cat", + inputs=concat_inputs, + outputs=[concat_output_id], + attrs={"axis": normalized_dim}, + meta=_base_meta(augmented_shape_tuple, dtype, torch_op, node), + ) + register_node(ir, concat_node, shape=augmented_shape_tuple, dtype=dtype) + diff_source_value_id = concat_output_id + + left_shape = list(diff_shape_tuple) left_shape_tuple = tuple(int(v) for v in left_shape) left_node = IRNode( id=f"{node_id(node)}__diff_left", op="slice", - inputs=[source_value_id], + inputs=[diff_source_value_id], outputs=[f"{value_id(node, ctx)}__diff_left"], - attrs={"axis": normalized_dim, "start": 0, "end": dim_extent - 1, "step": 1}, + attrs={"axis": normalized_dim, "start": 0, "end": int(augmented_shape[normalized_dim]) - 1, "step": 1}, meta=_base_meta(left_shape_tuple, dtype, torch_op, node), ) register_node(ir, left_node, shape=left_shape_tuple, dtype=dtype) @@ -919,9 +1440,9 @@ def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, right_node = IRNode( id=f"{node_id(node)}__diff_right", op="slice", - inputs=[source_value_id], + inputs=[diff_source_value_id], outputs=[f"{value_id(node, ctx)}__diff_right"], - attrs={"axis": normalized_dim, "start": 1, "end": dim_extent, "step": 1}, + attrs={"axis": normalized_dim, "start": 1, "end": int(augmented_shape[normalized_dim]), "step": 1}, meta=_base_meta(left_shape_tuple, dtype, torch_op, node), ) register_node(ir, right_node, shape=left_shape_tuple, dtype=dtype) @@ -931,9 +1452,9 @@ def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, op="subtract", inputs=[right_node.outputs[0], left_node.outputs[0]], outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), + meta=_base_meta(diff_shape_tuple, dtype, torch_op, node), ) - register_node(ir, diff_node, shape=shape, dtype=dtype) + register_node(ir, diff_node, shape=diff_shape_tuple, dtype=dtype) def import_index(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: @@ -1023,6 +1544,30 @@ def import_conv1d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[in register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_conv2d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported conv2d signature for {torch_op}: {node.args!r}") + + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + if len(node.args) > 2 and is_fx_node(node.args[2]): + inputs.append(value_id(node.args[2], ctx)) + + stride = _extract_int_list_or_scalar(node.args[3], default=1) if len(node.args) > 3 else 1 + padding = _extract_int_list_or_scalar(node.args[4], default=0) if len(node.args) > 4 else 0 + dilation = _extract_int_list_or_scalar(node.args[5], default=1) if len(node.args) > 5 else 1 + groups = _extract_int_list_or_scalar(node.args[6], default=1) if len(node.args) > 6 else 1 + + ir_node = IRNode( + id=node_id(node), + op="conv2d", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_pad(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: if len(node.args) < 2: ctx.fail(f"unsupported pad signature for {torch_op}: {node.args!r}") @@ -1058,12 +1603,8 @@ def import_ones(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, if not node.args: ctx.fail(f"unsupported ones signature for {torch_op}: {node.args!r}") - shape_literal = extract_literals(node.args[0]) - if isinstance(shape_literal, int): - output_shape = (int(shape_literal),) - elif isinstance(shape_literal, (tuple, list)) and all(isinstance(v, int) for v in shape_literal): - output_shape = tuple(int(v) for v in shape_literal) - else: + output_shape = _extract_int_sequence_literal(node.args[0]) + if output_shape is None: ctx.fail(f"unsupported ones shape for {torch_op}: {node.args!r}") node_dtype = dtype @@ -1086,8 +1627,39 @@ def import_ones(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, def import_layer_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx), value_id(node.args[3], ctx)] - attrs = {"normalized_shape": tuple(int(v) for v in node.args[1]), "eps": float(node.args[4])} + if not node.args: + ctx.fail(f"unsupported layer_norm signature for {torch_op}: {node.args!r}") + + normalized_shape_value = None + if len(node.args) > 1: + normalized_shape_value = _extract_int_sequence_literal(node.args[1]) + if normalized_shape_value is None: + normalized_shape_value = _extract_int_sequence_literal(getattr(node, "kwargs", {}).get("normalized_shape")) + if normalized_shape_value is None: + ctx.fail(f"unsupported normalized_shape for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + + eps_value = None + if len(node.args) > 4: + eps_value = _extract_numeric_literal(node.args[4]) + if eps_value is None: + eps_value = _extract_numeric_literal(getattr(node, "kwargs", {}).get("eps")) + if eps_value is None: + eps_value = 1e-5 + + inputs = [value_id(node.args[0], ctx)] + weight = node.args[2] if len(node.args) > 2 else getattr(node, "kwargs", {}).get("weight") + bias = node.args[3] if len(node.args) > 3 else getattr(node, "kwargs", {}).get("bias") + if not is_fx_node(weight): + ctx.fail(f"unsupported layer_norm weight form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + inputs.append(value_id(weight, ctx)) + if bias is None: + pass + elif is_fx_node(bias): + inputs.append(value_id(bias, ctx)) + else: + ctx.fail(f"unsupported layer_norm bias form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + + attrs = {"normalized_shape": tuple(int(v) for v in normalized_shape_value), "eps": float(eps_value)} ir_node = IRNode( id=node_id(node), op="layer_norm", @@ -1185,19 +1757,48 @@ def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i "type_as": import_type_as, "negate": import_negate, "abs": import_unary("abs"), + "clamp": import_clamp, "not_equal": import_not_equal, + "equal": import_equal, + "greater": lambda ir, node, ctx, *, shape, dtype, torch_op: import_compare( + ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="greater" + ), + "greater_equal": lambda ir, node, ctx, *, shape, dtype, torch_op: import_compare( + ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="greater_equal" + ), + "less": lambda ir, node, ctx, *, shape, dtype, torch_op: import_compare( + ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="less" + ), + "less_equal": lambda ir, node, ctx, *, shape, dtype, torch_op: import_compare( + ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="less_equal" + ), + "logical_and": import_binary("logical_and"), + "logical_or": import_binary("logical_or"), + "logical_not": import_unary("logical_not"), + "where": import_where, + "masked_scatter": import_masked_scatter, + "masked_fill": import_masked_fill, "cos": import_unary("cos"), "sin": import_unary("sin"), + "floor_divide": import_floor_divide, "scalar_exp": import_unary("scalar_exp"), "scalar_sqrt": import_unary("scalar_sqrt"), + "reciprocal": import_reciprocal, "scalar_log": import_unary("scalar_log"), "rsqrt": import_pow_rsqrt, "reshape": import_reshape, "flatten": import_flatten, "unsqueeze": import_unsqueeze, + "squeeze": import_squeeze, "expand": import_expand, + "repeat": import_repeat, + "one_hot": import_one_hot, + "tril": import_tril, + "unfold": import_unfold, + "numpy_T": import_numpy_t, "transpose": import_transpose, "permute": import_permute, + "movedim": import_movedim, "matmul": import_matmul, "linear": import_linear, "addmm": import_addmm, @@ -1206,6 +1807,7 @@ def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i "gelu": import_unary("gelu"), "gelu_erf": import_unary("gelu_erf"), "sigmoid": import_unary("sigmoid"), + "glu": import_glu, "softplus": import_unary("softplus"), "tanh": import_unary("tanh"), "softmax": import_softmax, @@ -1227,6 +1829,7 @@ def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i "gather": import_gather, "embedding": import_embedding, "conv1d": import_conv1d, + "conv2d": import_conv2d, "layer_norm": import_layer_norm, "rms_norm": import_rms_norm, "group_norm": import_group_norm, diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py new file mode 100644 index 000000000..43146ed59 --- /dev/null +++ b/python/cactus/transpile/lower.py @@ -0,0 +1,2925 @@ +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field +import math +import os +from typing import Any + +import numpy as np +import torch + +from cactus.transpile.runtime_compat import Graph +from cactus.transpile.runtime_compat import Tensor +from cactus.transpile.capture_pytorch import CapturedModel +from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.graph_ir import verify_ir +from cactus.transpile.optimize_graph import optimize_graph +from cactus.transpile.weight_compat import ensure_binding_compatible +from cactus.transpile.weight_binding import WeightBinding + + +_DYNAMIC_KV_POSITION_OFFSET = (1 << 64) - 1 + + +@dataclass +class TranspiledGraph: + graph: Graph + runtime_inputs: list[Tensor] + bound_constants: list[Tensor] + bound_constant_bindings: list[dict[str, object]] + bound_constant_value_ids: dict[int, str] + outputs: list[Tensor] + cache_state_tensors: list[tuple[str, Tensor, Tensor]] = field(default_factory=list) + + def set_input(self, index: int, data: Any, *, dtype: int | None = None) -> None: + if index < 0 or index >= len(self.runtime_inputs): + raise IndexError( + f"runtime input index out of range: {index} (have {len(self.runtime_inputs)})" + ) + self.graph.set_input(self.runtime_inputs[index], data, dtype=dtype) + + def set_inputs(self, inputs: list[Any] | tuple[Any, ...]) -> None: + if len(inputs) != len(self.runtime_inputs): + raise ValueError( + f"expected {len(self.runtime_inputs)} runtime inputs, got {len(inputs)}" + ) + for index, value in enumerate(inputs): + self.set_input(index, value) + + def execute(self) -> list[Tensor]: + self.graph.execute() + return self.outputs + + +@dataclass +class BroadcastAlias: + tensor: Tensor + logical_shape: tuple[int, ...] + kind: str + + +def _is_quantized_runtime_dtype(dtype: int) -> bool: + return int(dtype) in { + int(Graph.INT8), + int(getattr(Graph, "CQ1", 3)), + int(getattr(Graph, "CQ2", 4)), + int(getattr(Graph, "CQ3", 5)), + int(getattr(Graph, "CQ4", 6)), + } + + +def transpile_captured(captured: CapturedModel) -> TranspiledGraph: + return transpile_ir(captured.ir_graph) + + +def transpile_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + canonicalize_exported_graph(ir) + optimize_graph(ir) + return transpile_preoptimized_ir(ir) + + +def transpile_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + g = Graph() + g._transpile_materialized_constants = [] # type: ignore[attr-defined] + trace_lower = os.environ.get("CACTUS_LOWER_TRACE_NODES", "0") != "0" + env: dict[str, Any] = {} + runtime_inputs: list[Tensor] = [] + bound_constants: list[Tensor] = [] + bound_constant_bindings: list[dict[str, object]] = [] + bound_constant_value_ids: dict[int, str] = {} + + for value_id in ir.inputs: + value = ir.values[value_id] + tensor = _lower_input_value(g, value) + env[value_id] = tensor + runtime_inputs.append(tensor) + + for value_id, const in ir.constants.items(): + value = ir.values[value_id] + binding = _lookup_weight_binding(value) + if binding is not None: + binding = ensure_binding_compatible(binding, source_tensor=const) + lowered_const = _lower_constant_value(g, value, const, binding=binding) + env[value_id] = lowered_const + if isinstance(lowered_const, Tensor): + bound_constants.append(lowered_const) + bound_constant_value_ids[int(lowered_const.id)] = str(value_id) + if binding is not None: + bound_constant_bindings.append( + { + "node_id": int(lowered_const.id), + "value_id": str(value_id), + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + ) + + for node_id in ir.order: + node = ir.nodes[node_id] + outputs = _lower_ir_node(g, node, env, ir) + if len(outputs) != len(node.outputs): + raise ValueError( + f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" + ) + for output_id, tensor in zip(node.outputs, outputs): + env[output_id] = tensor + if trace_lower and isinstance(tensor, Tensor): + print( + "lower_trace" + f" cactus_id={int(tensor.id)}" + f" ir_node={node.id}" + f" op={node.op}" + f" output={output_id}" + f" shape={tuple(int(dim) for dim in tensor.shape)}", + flush=True, + ) + + outputs = [env[value_id] for value_id in ir.outputs] + seen_bound_constant_ids = {int(tensor.id) for tensor in bound_constants} + for tensor in getattr(g, "_transpile_materialized_constants", []): + tensor_id = int(tensor.id) + if tensor_id in seen_bound_constant_ids: + continue + bound_constants.append(tensor) + seen_bound_constant_ids.add(tensor_id) + cache_state_tensors = list(env.get("__internal_kv_cache_state_entries", [])) + return TranspiledGraph( + graph=g, + runtime_inputs=runtime_inputs, + bound_constants=bound_constants, + bound_constant_bindings=bound_constant_bindings, + bound_constant_value_ids=bound_constant_value_ids, + outputs=outputs, + cache_state_tensors=cache_state_tensors, + ) + + +def _lower_input_value(g: Graph, value: IRValue) -> Tensor: + if value.shape is None or value.dtype is None: + raise ValueError(f"IR input missing shape or dtype: {value.id}") + return g.input(shape=value.shape, dtype=_map_ir_dtype(value.dtype)) + + +def _should_lower_attention_with_internal_kv_cache(ir: IRGraph, node: IRNode) -> bool: + if not bool(ir.meta.get("use_internal_kv_cache", False)): + return False + if node.op not in {"attention", "scaled_dot_product_attention"}: + return False + component = str(ir.meta.get("component", "") or "").strip().lower() + return component in {"decoder_step", "decoder_prefill_chunk"} + + +def _lower_attention_with_internal_kv_cache( + g: Graph, + ir: IRGraph, + env: dict[str, Any], + node: IRNode, + *, + query: Tensor, + key: Tensor, + value: Tensor, + output_layout: str, +) -> list[Tensor]: + key_shape = tuple(int(dim) for dim in key.shape) + value_shape = tuple(int(dim) for dim in value.shape) + if len(key_shape) != 4 or len(value_shape) != 4: + raise NotImplementedError( + f"cached attention expects key/value as [batch, seq, heads, dim], got {key_shape} and {value_shape}" + ) + if int(key_shape[0]) != 1: + raise NotImplementedError("cached attention currently expects batch size 1") + + cache_states: dict[str, tuple[Tensor, Tensor]] = env.setdefault("__internal_kv_cache_states", {}) # type: ignore[assignment] + layer_key = str( + node.meta.get("attention_layer_index") + or node.meta.get("layer_index") + or node.id + ) + if layer_key not in cache_states: + default_cache_len = max(512, int(key_shape[1])) + max_cache_seq_len = int(ir.meta.get("max_cache_seq_len", default_cache_len) or default_cache_len) + sink_size = int(ir.meta.get("cache_sink_size", 4) or 4) + window_size = int(node.attrs.get("window_size", 0) or 0) + k_cache = g.kv_cache_state( + max_cache_seq_len, + int(key_shape[2]), + int(key_shape[3]), + window_size=window_size, + sink_size=sink_size, + ) + v_cache = g.kv_cache_state( + max_cache_seq_len, + int(value_shape[2]), + int(value_shape[3]), + window_size=window_size, + sink_size=sink_size, + ) + cache_states[layer_key] = (k_cache, v_cache) + cache_entries: list[tuple[str, Tensor, Tensor]] = env.setdefault("__internal_kv_cache_state_entries", []) # type: ignore[assignment] + cache_entries.append((layer_key, k_cache, v_cache)) + + k_cache, v_cache = cache_states[layer_key] + window_size = int(node.attrs.get("window_size", 0) or 0) + sink_size = int(ir.meta.get("cache_sink_size", 4) or 4) + g.kv_cache_append(key, k_cache, window_size=window_size, sink_size=sink_size) + g.kv_cache_append(value, v_cache, window_size=window_size, sink_size=sink_size) + out = g.attention_cached( + query, + key, + value, + k_cache, + v_cache, + scale=_resolve_attention_scale(node, query), + position_offset=_DYNAMIC_KV_POSITION_OFFSET, + window_size=window_size, + v_head_dim=int(value_shape[3]), + ) + if output_layout == "bthd": + return [out] + return [g.permute(out, (0, 2, 1, 3))] + +def _lower_constant_value( + g: Graph, + value: IRValue, + const: Any, + *, + binding: WeightBinding | None = None, +) -> Any: + if binding is None: + binding = _lookup_weight_binding(value) + if binding is not None: + _debug_mmap_binding(value.id, binding) + return g.mmap_weights(binding.path) + + _debug_constant_fallback(value, const) + + if isinstance(const, torch.nn.Parameter): + const = const.detach() + if isinstance(const, torch.Tensor): + tensor_value = const.detach().cpu() + else: + raise NotImplementedError( + f"unsupported IR constant type for {value.id}: {type(const).__name__}" + ) + + if tensor_value.numel() == 1: + return tensor_value.item() + + return _materialize_constant_tensor(g, tensor_value) + + +def _lookup_weight_binding(value: IRValue) -> WeightBinding | None: + meta = getattr(value, "meta", None) + if isinstance(meta, dict): + path = meta.get("path") + kind = meta.get("kind") + source_name = meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + return WeightBinding(path=path, kind=kind, source_name=source_name) + return None + + +def _debug_mmap_binding(value_id: str, binding: WeightBinding) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + print( + "[transpile:mmap] " + f"value={value_id} " + f"kind={binding.kind} " + f"source={binding.source_name} " + f"path={binding.path}" + ) + + +def _debug_constant_fallback(value: IRValue, const: Any) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + const_type = type(const).__name__ + shape = None + dtype = None + if isinstance(const, torch.Tensor): + shape = tuple(const.shape) + dtype = str(const.dtype) + source_name = None + if isinstance(value.meta, dict): + source_name = value.meta.get("source_name") + print( + "[transpile:fallback] " + f"value={value.id} " + f"source={source_name} " + f"const_type={const_type} " + f"shape={shape} " + f"dtype={dtype}" + ) + + +def _debug_embedding_lowering(node: IRNode, embedding_tensor: Tensor, indices_tensor: Tensor) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + print( + "[transpile:embedding] " + f"node={node.id} " + f"embedding_id={embedding_tensor.id} " + f"embedding_shape={tuple(embedding_tensor.shape)} " + f"embedding_dtype={embedding_tensor.dtype} " + f"indices_id={indices_tensor.id} " + f"indices_shape={tuple(indices_tensor.shape)} " + f"indices_dtype={indices_tensor.dtype}" + ) + + +def _matmul_with_quantized_rhs_legalization( + g: Graph, + lhs: Tensor, + rhs: Tensor, + *, + pretransposed_rhs: bool = False, + output_dtype: int | None = None, +) -> Tensor: + if _is_quantized_runtime_dtype(rhs.dtype) and lhs.dtype == Graph.FP32: + lhs = g.precision_cast(lhs, Graph.FP16) + return g.matmul(lhs, rhs, pretransposed_rhs=pretransposed_rhs, output_dtype=output_dtype) + + +def _matmul_output_dtype( + lhs: Tensor, + rhs: Tensor, + *, + output_dtype: int | None, +) -> int | None: + if output_dtype != Graph.FP16: + return output_dtype + if _is_quantized_runtime_dtype(rhs.dtype): + return output_dtype + if lhs.dtype == Graph.FP32 or rhs.dtype == Graph.FP32: + return Graph.FP32 + return output_dtype + + +def _trim_padded_last_dim(g: Graph, tensor: Tensor, expected_shape: tuple[int, ...] | None) -> Tensor: + if expected_shape is None: + return tensor + actual_shape = tuple(int(dim) for dim in tensor.shape) + target_shape = tuple(int(dim) for dim in expected_shape) + if actual_shape == target_shape: + return tensor + if ( + len(actual_shape) == len(target_shape) + and len(actual_shape) >= 1 + and actual_shape[:-1] == target_shape[:-1] + and actual_shape[-1] > target_shape[-1] + ): + return g.slice(tensor, len(actual_shape) - 1, 0, target_shape[-1]) + return tensor + + +def _legalize_for_transpose(g: Graph, x: Tensor) -> Tensor: + return x + + +def _reshape_attention_output_for_linear( + g: Graph, + attn_out: Tensor, + target_shape: tuple[int, ...], +) -> Tensor: + actual_shape = tuple(int(dim) for dim in attn_out.shape) + target_shape = tuple(int(dim) for dim in target_shape) + if len(actual_shape) == 4 and len(target_shape) == 3 and actual_shape[0] == target_shape[0]: + batch, dim1, dim2, dim3 = actual_shape + _, target_seq, target_hidden = target_shape + if dim1 == target_seq and dim2 * dim3 == target_hidden: + return g.reshape(attn_out, target_shape) + if dim2 == target_seq and dim1 * dim3 == target_hidden: + return g.reshape(g.permute(attn_out, (0, 2, 1, 3)), target_shape) + return g.reshape(attn_out, target_shape) + + +def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> list[Any]: + op = node.op + + if op == "arange": + start = int(node.attrs.get("start", 0)) + end = int(node.attrs["end"]) + step = int(node.attrs.get("step", 1)) + dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) + tensor_value = torch.arange(start, end, step=step, dtype=dtype) + return [_materialize_constant_tensor(g, tensor_value)] + + if op == "add": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "add")] + + if op == "add_clipped": + lhs, rhs = _legalize_elementwise_binary_inputs(g, _tensor(env, node.inputs[0]), _tensor(env, node.inputs[1])) + return [g.add_clipped(lhs, rhs)] + + if op == "subtract": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "subtract")] + + if op == "multiply": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] + + if op == "multiply_inplace": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] + + if op == "divide": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "divide")] + + if op == "not_equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "not_equal")] + + if op == "equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "equal")] + + if op == "greater": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "greater")] + + if op == "greater_equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "greater_equal")] + + if op == "less": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "less")] + + if op == "less_equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "less_equal")] + + if op == "logical_and": + lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + return [_lower_binary_op(g, lhs, rhs, "multiply")] + + if op == "logical_or": + lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + return [_lower_compare_op(g, _lower_binary_op(g, lhs, rhs, "add"), 0.0, "not_equal")] + + if op == "logical_not": + return [_lower_compare_op(g, env[node.inputs[0]], 0.0, "equal")] + + if op == "where": + return [_lower_where_op(g, node, env)] + + if op == "scalar_add": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_add(x, float(node.attrs["value"]))] + + if op == "scalar_subtract": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_subtract(x, float(node.attrs["value"]))] + + if op == "scalar_subtract_reverse": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_add(g.scalar_multiply(x, -1.0), float(node.attrs["value"]))] + + if op == "scalar_multiply": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_multiply(x, float(node.attrs["value"]))] + + if op == "scalar_divide": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_divide(x, float(node.attrs["value"]))] + + if op == "scalar_floor_divide": + source = _tensor(env, node.inputs[0]) + x = _ensure_scalar_math_tensor(g, source) + y = g.scalar_floor_divide(x, float(node.attrs["value"])) + output_dtype = ir.values[node.outputs[0]].dtype + if output_dtype is None: + return [y] + target_dtype = _map_ir_dtype(output_dtype) + if y.dtype == target_dtype: + return [y] + return [g.precision_cast(y, target_dtype)] + + if op == "scalar_not_equal": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_not_equal(x, float(node.attrs["value"]))] + + if op == "scalar_equal": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + not_equal = g.scalar_not_equal(x, float(node.attrs["value"])) + return [g.scalar_not_equal(not_equal, 1.0)] + + if op == "scalar_greater": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [_lower_compare_op(g, x, float(node.attrs["value"]), "greater")] + + if op == "scalar_greater_equal": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [_lower_compare_op(g, x, float(node.attrs["value"]), "greater_equal")] + + if op == "scalar_less": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [_lower_compare_op(g, x, float(node.attrs["value"]), "less")] + + if op == "scalar_less_equal": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [_lower_compare_op(g, x, float(node.attrs["value"]), "less_equal")] + + if op == "scalar_divide_reverse": + raise NotImplementedError("scalar_divide_reverse is not directly supported by Cactus graph ops") + + if op == "precision_cast": + target_dtype = node.attrs.get("dtype") + if target_dtype is None: + return [_tensor(env, node.inputs[0])] + return [g.precision_cast(_tensor(env, node.inputs[0]), _map_ir_or_torch_dtype(target_dtype))] + + if op == "type_as": + source = _tensor(env, node.inputs[0]) + target = _tensor(env, node.inputs[1]) + return [g.precision_cast(source, target.dtype)] + + if op == "abs": + return [g.abs(_tensor(env, node.inputs[0]))] + + if op == "clamp": + x = _ensure_fp16_tensor(g, _tensor(env, node.inputs[0])) + min_value = node.attrs.get("min") + max_value = node.attrs.get("max") + lo = float(min_value) if min_value is not None and math.isfinite(float(min_value)) else -65504.0 + hi = float(max_value) if max_value is not None and math.isfinite(float(max_value)) else 65504.0 + return [g.clamp(x, lo, hi)] + + if op == "negate": + return [g.scalar_multiply(_tensor(env, node.inputs[0]), -1.0)] + + if op == "pow": + exponent = node.attrs.get("exponent") + if exponent is None: + raise NotImplementedError("tensor-tensor pow is not supported by Cactus graph ops") + return [g.pow(_tensor(env, node.inputs[0]), float(exponent))] + + if op == "scalar_exp": + return [g.scalar_exp(_tensor(env, node.inputs[0]))] + + if op == "scalar_sqrt": + return [g.scalar_sqrt(_tensor(env, node.inputs[0]))] + + if op == "scalar_log": + return [g.scalar_log(_tensor(env, node.inputs[0]))] + + if op == "scalar_cos": + return [g.scalar_cos(_tensor(env, node.inputs[0]))] + + if op == "scalar_sin": + return [g.scalar_sin(_tensor(env, node.inputs[0]))] + + if op in {"reshape", "view"}: + source = env[node.inputs[0]] + if isinstance(source, BroadcastAlias): + input_shape = tuple(source.logical_shape) + else: + input_shape = tuple(_tensor(env, node.inputs[0]).shape) + target_shape = _resolve_reshape_shape(input_shape, tuple(node.attrs["shape"])) + if isinstance(source, BroadcastAlias): + if source.kind != "gqa_repeat_kv": + raise NotImplementedError(f"unsupported broadcast alias kind in reshape: {source.kind}") + base_shape = tuple(source.tensor.shape) + if len(base_shape) != 4: + raise NotImplementedError(f"gqa_repeat_kv alias requires 4D base tensor, got {base_shape}") + if len(target_shape) != 4: + raise NotImplementedError(f"gqa_repeat_kv reshape requires 4D target shape, got {target_shape}") + batch, kv_heads, seq_len, head_dim = base_shape + if target_shape[0] != batch or target_shape[2] != seq_len or target_shape[3] != head_dim: + raise NotImplementedError( + f"gqa_repeat_kv reshape mismatch: base {base_shape}, target {target_shape}" + ) + if target_shape[1] % max(kv_heads, 1) != 0: + raise NotImplementedError( + f"gqa_repeat_kv target head count must be a multiple of kv heads: {base_shape} -> {target_shape}" + ) + return [BroadcastAlias(source.tensor, target_shape, source.kind)] + return [g.reshape(_tensor(env, node.inputs[0]), target_shape)] + + if op == "flatten": + return [ + g.flatten( + _tensor(env, node.inputs[0]), + start_dim=int(node.attrs["start_dim"]), + end_dim=int(node.attrs["end_dim"]), + ) + ] + + if op == "unsqueeze": + x = _tensor(env, node.inputs[0]) + target_shape = node.attrs.get("shape") + if target_shape is None: + dim = _normalize_dim(int(node.attrs["dim"]), len(x.shape) + 1) + shape_list = list(x.shape) + shape_list.insert(dim, 1) + target_shape = tuple(shape_list) + return [g.reshape(x, tuple(target_shape))] + + if op == "expand": + matched_base = _match_gqa_expand_alias(ir, node) + if matched_base is not None: + return [BroadcastAlias(_tensor(env, matched_base), tuple(node.attrs["shape"]), "gqa_repeat_kv")] + x = _tensor(env, node.inputs[0]) + target_shape = _resolve_expand_shape(tuple(x.shape), tuple(node.attrs["shape"])) + if target_shape == tuple(x.shape): + return [x] + return [g.expand(x, target_shape)] + + if op == "repeat": + x = _tensor(env, node.inputs[0]) + repeats = tuple(int(value) for value in node.attrs["repeats"]) + return [_lower_repeat(g, x, repeats)] + + if op == "one_hot": + indices = _tensor(env, node.inputs[0]) + num_classes = int(node.attrs["num_classes"]) + eye = torch.eye(num_classes, dtype=torch.float16) + embedding = _materialize_constant_tensor(g, eye) + return [g.embedding_from_tensor(embedding, indices)] + + if op == "transpose": + x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) + dim0 = _normalize_dim(int(node.attrs["dim0"]), len(x.shape)) + dim1 = _normalize_dim(int(node.attrs["dim1"]), len(x.shape)) + rank = len(x.shape) + permutation = list(range(rank)) + permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] + if rank == 2 and permutation == [1, 0]: + return [g.transpose(x)] + return [g.permute(x, permutation)] + + if op == "permute": + x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) + permutation = tuple(_normalize_dim(int(dim), len(x.shape)) for dim in node.attrs["permutation"]) + if len(permutation) == 2 and permutation == (1, 0): + return [g.transpose(x)] + return [g.permute(x, permutation)] + + if op == "matmul": + lhs = _tensor(env, node.inputs[0]) + rhs = _tensor(env, node.inputs[1]) + output_value = ir.values.get(node.outputs[0]) + output_dtype = ( + _map_ir_dtype(output_value.dtype) + if output_value is not None and output_value.dtype is not None + else None + ) + matmul_output_dtype = _matmul_output_dtype(lhs, rhs, output_dtype=output_dtype) + if len(lhs.shape) == 2 and len(rhs.shape) == 2: + out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + if output_dtype is not None and out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + legalized = _legalize_matmul_inputs(g, lhs, rhs, node) + if legalized is None: + static_batched = _lower_static_batched_matmul(g, lhs, rhs, output_dtype=matmul_output_dtype) + if static_batched is not None: + if output_dtype is not None and static_batched.dtype != output_dtype: + static_batched = g.precision_cast(static_batched, output_dtype) + return [static_batched] + out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + if output_dtype is not None and out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + lhs_2d, rhs_2d, output_shape = legalized + out = _matmul_with_quantized_rhs_legalization(g, lhs_2d, rhs_2d, output_dtype=matmul_output_dtype) + out = g.reshape(out, output_shape) + if output_dtype is not None and out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "linear": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else x.dtype + reshape_back: tuple[int, ...] | None = None + if len(x.shape) > 2: + x = _flatten_to_2d_for_linear(g, x) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"linear missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + matmul_output_dtype = _matmul_output_dtype(x, weight, output_dtype=output_dtype) + if node.attrs.get("has_bias"): + bias = _tensor(env, node.inputs[2]) + if ( + x.dtype == Graph.FP16 + and weight.dtype == Graph.FP16 + and output_dtype == Graph.FP16 + ): + matmul_output_dtype = Graph.FP32 + elif bias.dtype == Graph.FP32 and not _is_quantized_runtime_dtype(weight.dtype): + matmul_output_dtype = Graph.FP32 + out = _matmul_with_quantized_rhs_legalization( + g, + x, + weight, + pretransposed_rhs=True, + output_dtype=matmul_output_dtype, + ) + out = _trim_padded_last_dim(g, out, output_value.shape if output_value is not None else None) + if node.attrs.get("has_bias"): + bias = _ensure_tensor_dtype(g, bias, matmul_output_dtype) + if output_value is not None and output_value.shape is not None: + expected_last_dim = int(output_value.shape[-1]) + if len(bias.shape) >= 1 and int(bias.shape[-1]) > expected_last_dim: + bias = g.slice(bias, len(bias.shape) - 1, 0, expected_last_dim) + out, bias = _legalize_elementwise_binary_inputs(g, out, bias) + try: + out = g.add(out, bias) + except RuntimeError as exc: + raise RuntimeError( + "linear bias add failed while lowering " + f"{node.id}: out_shape={tuple(out.shape)} bias_shape={tuple(bias.shape)} " + f"expected_shape={None if output_value is None else output_value.shape}" + ) from exc + if reshape_back is not None: + out = g.reshape(out, reshape_back) + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "dense_mlp_tq_fused": + hidden = _tensor(env, node.inputs[0]) + gate_weight = _tensor(env, node.inputs[1]) + up_weight = _tensor(env, node.inputs[2]) + down_weight = _tensor(env, node.inputs[3]) + product_scale = ( + node.attrs.get("product_scale") + if "product_scale" in node.attrs + else node.meta.get("product_scale_from_export", node.meta.get("product_scale_elided_by_post_norm", 1.0)) + ) + out = g.dense_mlp_tq_fused( + hidden, + gate_weight, + up_weight, + down_weight, + product_scale=float(product_scale), + ) + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else out.dtype + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "addmm": + bias = _tensor(env, node.inputs[0]) + lhs = _tensor(env, node.inputs[1]) + rhs = _tensor(env, node.inputs[2]) + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else lhs.dtype + matmul_output_dtype = _matmul_output_dtype(lhs, rhs, output_dtype=output_dtype) + if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 and bias.dtype == Graph.FP16 and output_dtype == Graph.FP16: + matmul_output_dtype = Graph.FP32 + elif bias.dtype == Graph.FP32 and not _is_quantized_runtime_dtype(rhs.dtype): + matmul_output_dtype = Graph.FP32 + out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + out = _trim_padded_last_dim(g, out, output_value.shape if output_value is not None else None) + bias = _ensure_tensor_dtype(g, bias, matmul_output_dtype) + out = g.add(bias, out) + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "relu": + return [g.relu(_tensor(env, node.inputs[0]))] + + if op == "silu": + return [g.silu(_tensor(env, node.inputs[0]))] + + if op == "gelu": + return [g.gelu(_tensor(env, node.inputs[0]))] + + if op == "gelu_erf": + return [g.gelu_erf(_tensor(env, node.inputs[0]))] + + if op == "sigmoid": + return [g.sigmoid(_tensor(env, node.inputs[0]))] + + if op == "glu": + return [g.glu(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] + + if op == "softplus": + x = _tensor(env, node.inputs[0]) + return [_lower_softplus(g, x)] + + if op == "tanh": + return [g.tanh(_tensor(env, node.inputs[0]))] + + if op == "softmax": + return [g.softmax(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] + + if op == "lstm_cell": + x = _tensor(env, node.inputs[0]) + h_prev = _tensor(env, node.inputs[1]) + c_prev = _tensor(env, node.inputs[2]) + weight_ih = _tensor(env, node.inputs[3]) + weight_hh = _tensor(env, node.inputs[4]) + bias_ih = _tensor(env, node.inputs[5]) + bias_hh = _tensor(env, node.inputs[6]) + + lstm_out = g.lstm_cell( + _ensure_fp16_tensor(g, x), + _ensure_fp16_tensor(g, h_prev), + _ensure_fp16_tensor(g, c_prev), + _ensure_fp16_tensor(g, weight_ih), + _ensure_fp16_tensor(g, weight_hh), + _ensure_fp16_tensor(g, bias_ih), + _ensure_fp16_tensor(g, bias_hh), + ) + h_new = g.slice(lstm_out, axis=2, start=0, length=1) + c_new = g.slice(lstm_out, axis=2, start=1, length=1) + h_shape = ir.values[node.outputs[0]].shape + c_shape = ir.values[node.outputs[1]].shape + if h_shape is not None: + h_new = g.reshape(h_new, tuple(int(dim) for dim in h_shape)) + if c_shape is not None: + c_new = g.reshape(c_new, tuple(int(dim) for dim in c_shape)) + return [h_new, c_new] + + if op in {"scaled_dot_product_attention", "attention"}: + if _should_lower_gemma4_decoder_attention_without_kernel(ir, node): + return [_lower_gemma4_decoder_attention_without_kernel(g, ir, env, node)] + mask = node.attrs.get("mask") + mask_tensor: Tensor | None = None + additive_mask = bool(node.attrs.get("additive_mask", False)) + if len(node.inputs) > 3: + mask_tensor = _tensor(env, node.inputs[3]) + elif mask is not None: + raise NotImplementedError(f"{op} with literal mask is not supported yet") + if op == "scaled_dot_product_attention": + dropout_p = float(node.attrs.get("dropout_p", 0.0)) + if dropout_p != 0.0: + raise NotImplementedError("scaled_dot_product_attention with dropout is not supported yet") + + # PyTorch SDPA exports tensors as [batch, heads, seq, dim], while Cactus attention + # expects [batch, seq, heads, dim]. The optimizer may strip surrounding layout + # permutes and mark the node as already-native to avoid large round-trip copies. + qkv_layout = str(node.attrs.get("qkv_layout", "bhsd") or "bhsd").lower() + q_layout = str(node.attrs.get("q_layout", qkv_layout) or qkv_layout).lower() + k_layout = str(node.attrs.get("k_layout", qkv_layout) or qkv_layout).lower() + v_layout = str(node.attrs.get("v_layout", qkv_layout) or qkv_layout).lower() + output_layout = str(node.attrs.get("output_layout", "bhsd") or "bhsd").lower() + query = _attention_tensor(env, node.inputs[0]) + key = _attention_tensor(env, node.inputs[1]) + value = _attention_tensor(env, node.inputs[2]) + if q_layout != "bthd": + query = g.permute(query, (0, 2, 1, 3)) + if k_layout != "bthd": + key = g.permute(key, (0, 2, 1, 3)) + if v_layout != "bthd": + value = g.permute(value, (0, 2, 1, 3)) + if mask_tensor is not None: + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + if _should_lower_attention_with_internal_kv_cache(ir, node): + return _lower_attention_with_internal_kv_cache( + g, + ir, + env, + node, + query=query, + key=key, + value=value, + output_layout=output_layout, + ) + out = g.attention( + query, + key, + value, + scale=_resolve_attention_scale(node, query), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=additive_mask, + ) + if output_layout == "bthd": + return [out] + return [g.permute(out, (0, 2, 1, 3))] + + if op == "attention_block": + has_mask = bool(node.attrs.get("has_mask", False)) + has_gate = bool(node.attrs.get("has_gate", False)) + has_bias = bool(node.attrs.get("has_bias", False)) + qkv_layout = str(node.attrs.get("qkv_layout", "bhsd") or "bhsd").lower() + input_index = 0 + query = _attention_tensor(env, node.inputs[input_index]) + if qkv_layout != "bthd": + query = g.permute(query, (0, 2, 1, 3)) + input_index += 1 + key = _attention_tensor(env, node.inputs[input_index]) + if qkv_layout != "bthd": + key = g.permute(key, (0, 2, 1, 3)) + input_index += 1 + value = _attention_tensor(env, node.inputs[input_index]) + if qkv_layout != "bthd": + value = g.permute(value, (0, 2, 1, 3)) + input_index += 1 + mask_tensor: Tensor | None = None + if has_mask: + mask_tensor = _tensor(env, node.inputs[input_index]) + input_index += 1 + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + + attn_out = g.attention( + query, + key, + value, + scale=_resolve_attention_scale(node, query), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=bool(node.attrs.get("additive_mask", False)), + ) + flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) + flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () + if flat_shape: + attn_out = _reshape_attention_output_for_linear(g, attn_out, flat_shape) + + if has_gate: + gate = _tensor(env, node.inputs[input_index]) + input_index += 1 + attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) + attn_out = g.multiply(attn_out, gate) + + weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + reshape_back: tuple[int, ...] | None = None + linear_input = attn_out + if len(linear_input.shape) > 2: + linear_input = _flatten_to_2d_for_linear(g, linear_input) + output_value = ir.values.get(node.outputs[0]) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"attention_block missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) + if has_bias: + out = g.add(out, _tensor(env, node.inputs[input_index])) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op == "self_attention_block": + has_mask = bool(node.attrs.get("has_mask", False)) + has_gate = bool(node.attrs.get("has_gate", False)) + has_bias = bool(node.attrs.get("has_bias", False)) + has_query_projection_bias = bool(node.attrs.get("has_query_projection_bias", False)) + has_query_add = bool(node.attrs.get("has_query_add", False)) + has_rel_query_add = bool(node.attrs.get("has_rel_query_add", False)) + has_key_projection_bias = bool(node.attrs.get("has_key_projection_bias", False)) + has_value_projection_bias = bool(node.attrs.get("has_value_projection_bias", False)) + has_rel_pos_bias = bool(node.attrs.get("has_rel_pos_bias", False)) + has_relative_key_projection_bias = bool(node.attrs.get("has_relative_key_projection_bias", False)) + + input_index = 0 + hidden = _tensor(env, node.inputs[input_index]) + input_index += 1 + + query_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + query_projection_bias = None + if has_query_projection_bias: + query_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + query_add = None + if has_query_add: + query_add = _tensor(env, node.inputs[input_index]) + input_index += 1 + rel_query_add = None + if has_rel_query_add: + rel_query_add = _tensor(env, node.inputs[input_index]) + input_index += 1 + + key_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + key_projection_bias = None + if has_key_projection_bias: + key_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + value_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + value_projection_bias = None + if has_value_projection_bias: + value_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + mask_tensor: Tensor | None = None + if has_mask: + mask_tensor = _tensor(env, node.inputs[input_index]) + input_index += 1 + + relative_key_input = None + relative_key_weight = None + relative_key_projection_bias = None + if has_rel_pos_bias: + relative_key_input = _tensor(env, node.inputs[input_index]) + input_index += 1 + relative_key_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + if has_relative_key_projection_bias: + relative_key_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + gate = None + if has_gate: + gate = _tensor(env, node.inputs[input_index]) + input_index += 1 + + output_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + output_bias = None + if has_bias: + output_bias = _tensor(env, node.inputs[input_index]) + + query_shape = tuple(int(v) for v in node.attrs.get("query_shape", ())) + key_shape = tuple(int(v) for v in node.attrs.get("key_shape", ())) + value_shape = tuple(int(v) for v in node.attrs.get("value_shape", ())) + relative_key_shape = tuple(int(v) for v in node.attrs.get("relative_key_shape", ())) + + query_base = _lower_projected_attention_tensor(g, hidden, query_weight, query_projection_bias, query_shape) + query = query_base + if query_add is not None: + query_add = _normalize_attention_add_tensor(g, query_add, query_shape) + query, query_add = _legalize_elementwise_binary_inputs(g, query, query_add) + query = g.add(query, query_add) + + key = _lower_projected_attention_tensor(g, hidden, key_weight, key_projection_bias, key_shape) + value = _lower_projected_attention_tensor(g, hidden, value_weight, value_projection_bias, value_shape) + + if has_rel_pos_bias: + if relative_key_input is None or relative_key_weight is None or not relative_key_shape: + raise NotImplementedError(f"self_attention_block missing relative position bias inputs for node {node.id}") + rel_query = query_base + if rel_query_add is not None: + rel_query_add = _normalize_attention_add_tensor(g, rel_query_add, query_shape) + rel_query, rel_query_add = _legalize_elementwise_binary_inputs(g, rel_query, rel_query_add) + rel_query = g.add(rel_query, rel_query_add) + relative_key = _lower_projected_attention_tensor( + g, + relative_key_input, + relative_key_weight, + relative_key_projection_bias, + relative_key_shape, + ) + rel_mask = g.rel_pos_bias(rel_query, relative_key, float(node.attrs.get("rel_pos_scale", 1.0))) + if mask_tensor is None: + mask_tensor = rel_mask + else: + mask_tensor, rel_mask = _legalize_elementwise_binary_inputs(g, mask_tensor, rel_mask) + mask_tensor = g.add(mask_tensor, rel_mask) + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + elif mask_tensor is not None: + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + if mask_tensor is not None: + mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + + attn_out = g.attention( + query, + key, + value, + scale=_resolve_attention_scale(node, query), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=bool(node.attrs.get("additive_mask", False)), + ) + flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) + flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () + if flat_shape: + attn_out = _reshape_attention_output_for_linear(g, attn_out, flat_shape) + + if gate is not None: + attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) + attn_out = g.multiply(attn_out, gate) + + linear_input = attn_out + reshape_back: tuple[int, ...] | None = None + if len(linear_input.shape) > 2: + linear_input = _flatten_to_2d_for_linear(g, linear_input) + output_value = ir.values.get(node.outputs[0]) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"self_attention_block missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + out = _matmul_with_quantized_rhs_legalization(g, linear_input, output_weight, pretransposed_rhs=True) + if output_bias is not None: + out = g.add(out, output_bias) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op in ("sum", "mean", "variance", "min", "max"): + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + axes = _normalize_reduction_axes(node.attrs.get("axis"), len(x.shape)) + keepdim = bool(node.attrs.get("keepdim", False)) + out = _lower_reduction(g, op, x, axes=axes) + if keepdim: + new_shape = list(x.shape) + for reduced_axis in axes: + new_shape[reduced_axis] = 1 + out = g.reshape(out, tuple(new_shape)) + return [out] + + if op == "cumsum": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + raw_axis = node.attrs.get("axis") + if not isinstance(raw_axis, int): + args = node.attrs.get("args", ()) + if isinstance(args, (list, tuple)) and len(args) > 1 and isinstance(args[1], int): + raw_axis = int(args[1]) + else: + raw_axis = -1 + axis = _normalize_dim(int(raw_axis), len(x.shape)) + return [g.cumsum(x, axis)] + + if op == "cat": + tensors = [_tensor(env, value_id) for value_id in node.inputs] + output_dtype: int | None = None + if node.outputs: + output_value = ir.values.get(node.outputs[0]) + if output_value is not None and output_value.dtype is not None: + output_dtype = _map_ir_or_torch_dtype(output_value.dtype) + return [_cat_with_legalized_dtype(g, tensors, axis=int(node.attrs.get("axis", 0)), dtype=output_dtype)] + + if op in {"stack", "aten.stack.default"}: + tensors = [_tensor(env, value_id) for value_id in node.inputs] + if not tensors: + raise NotImplementedError("stack requires at least one input tensor") + + raw_axis = node.attrs.get("axis") + if not isinstance(raw_axis, int): + args = node.attrs.get("args", ()) + if isinstance(args, (list, tuple)) and len(args) > 1 and isinstance(args[1], int): + raw_axis = int(args[1]) + else: + raw_axis = 0 + + base_shape = tuple(int(dim) for dim in tensors[0].shape) + axis = _normalize_dim(int(raw_axis), len(base_shape) + 1) + reshaped: list[Tensor] = [] + for tensor in tensors: + if tuple(int(dim) for dim in tensor.shape) != base_shape: + raise NotImplementedError( + f"stack requires equal input shapes, got {[tuple(int(dim) for dim in t.shape) for t in tensors]}" + ) + expanded_shape = list(base_shape) + expanded_shape.insert(axis, 1) + reshaped.append(g.reshape(tensor, tuple(expanded_shape))) + return [g.cat(reshaped, axis=axis)] + + if op == "slice": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs["axis"]), len(x.shape)) + start = int(node.attrs["start"]) + end = int(node.attrs["end"]) + step = int(node.attrs.get("step", 1)) + dim_size = x.shape[axis] + if step == 0: + raise NotImplementedError("slice with step == 0 is invalid") + if step != 1: + indices = list(range(*slice(start, end, step).indices(dim_size))) + if axis == 0 or x.dtype == Graph.FP16: + return [_lower_static_strided_slice_via_gather(g, x, axis=axis, indices=indices)] + if not indices: + return [g.slice(x, axis=axis, start=0, length=0)] + expanded_shape = list(int(dim) for dim in x.shape) + expanded_shape[axis] = 1 + pieces: list[Tensor] = [] + for idx in indices: + piece = g.index(x, index_value=idx, axis=axis) + piece = g.reshape(piece, tuple(expanded_shape)) + pieces.append(piece) + if len(pieces) == 1: + return [pieces[0]] + return [g.cat(pieces, axis=axis)] + start = _normalize_index(start, dim_size) + end = _normalize_slice_end(end, dim_size) + length = max(0, end - start) + return [g.slice(x, axis=axis, start=start, length=length)] + + if op == "split_with_sizes": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", -1)), len(x.shape)) + sizes = tuple(int(v) for v in node.attrs["sizes"]) + start = 0 + outputs: list[Tensor] = [] + for size in sizes: + outputs.append(g.slice(x, axis=axis, start=start, length=int(size))) + start += int(size) + return [outputs] + + if op == "chunk": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) + chunks = int(node.attrs["chunks"]) + if chunks <= 0: + raise NotImplementedError(f"chunk requires a positive chunk count, got {chunks}") + dim_size = int(x.shape[axis]) + if dim_size <= 0: + return [[g.slice(x, axis=axis, start=0, length=0)]] + chunk_size = (dim_size + chunks - 1) // chunks + outputs: list[Tensor] = [] + start = 0 + while start < dim_size: + length = min(chunk_size, dim_size - start) + outputs.append(g.slice(x, axis=axis, start=start, length=length)) + start += length + return [outputs] + + if op == "ones": + shape = tuple(int(v) for v in node.attrs.get("shape", ())) + if not shape: + raise NotImplementedError(f"ones requires a static shape, got {shape}") + torch_dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) + return [_materialize_constant_tensor(g, torch.ones(shape, dtype=torch_dtype))] + + if op == "tril": + x = _tensor(env, node.inputs[0]) + shape = tuple(int(dim) for dim in x.shape) + if len(shape) < 2: + raise NotImplementedError(f"tril expects rank >= 2, got shape {shape}") + diagonal = int(node.attrs.get("diagonal", 0)) + mask_dtype = _torch_dtype_for_graph_dtype(x.dtype) + mask = torch.ones(shape, dtype=mask_dtype).tril(diagonal=diagonal) + mask_tensor = _materialize_constant_tensor(g, mask) + return [_lower_binary_op(g, x, mask_tensor, "multiply")] + + if op == "unfold": + x = _tensor(env, node.inputs[0]) + rank = len(x.shape) + axis = _normalize_dim(int(node.attrs["dimension"]), rank) + size = int(node.attrs["size"]) + step = int(node.attrs["step"]) + if size <= 0: + raise NotImplementedError(f"unfold requires a positive size, got {size}") + if step <= 0: + raise NotImplementedError(f"unfold requires a positive step, got {step}") + dim_size = int(x.shape[axis]) + if size > dim_size: + raise NotImplementedError( + f"unfold size {size} exceeds dimension size {dim_size} for axis {axis}" + ) + + window_starts = list(range(0, dim_size - size + 1, step)) + if not window_starts: + raise NotImplementedError( + f"unfold produced no windows for axis={axis}, size={size}, step={step}, dim_size={dim_size}" + ) + + base_shape = [int(dim) for dim in x.shape] + trailing_shape = [base_shape[idx] for idx in range(rank) if idx != axis] + [size] + expanded_shape = tuple(trailing_shape[:axis] + [1] + trailing_shape[axis:]) + permutation = tuple(idx for idx in range(rank) if idx != axis) + (axis,) + + windows: list[Tensor] = [] + for start in window_starts: + piece = g.slice(x, axis=axis, start=start, length=size) + if axis != rank - 1: + piece = g.permute(piece, permutation=permutation) + piece = g.reshape(piece, expanded_shape) + windows.append(piece) + if len(windows) == 1: + return [windows[0]] + return [g.cat(windows, axis=axis)] + + if op == "pad": + x = _tensor(env, node.inputs[0]) + mode = str(node.attrs.get("mode", "constant")) + if mode != "constant": + raise NotImplementedError(f"pad mode is unsupported: {mode}") + pads = tuple(int(v) for v in node.attrs.get("pads", ())) + if len(pads) % 2 != 0: + raise NotImplementedError(f"pad expects an even-length pads tuple, got {pads}") + value = float(node.attrs.get("value", 0.0)) + + current = x + current_shape = list(int(dim) for dim in x.shape) + pad_dims = len(pads) // 2 + if pad_dims > len(current_shape): + raise NotImplementedError( + f"pad rank mismatch: pads={pads} for input shape {tuple(current_shape)}" + ) + + for pad_index in range(pad_dims): + before = pads[2 * pad_index] + after = pads[2 * pad_index + 1] + axis = len(current_shape) - 1 - pad_index + if before < 0 or after < 0: + raise NotImplementedError(f"negative pad is unsupported: {pads}") + pieces = [] + torch_dtype = _torch_dtype_for_graph_dtype(current.dtype) + if before > 0: + left_shape = list(current_shape) + left_shape[axis] = before + left = _materialize_constant_tensor( + g, + torch.full(tuple(left_shape), value, dtype=torch_dtype), + ) + pieces.append(left) + pieces.append(current) + if after > 0: + right_shape = list(current_shape) + right_shape[axis] = after + right = _materialize_constant_tensor( + g, + torch.full(tuple(right_shape), value, dtype=torch_dtype), + ) + pieces.append(right) + if len(pieces) > 1: + current = g.cat(pieces, axis=axis) + current_shape[axis] += before + after + return [current] + + if op == "index": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) + index_value = _normalize_index(int(node.attrs["index_value"]), x.shape[axis]) + return [g.index(x, index_value=index_value, axis=axis)] + + if op == "gather": + return [ + g.gather( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + axis=int(node.attrs.get("axis", 0)), + ) + ] + + if op == "embedding": + embedding_tensor = _tensor(env, node.inputs[0]) + indices_tensor = _tensor(env, node.inputs[1]) + _debug_embedding_lowering(node, embedding_tensor, indices_tensor) + return [g.embedding_from_tensor(embedding_tensor, indices_tensor)] + + if op == "conv_module": + input_index = 0 + x_nlc = _tensor(env, node.inputs[input_index]) + input_index += 1 + + pointwise1_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + pointwise1_bias = None + if bool(node.attrs.get("has_pointwise1_bias", False)): + pointwise1_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + depthwise_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + depthwise_bias = None + if bool(node.attrs.get("has_depthwise_bias", False)): + depthwise_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + batch_norm_weight = _tensor(env, node.inputs[input_index]) + batch_norm_bias = _tensor(env, node.inputs[input_index + 1]) + batch_norm_running_mean = _tensor(env, node.inputs[input_index + 2]) + batch_norm_running_var = _tensor(env, node.inputs[input_index + 3]) + input_index += 4 + + pointwise2_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + pointwise2_bias = None + if bool(node.attrs.get("has_pointwise2_bias", False)): + pointwise2_bias = _tensor(env, node.inputs[input_index]) + + if len(x_nlc.shape) != 3: + raise NotImplementedError(f"conv_module expects rank-3 NLC input, got {x_nlc.shape}") + kernel_size = int(node.attrs.get("depthwise_kernel_size", 0)) + padding = int(node.attrs.get("depthwise_padding", 0)) + if len(depthwise_weight.shape) != 3 or kernel_size != 9 or padding != 4: + raise NotImplementedError( + f"conv_module currently supports same depthwise kernel-9 lowering, got " + f"kernel_size={kernel_size} padding={padding} weight_shape={depthwise_weight.shape}" + ) + + current = g.conv1d_pointwise(x_nlc, pointwise1_weight, bias=pointwise1_bias) + current = g.glu(current, axis=-1) + current = g.conv1d_same_depthwise_k9(current, depthwise_weight, bias=depthwise_bias) + current = g.batch_norm( + current, + batch_norm_weight, + batch_norm_bias, + batch_norm_running_mean, + batch_norm_running_var, + axis=len(current.shape) - 1, + eps=float(node.attrs.get("eps", 1e-5)), + ) + current = g.silu(current) + current = g.conv1d_pointwise(current, pointwise2_weight, bias=pointwise2_bias) + return [current] + + if op == "conv1d": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + stride = int(node.attrs.get("stride", 1)) + padding = int(node.attrs.get("padding", 0)) + dilation = int(node.attrs.get("dilation", 1)) + groups = int(node.attrs.get("groups", 1)) + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and weight.shape[2] == 1 + and stride == 1 + and padding == 0 + and dilation == 1 + and groups == 1 + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_pointwise(x_nlc, weight, bias=bias) + return [g.permute(out_nlc, (0, 2, 1))] + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and weight.shape[2] == 9 + and stride == 1 + and padding == 4 + and dilation == 1 + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_same_depthwise_k9(x_nlc, weight, bias=bias) + return [g.permute(out_nlc, (0, 2, 1))] + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and stride == 1 + and padding == dilation * max(weight.shape[2] - 1, 0) + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_causal(x_nlc, weight, kernel_size=weight.shape[2], dilation=dilation) + if bias is not None: + bias_reshaped = g.reshape(bias, (1, 1, int(weight.shape[0]))) + out_nlc, bias_reshaped = _legalize_elementwise_binary_inputs(g, out_nlc, bias_reshaped) + out_nlc = g.add(out_nlc, bias_reshaped) + return [g.permute(out_nlc, (0, 2, 1))] + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and groups == 1 + and dilation == 1 + and weight.shape[2] == 3 + and padding == 1 + and stride in {1, 2} + ): + out = g.conv1d_k3(x, weight, stride=stride) + if bias is not None: + bias_reshaped = g.reshape(bias, (1, int(weight.shape[0]), 1)) + out, bias_reshaped = _legalize_elementwise_binary_inputs(g, out, bias_reshaped) + out = g.add(out, bias_reshaped) + return [out] + + if dilation != 1: + raise NotImplementedError(f"conv1d with dilation != 1 is unsupported by generic lowering: {dilation}") + if padding != 0: + if len(x.shape) != 3: + raise NotImplementedError(f"generic conv1d padding expects rank-3 input, got {x.shape}") + batch_size, channels, _ = (int(dim) for dim in x.shape) + pad_dtype = _torch_dtype_for_graph_dtype(x.dtype) + pieces: list[Tensor] = [] + if padding > 0: + left = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, padding), dtype=pad_dtype), + ) + pieces.append(left) + pieces.append(x) + if padding > 0: + right = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, padding), dtype=pad_dtype), + ) + pieces.append(right) + x = g.cat(pieces, axis=2) + if groups != 1: + if len(x.shape) != 3 or len(weight.shape) != 3: + raise NotImplementedError( + f"grouped conv1d lowering expects rank-3 input and weight, got {x.shape} and {weight.shape}" + ) + input_channels = int(x.shape[1]) + if input_channels % groups != 0: + raise NotImplementedError( + f"grouped conv1d requires input channels divisible by groups, got C_in={input_channels}, groups={groups}" + ) + output_channels = int(weight.shape[0]) + if output_channels % groups != 0: + raise NotImplementedError( + f"grouped conv1d requires output channels divisible by groups, got C_out={output_channels}, groups={groups}" + ) + input_channels_per_group = input_channels // groups + output_channels_per_group = output_channels // groups + if int(weight.shape[1]) != input_channels_per_group: + raise NotImplementedError( + "grouped conv1d weight has incompatible per-group input channels: " + f"weight C_in/group={int(weight.shape[1])}, expected {input_channels_per_group}" + ) + outputs: list[Tensor] = [] + for group_index in range(groups): + x_group = g.slice( + x, + axis=1, + start=group_index * input_channels_per_group, + length=input_channels_per_group, + ) + weight_group = g.slice( + weight, + axis=0, + start=group_index * output_channels_per_group, + length=output_channels_per_group, + ) + bias_group = None + if bias is not None: + bias_group = g.slice( + bias, + axis=0, + start=group_index * output_channels_per_group, + length=output_channels_per_group, + ) + outputs.append(g.conv1d(x_group, weight_group, bias=bias_group, stride=stride)) + if len(outputs) == 1: + return [outputs[0]] + return [g.cat(outputs, axis=1)] + return [g.conv1d(x, weight, bias=bias, stride=stride)] + + if op == "conv2d": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + stride = int(node.attrs.get("stride", 1)) + padding = int(node.attrs.get("padding", 0)) + dilation = int(node.attrs.get("dilation", 1)) + groups = int(node.attrs.get("groups", 1)) + + if len(x.shape) != 4 or len(weight.shape) != 4: + raise NotImplementedError(f"conv2d lowering expects rank-4 tensors, got {x.shape} and {weight.shape}") + if dilation != 1: + raise NotImplementedError(f"conv2d with dilation != 1 is unsupported: {dilation}") + + if stride == 2 and padding == 0 and groups == 1 and weight.shape[2:] == (3, 3): + batch_size, channels, height, width = (int(dim) for dim in x.shape) + pad_dtype = _torch_dtype_for_graph_dtype(x.dtype) + top = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, 1, width), dtype=pad_dtype), + ) + bottom = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, 1, width), dtype=pad_dtype), + ) + x = g.cat([top, x, bottom], axis=2) + left = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, height + 2, 1), dtype=pad_dtype), + ) + right = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, height + 2, 1), dtype=pad_dtype), + ) + x = g.cat([left, x, right], axis=3) + y = g.conv2d_k3s2p1(x, weight, bias=bias) + output_height = ((height - 3) // 2) + 1 + output_width = ((width - 3) // 2) + 1 + y = g.slice(y, axis=2, start=1, length=output_height) + y = g.slice(y, axis=3, start=1, length=output_width) + return [y] + + if stride == 2 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): + return [g.conv2d_k3s2p1(x, weight, bias=bias)] + + if ( + stride == 2 + and padding == 1 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and weight.shape[2:] == (3, 3) + ): + return [g.conv2d_depthwise_k3s2p1(x, weight, bias=bias)] + + if ( + stride == 1 + and padding == 0 + and groups == 1 + and weight.shape[2:] == (1, 1) + ): + return [g.conv2d_pointwise_1x1(x, weight, bias=bias)] + + if stride == 1 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): + return [g.conv2d_k3s1p1(x, weight, bias=bias)] + + return [g.conv2d(x, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups)] + + if op == "layer_norm": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + return [g.layer_norm(x, weight, bias=bias, eps=float(node.attrs["eps"]))] + + if op == "rms_norm": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + reshape_back: tuple[int, ...] | None = None + if len(x.shape) > 2: + reshape_back = tuple(int(dim) for dim in x.shape) + x = _flatten_to_2d_for_linear(g, x) + out = g.rms_norm(x, weight, eps=float(node.attrs["eps"])) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op == "rope": + rope_input = _tensor(env, node.inputs[0]) + if _rope_input_is_bhsd(ir, node.inputs[0]): + rope_input = g.permute(rope_input, (0, 2, 1, 3)) + rope_out = g.rope( + rope_input, + float(node.attrs["theta"]), + position_offset=int(node.attrs.get("position_offset", 0)), + ) + return [g.permute(rope_out, (0, 2, 1, 3))] + return [ + g.rope( + rope_input, + float(node.attrs["theta"]), + position_offset=int(node.attrs.get("position_offset", 0)), + ) + ] + + if op == "rel_pos_bias": + query = _attention_tensor(env, node.inputs[0]) + if len(query.shape) != 4: + raise NotImplementedError(f"rel_pos_bias expects rank-4 query input, got {query.shape}") + relative_key = _tensor(env, node.inputs[1]) + if len(relative_key.shape) != 4: + raise NotImplementedError(f"rel_pos_bias expects rank-4 relative_key input, got {relative_key.shape}") + if int(query.shape[2]) == int(relative_key.shape[2]) and int(query.shape[3]) == int(relative_key.shape[3]): + pass + elif int(query.shape[1]) == int(relative_key.shape[2]) and int(query.shape[3]) == int(relative_key.shape[3]): + query = g.permute(query, (0, 2, 1, 3)) + else: + raise NotImplementedError( + "rel_pos_bias query/relative_key layout mismatch: " + f"query={query.shape}, relative_key={relative_key.shape}" + ) + return [g.rel_pos_bias(query, relative_key, float(node.attrs.get("scale", 1.0)))] + + if op in {"gated_deltanet_prefill", "gated_deltanet_decode"}: + x = _tensor(env, node.inputs[0]) + qkv_weight = _tensor(env, node.inputs[1]) + a_weight = _tensor(env, node.inputs[2]) + b_weight = _tensor(env, node.inputs[3]) + norm_weight = _tensor(env, node.inputs[4]) + + input_index = 5 + z_weight = None + if bool(node.attrs.get("has_z", False)): + z_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + dt_bias = None + if bool(node.attrs.get("has_dt_bias", False)): + dt_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + a_log = None + if bool(node.attrs.get("has_a_log", False)): + a_log = _tensor(env, node.inputs[input_index]) + input_index += 1 + conv_weight = None + if bool(node.attrs.get("has_conv", False)): + conv_weight = _tensor(env, node.inputs[input_index]) + + if len(x.shape) != 3: + raise NotImplementedError(f"{op} currently expects rank-3 normalized input, got {x.shape}") + + batch_size, seq_len, hidden_dim = (int(dim) for dim in x.shape) + if batch_size != 1: + raise NotImplementedError(f"{op} currently supports batch size 1, got {x.shape}") + + num_k_heads = int(node.attrs["num_k_heads"]) + num_v_heads = int(node.attrs["num_v_heads"]) + key_dim = int(node.attrs["key_dim"]) + value_dim = int(node.attrs["value_dim"]) + eps = float(node.attrs.get("eps", 1e-6)) + chunk_size = int(node.attrs.get("chunk_size", 64)) + + mixed_qkv = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + qkv_weight, + pretransposed_rhs=True, + ) + mixed_qkv_dim = int(qkv_weight.shape[0]) + mixed_qkv = g.reshape(mixed_qkv, (batch_size, seq_len, mixed_qkv_dim)) + if conv_weight is not None: + kernel_size = int(conv_weight.shape[2]) + mixed_qkv = g.conv1d_causal(mixed_qkv, conv_weight, kernel_size=kernel_size, dilation=1) + mixed_qkv = g.silu(mixed_qkv) + + q_proj_dim = num_k_heads * key_dim + v_proj_dim = num_v_heads * value_dim + k_proj_dim = num_k_heads * key_dim + q_proj = g.slice(mixed_qkv, axis=2, start=0, length=q_proj_dim) + k_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim, length=k_proj_dim) + v_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim + k_proj_dim, length=v_proj_dim) + + q_4d = g.reshape(q_proj, (batch_size, seq_len, num_k_heads, key_dim)) + k_4d = g.reshape(k_proj, (batch_size, seq_len, num_k_heads, key_dim)) + v_4d = g.reshape(v_proj, (batch_size, seq_len, num_v_heads, value_dim)) + + q_norm = g.sum(g.multiply(q_4d, q_4d), axis=3) + q_norm = g.scalar_sqrt(g.scalar_add(q_norm, eps)) + q_norm = g.reshape(q_norm, (batch_size, seq_len, num_k_heads, 1)) + q_4d = g.divide(q_4d, q_norm) + + k_norm = g.sum(g.multiply(k_4d, k_4d), axis=3) + k_norm = g.scalar_sqrt(g.scalar_add(k_norm, eps)) + k_norm = g.reshape(k_norm, (batch_size, seq_len, num_k_heads, 1)) + k_4d = g.divide(k_4d, k_norm) + + a_logits = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + a_weight, + pretransposed_rhs=True, + ) + a_logits = g.reshape(a_logits, (batch_size, seq_len, int(a_weight.shape[0]))) + b_logits = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + b_weight, + pretransposed_rhs=True, + ) + b_logits = g.reshape(b_logits, (batch_size, seq_len, int(b_weight.shape[0]))) + + if dt_bias is not None: + dt_bias_2d = g.reshape(dt_bias, (1, int(dt_bias.shape[0]))) + a_logits, dt_bias_2d = _legalize_elementwise_binary_inputs(g, a_logits, dt_bias_2d) + a_logits = g.add(a_logits, dt_bias_2d) + a_softplus = _lower_softplus(g, a_logits) + + if a_log is not None: + a_log_2d = g.reshape(a_log, (1, int(a_log.shape[0]))) + neg_exp_a = g.scalar_multiply(g.scalar_exp(a_log_2d), -1.0) + neg_exp_a, a_softplus = _legalize_elementwise_binary_inputs(g, neg_exp_a, a_softplus) + gate_log = g.multiply(neg_exp_a, a_softplus) + else: + gate_log = g.scalar_multiply(a_softplus, -1.0) + beta = g.sigmoid(b_logits) + + initial_state = _materialize_constant_tensor( + g, + torch.zeros((batch_size, key_dim, num_v_heads, value_dim), dtype=torch.float16), + ) + + if op == "gated_deltanet_decode": + deltanet_out = g.gated_deltanet_decode(q_4d, k_4d, v_4d, gate_log, beta, initial_state, 0.0) + else: + deltanet_out = g.gated_deltanet_prefill(q_4d, k_4d, v_4d, gate_log, beta, initial_state, chunk_size, 0.0) + + y_4d = g.slice(deltanet_out, axis=1, start=0, length=seq_len) + y_2d = g.reshape(y_4d, (seq_len * num_v_heads, value_dim)) + + if z_weight is not None: + z_proj = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + z_weight, + pretransposed_rhs=True, + ) + z_proj = g.reshape(z_proj, (seq_len * num_v_heads, value_dim)) + y_2d = g.multiply(g.rms_norm(y_2d, norm_weight, eps=eps), g.silu(z_proj)) + + return [g.reshape(y_2d, (batch_size, seq_len, num_v_heads * value_dim))] + + if op == "group_norm": + return [ + g.group_norm( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + _tensor(env, node.inputs[2]), + num_groups=int(node.attrs["num_groups"]), + eps=float(node.attrs["eps"]), + ) + ] + + if op == "batch_norm": + axis = int(node.attrs.get("axis", 1)) + return [ + g.batch_norm( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + _tensor(env, node.inputs[2]), + _tensor(env, node.inputs[3]), + _tensor(env, node.inputs[4]), + axis=axis, + eps=float(node.attrs["eps"]), + ) + ] + + if op == "identity": + return [env[node.inputs[0]]] + + if op == "contiguous": + return [_tensor(env, node.inputs[0])] + + if op == "advanced_index": + lowered = _try_lower_advanced_index(g, node, env, ir) + if lowered is not None: + return [lowered] + raise NotImplementedError(f"unsupported advanced_index pattern for node {node.id}") + + if op in {"masked_scatter", "aten.masked_scatter.default"}: + base = _tensor(env, node.inputs[0]) + mask = _tensor(env, node.inputs[1]) + source = _tensor(env, node.inputs[2]) + if source.dtype != base.dtype: + source = _ensure_tensor_dtype(g, source, base.dtype) + return [g.masked_scatter(base, mask, source)] + + if op == "masked_fill": + base = _tensor(env, node.inputs[0]) + mask = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + inverse_mask = g.scalar_add(g.scalar_multiply(mask, -1.0), 1.0) + fill_value = _clamp_scalar_for_dtype(float(node.attrs["value"]), base.dtype) + fill_term = g.scalar_multiply(_ensure_tensor_dtype(g, mask, base.dtype), fill_value) + base_term = _lower_binary_op(g, base, _ensure_tensor_dtype(g, inverse_mask, base.dtype), "multiply") + return [_lower_binary_op(g, fill_term, base_term, "add")] + + if op == "getitem": + source = env[node.inputs[0]] + index = int(node.attrs["index"]) + if isinstance(source, (tuple, list)): + return [source[index]] + if isinstance(source, Tensor): + normalized_index = _normalize_index(index, int(source.shape[0])) + return [g.index(source, normalized_index, axis=0)] + raise NotImplementedError(f"getitem source is not tuple/list for node {node.id}") + + raise NotImplementedError(f"unsupported IR op in lowering: {op}") + + +def _tensor(env: dict[str, Any], value_id: str) -> Tensor: + try: + value = env[value_id] + except KeyError as exc: + raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc + if not isinstance(value, Tensor): + raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") + return value + + +def _attention_tensor(env: dict[str, Any], value_id: str) -> Tensor: + try: + value = env[value_id] + except KeyError as exc: + raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc + if isinstance(value, BroadcastAlias): + if value.kind != "gqa_repeat_kv": + raise TypeError(f"unsupported broadcast alias for attention input {value_id}: {value.kind}") + return value.tensor + if not isinstance(value, Tensor): + raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") + return value + + +def _resolve_attention_scale(node: IRNode, query: Tensor) -> float: + scale = float(node.attrs.get("scale", 0.0) or 0.0) + if scale > 0.0: + return scale + if len(query.shape) >= 1 and int(query.shape[-1]) > 0: + return float(int(query.shape[-1]) ** -0.5) + return 1.0 + + +def _normalize_attention_mask_for_cactus(g: Graph, mask: Tensor, query: Tensor) -> Tensor: + """Convert broadcastable HF masks into the native Cactus attention contract.""" + + mask_shape = tuple(int(dim) for dim in mask.shape) + query_shape = tuple(int(dim) for dim in query.shape) + if len(query_shape) != 4: + return mask + + batch_size, seq_len, num_heads, _ = query_shape + if len(mask_shape) == 2 and batch_size == 1 and mask_shape[0] == seq_len: + return g.reshape(mask, (1, mask_shape[0], mask_shape[1])) + + if len(mask_shape) == 4: + if ( + mask_shape[0] == batch_size + and mask_shape[1] == 1 + and mask_shape[2] == seq_len + ): + return g.reshape(mask, (mask_shape[0], mask_shape[2], mask_shape[3])) + if ( + mask_shape[0] == batch_size + and mask_shape[1] == num_heads + and mask_shape[2] == seq_len + ): + return mask + + return mask + + +def _lower_projected_attention_tensor( + g: Graph, + hidden: Tensor, + weight: Tensor, + bias: Tensor | None, + target_shape: tuple[int, ...], +) -> Tensor: + if len(target_shape) != 4: + raise NotImplementedError(f"projected attention tensor expects rank-4 target shape, got {target_shape}") + + linear_input = _flatten_to_2d_for_linear(g, hidden) + out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) + if bias is not None: + out = g.add(out, bias) + return g.reshape(out, target_shape) + + +def _normalize_attention_add_tensor(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: + if tuple(tensor.shape) == target_shape: + return tensor + + if len(tensor.shape) != 4 or len(target_shape) != 4: + raise NotImplementedError( + f"unsupported attention add tensor shape {tuple(tensor.shape)} for target {target_shape}" + ) + + batch, seq_len, heads, head_dim = target_shape + shape = tuple(int(v) for v in tensor.shape) + + if shape[0] in (1, batch) and shape[1] == heads and shape[2] in (1, seq_len) and shape[3] == head_dim: + return g.permute(tensor, (0, 2, 1, 3)) + if shape[0] in (1, batch) and shape[1] in (1, seq_len) and shape[2] == heads and shape[3] == head_dim: + return tensor + + raise NotImplementedError( + f"unsupported attention add tensor shape {shape} for target {target_shape}" + ) + + +def _rope_input_is_bhsd(ir: IRGraph, value_id: str) -> bool: + value = ir.values.get(value_id) + if value is None or value.producer is None: + return False + node = ir.nodes.get(value.producer) + if node is None: + return False + if node.op == "permute": + permutation = tuple(int(dim) for dim in node.attrs.get("permutation", ())) + return permutation == (0, 2, 1, 3) + if node.op == "transpose": + return int(node.attrs.get("dim0", -1)) == 1 and int(node.attrs.get("dim1", -1)) == 2 + return False + + +def _lower_softplus(g: Graph, x: Tensor) -> Tensor: + # Stable fp16 softplus: relu(x) + log(1 + exp(-abs(x))). + abs_x = g.abs(x) + neg_abs_x = g.scalar_multiply(abs_x, -1.0) + exp_term = g.scalar_exp(neg_abs_x) + log_term = g.scalar_log(g.scalar_add(exp_term, 1.0)) + return g.add(g.relu(x), log_term) + + +def _normalize_dim(dim: int, rank: int) -> int: + if dim < 0: + dim += rank + return dim + + +def _normalize_reduction_axes(axis: Any, rank: int) -> tuple[int, ...]: + if axis is None: + return tuple(range(rank)) + if isinstance(axis, int): + return (_normalize_dim(axis, rank),) + if isinstance(axis, (list, tuple)): + normalized: list[int] = [] + seen: set[int] = set() + for raw_axis in axis: + if not isinstance(raw_axis, int): + raise NotImplementedError("reduction axes must be integers") + reduced_axis = _normalize_dim(raw_axis, rank) + if reduced_axis in seen: + continue + seen.add(reduced_axis) + normalized.append(reduced_axis) + if normalized: + return tuple(sorted(normalized)) + raise NotImplementedError("reduction axes must be an int, a tuple/list of ints, or None") + + +def _lower_reduction(g: Graph, op: str, x: Tensor, *, axes: tuple[int, ...]) -> Tensor: + if not axes: + return x + + fn = getattr(g, op) + if len(axes) == 1: + return fn(x, axes[0]) + + if op == "variance": + flattened = g.flatten(x, start_dim=axes[0], end_dim=axes[-1]) + collapsed_axes = set(axes) + expected_axes = set(range(axes[0], axes[-1] + 1)) + if collapsed_axes != expected_axes: + raise NotImplementedError("variance currently requires contiguous multi-axis reductions") + return fn(flattened, axes[0]) + + reduced = x + for axis in sorted(axes, reverse=True): + reduced = fn(reduced, axis) + return reduced + + +def _normalize_index(index: int, dim_size: int) -> int: + if index < 0: + index += dim_size + return index + + +def _static_shape(shape: Any) -> tuple[int, ...] | None: + if not isinstance(shape, tuple): + return None + dims: list[int] = [] + for dim in shape: + try: + dims.append(int(dim)) + except Exception: + return None + return tuple(dims) + + +def _ir_value_dtype(ir: IRGraph, value_id: str) -> str | None: + value = ir.values.get(value_id) + if value is None or value.dtype is None: + return None + return str(value.dtype).strip().lower() + + +def _is_integer_index_dtype(dtype: str | None) -> bool: + return dtype in {"uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64"} + + +def _constant_tensor_from_ir(ir: IRGraph, value_id: str) -> torch.Tensor | None: + const = ir.constants.get(value_id) + if isinstance(const, torch.nn.Parameter): + const = const.detach() + if isinstance(const, torch.Tensor): + return const.detach().cpu() + return None + + +def _try_lower_identity_broadcast_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + if len(node.inputs) != 3: + return None + + source = _tensor(env, node.inputs[0]) + source_shape = tuple(int(dim) for dim in source.shape) + if len(source_shape) != 2 or int(source_shape[0]) != 1: + return None + + output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) + if output_shape is None: + return None + + batch_index = _constant_tensor_from_ir(ir, node.inputs[1]) + token_index = _constant_tensor_from_ir(ir, node.inputs[2]) + if batch_index is None or token_index is None: + return None + + try: + batch_index = torch.broadcast_to(batch_index.to(torch.int64), output_shape) + token_index = torch.broadcast_to(token_index.to(torch.int64), output_shape) + except RuntimeError: + return None + + if torch.count_nonzero(batch_index).item() != 0: + return None + + flat_index = token_index.reshape(-1) + expected = torch.arange(int(source_shape[1]), dtype=torch.int64) + if flat_index.numel() != expected.numel() or not torch.equal(flat_index, expected): + return None + + return g.reshape(source, output_shape) + + +def _try_lower_embedding_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + if len(node.inputs) != 2: + return None + if not _is_integer_index_dtype(_ir_value_dtype(ir, node.inputs[1])): + return None + + source = _tensor(env, node.inputs[0]) + indices = _tensor(env, node.inputs[1]) + source_shape = tuple(int(dim) for dim in source.shape) + if len(source_shape) != 2: + return None + + output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) + gathered = g.embedding_from_tensor(source, indices) + if output_shape is not None and tuple(int(dim) for dim in gathered.shape) != tuple(int(dim) for dim in output_shape): + gathered = g.reshape(gathered, output_shape) + return gathered + + +def _try_lower_prefix_mask_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + if len(node.inputs) != 2: + return None + if _is_integer_index_dtype(_ir_value_dtype(ir, node.inputs[1])): + return None + + source = _tensor(env, node.inputs[0]) + mask = _tensor(env, node.inputs[1]) + source_shape = tuple(int(dim) for dim in source.shape) + mask_shape = tuple(int(dim) for dim in mask.shape) + if not mask_shape or len(mask_shape) > len(source_shape): + return None + if source_shape[: len(mask_shape)] != mask_shape: + return None + + output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) + if output_shape: + prefix_elems = 1 + for dim in mask_shape: + prefix_elems *= int(dim) + trailing_shape = source_shape[len(mask_shape) :] + flattened_shape = (prefix_elems, *trailing_shape) + flattened = source if source_shape == flattened_shape else g.reshape(source, flattened_shape) + expected_prefix = int(output_shape[0]) + if 0 <= expected_prefix <= prefix_elems: + selected = g.slice(flattened, axis=0, start=0, length=expected_prefix) + if tuple(int(dim) for dim in selected.shape) != tuple(int(dim) for dim in output_shape): + selected = g.reshape(selected, output_shape) + return selected + + selected = g.masked_select_prefix(source, mask) + if output_shape is None or not output_shape: + return selected + if len(output_shape) != len(selected.shape): + return selected + expected_prefix = int(output_shape[0]) + if expected_prefix < int(selected.shape[0]): + selected = g.slice(selected, axis=0, start=0, length=expected_prefix) + return selected + + +def _try_lower_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + lowered = _try_lower_identity_broadcast_advanced_index(g, node, env, ir) + if lowered is not None: + return lowered + lowered = _try_lower_prefix_mask_advanced_index(g, node, env, ir) + if lowered is not None: + return lowered + return _try_lower_embedding_advanced_index(g, node, env, ir) + + +def _normalize_slice_end(end: int, dim_size: int) -> int: + if end < 0: + end += dim_size + return max(0, min(end, dim_size)) + + +def _legalize_elementwise_binary_inputs(g: Graph, lhs: Tensor, rhs: Tensor) -> tuple[Tensor, Tensor]: + if lhs.shape == rhs.shape: + return lhs, rhs + + lhs_rank = len(lhs.shape) + rhs_rank = len(rhs.shape) + + if lhs_rank > rhs_rank: + rhs = _reshape_for_trailing_broadcast(g, rhs, lhs.shape) + elif rhs_rank > lhs_rank: + lhs = _reshape_for_trailing_broadcast(g, lhs, rhs.shape) + + if lhs.shape == rhs.shape: + return lhs, rhs + + return lhs, rhs + + +def _is_scalar_like(value: Any) -> bool: + return isinstance(value, (int, float, bool)) + + +def _lower_binary_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: + if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): + lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) + target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 + lhs = _ensure_tensor_dtype(g, lhs, target_dtype) + rhs = _ensure_tensor_dtype(g, rhs, target_dtype) + if op == "add": + return g.add(lhs, rhs) + if op == "subtract": + return g.subtract(lhs, rhs) + if op == "multiply": + return g.multiply(lhs, rhs) + if op == "divide": + return g.divide(lhs, rhs) + raise NotImplementedError(f"unsupported binary op: {op}") + + if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): + lhs_value = _ensure_scalar_math_tensor(g, lhs_value) + scalar = float(rhs_value) + if op == "add": + return g.scalar_add(lhs_value, scalar) + if op == "subtract": + return g.scalar_subtract(lhs_value, scalar) + if op == "multiply": + return g.scalar_multiply(lhs_value, scalar) + if op == "divide": + return g.scalar_divide(lhs_value, scalar) + raise NotImplementedError(f"unsupported binary op: {op}") + + if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): + rhs_value = _ensure_scalar_math_tensor(g, rhs_value) + scalar = float(lhs_value) + if op == "add": + return g.scalar_add(rhs_value, scalar) + if op == "subtract": + return g.scalar_add(g.scalar_multiply(rhs_value, -1.0), scalar) + if op == "multiply": + return g.scalar_multiply(rhs_value, scalar) + if op == "divide": + raise NotImplementedError("scalar/tensor divide is not directly supported by Cactus graph ops") + raise NotImplementedError(f"unsupported binary op: {op}") + + raise TypeError( + f"unsupported lowered operand types for {op}: " + f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" + ) + + +def _lower_compare_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: + if op == "equal": + not_equal = _lower_compare_op(g, lhs_value, rhs_value, "not_equal") + return g.scalar_not_equal(not_equal, 1.0) + + if op == "greater": + delta = _lower_binary_op(g, lhs_value, rhs_value, "subtract") + delta = _ensure_fp16_tensor(g, delta) + return g.scalar_not_equal(g.relu(delta), 0.0) + + if op == "less": + return _lower_compare_op(g, rhs_value, lhs_value, "greater") + + if op == "greater_equal": + less = _lower_compare_op(g, lhs_value, rhs_value, "less") + return g.scalar_not_equal(less, 1.0) + + if op == "less_equal": + greater = _lower_compare_op(g, lhs_value, rhs_value, "greater") + return g.scalar_not_equal(greater, 1.0) + + if op != "not_equal": + raise NotImplementedError(f"unsupported compare op: {op}") + + if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): + lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) + target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 + lhs = _ensure_tensor_dtype(g, lhs, target_dtype) + rhs = _ensure_tensor_dtype(g, rhs, target_dtype) + return g.not_equal(lhs, rhs) + + if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): + lhs_value = _ensure_tensor_dtype( + g, + lhs_value, + Graph.FP16 if lhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + return g.scalar_not_equal(lhs_value, float(rhs_value)) + + if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): + rhs_value = _ensure_tensor_dtype( + g, + rhs_value, + Graph.FP16 if rhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + return g.scalar_not_equal(rhs_value, float(lhs_value)) + + if _is_scalar_like(lhs_value) and _is_scalar_like(rhs_value): + result = 1.0 if float(lhs_value) != float(rhs_value) else 0.0 + return _materialize_constant_tensor(g, torch.tensor([result], dtype=torch.float32)) + + raise TypeError( + f"unsupported lowered operand types for {op}: " + f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" + ) + + +def _reshape_for_trailing_broadcast(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: + tensor_shape = tuple(tensor.shape) + target_rank = len(target_shape) + tensor_rank = len(tensor_shape) + + if tensor_rank > target_rank: + return tensor + + padded_shape = (1,) * (target_rank - tensor_rank) + tensor_shape + + # Only legalize cases that are valid trailing broadcasts, e.g.: + # (H,) -> (1, H), (1, 1, H), etc. + for src_dim, tgt_dim in zip(padded_shape, target_shape): + if src_dim != 1 and src_dim != tgt_dim: + return tensor + + if padded_shape == tensor_shape: + return tensor + + return g.reshape(tensor, padded_shape) + + +def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_scalar_math_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_tensor_dtype(g: Graph, tensor: Tensor, dtype: int) -> Tensor: + if tensor.dtype == dtype: + return tensor + return g.precision_cast(tensor, dtype) + + +def _cat_with_legalized_dtype(g: Graph, tensors: list[Tensor], *, axis: int, dtype: int | None) -> Tensor: + if not tensors: + raise NotImplementedError("cat requires at least one tensor") + if dtype is None: + dtypes = {tensor.dtype for tensor in tensors} + dtype = tensors[0].dtype if len(dtypes) == 1 else Graph.FP16 + legalized = [_ensure_tensor_dtype(g, tensor, dtype) for tensor in tensors] + return g.cat(legalized, axis=axis) + + +def _invert_permutation(permutation: tuple[int, ...]) -> tuple[int, ...]: + inverse = [0] * len(permutation) + for index, source_axis in enumerate(permutation): + inverse[int(source_axis)] = index + return tuple(inverse) + + +def _lower_static_strided_slice_via_gather( + g: Graph, + x: Tensor, + *, + axis: int, + indices: list[int], +) -> Tensor: + if not indices: + return g.slice(x, axis=axis, start=0, length=0) + + indices_tensor = _materialize_constant_tensor(g, torch.tensor(indices, dtype=torch.float32)) + if axis == 0: + return g.gather(x, indices_tensor) + + permutation = (axis, *[dim for dim in range(len(x.shape)) if dim != axis]) + transposed = g.permute(x, permutation) + gathered = g.gather(transposed, indices_tensor) + inverse_permutation = _invert_permutation(permutation) + return g.permute(gathered, inverse_permutation) + + +def _lower_where_branch_term(g: Graph, branch_value: Any, mask: Tensor, *, dtype: int | None) -> Tensor: + if dtype is not None: + mask = _ensure_tensor_dtype(g, mask, dtype) + if isinstance(branch_value, Tensor): + branch_mask = mask if dtype is None else _ensure_tensor_dtype(g, mask, branch_value.dtype) + return _lower_binary_op(g, branch_value, branch_mask, "multiply") + return g.scalar_multiply(mask, _clamp_scalar_for_dtype(float(branch_value), mask.dtype)) + + +def _clamp_scalar_for_dtype(value: float, dtype: int | None) -> float: + if dtype != Graph.FP16: + return value + if math.isnan(value): + return value + if value == math.inf: + return 65504.0 + if value == -math.inf: + return -65504.0 + return max(-65504.0, min(65504.0, value)) + + +def _lower_where_op(g: Graph, node: IRNode, env: dict[str, Any]) -> Tensor: + condition = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + false_mask = g.scalar_add(g.scalar_multiply(condition, -1.0), 1.0) + + input_index = 1 + if bool(node.attrs.get("true_is_scalar", False)): + true_value: Any = float(node.attrs["true_value"]) + else: + true_value = env[node.inputs[input_index]] + input_index += 1 + + if bool(node.attrs.get("false_is_scalar", False)): + false_value: Any = float(node.attrs["false_value"]) + else: + false_value = env[node.inputs[input_index]] + + result_dtype: int | None = None + if isinstance(true_value, Tensor): + result_dtype = true_value.dtype + elif isinstance(false_value, Tensor): + result_dtype = false_value.dtype + + true_term = _lower_where_branch_term(g, true_value, condition, dtype=result_dtype) + false_term = _lower_where_branch_term(g, false_value, false_mask, dtype=result_dtype) + return _lower_binary_op(g, true_term, false_term, "add") + + +def _flatten_to_2d_for_linear(g: Graph, tensor: Tensor) -> Tensor: + shape = tuple(tensor.shape) + if len(shape) <= 2: + return tensor + leading = 1 + for dim in shape[:-1]: + leading *= int(dim) + return g.view(tensor, (leading, int(shape[-1]))) + + +def _broadcast_batch_shape(lhs_batch: tuple[int, ...], rhs_batch: tuple[int, ...]) -> tuple[int, ...] | None: + rank = max(len(lhs_batch), len(rhs_batch)) + lhs_padded = (1,) * (rank - len(lhs_batch)) + lhs_batch + rhs_padded = (1,) * (rank - len(rhs_batch)) + rhs_batch + result: list[int] = [] + for lhs_dim, rhs_dim in zip(lhs_padded, rhs_padded): + if lhs_dim == rhs_dim: + result.append(int(lhs_dim)) + elif lhs_dim == 1: + result.append(int(rhs_dim)) + elif rhs_dim == 1: + result.append(int(lhs_dim)) + else: + return None + return tuple(result) + + +def _project_broadcast_coord(coord: tuple[int, ...], operand_batch: tuple[int, ...]) -> tuple[int, ...]: + if not operand_batch: + return () + offset = len(coord) - len(operand_batch) + projected: list[int] = [] + for axis, dim in enumerate(operand_batch): + value = coord[offset + axis] + projected.append(0 if int(dim) == 1 else int(value)) + return tuple(projected) + + +def _index_batch_prefix(g: Graph, tensor: Tensor, coord: tuple[int, ...]) -> Tensor: + for index_value in coord: + tensor = g.index(tensor, index_value, axis=0) + return tensor + + +def _lower_static_batched_matmul( + g: Graph, + lhs: Tensor, + rhs: Tensor, + *, + output_dtype: int | None = None, +) -> Tensor | None: + lhs_shape = tuple(int(dim) for dim in lhs.shape) + rhs_shape = tuple(int(dim) for dim in rhs.shape) + if len(lhs_shape) < 3 or len(rhs_shape) < 3: + return None + if lhs_shape[-1] != rhs_shape[-2]: + return None + + lhs_batch = lhs_shape[:-2] + rhs_batch = rhs_shape[:-2] + batch_shape = _broadcast_batch_shape(lhs_batch, rhs_batch) + if batch_shape is None: + return None + + output_matrix_shape = (int(lhs_shape[-2]), int(rhs_shape[-1])) + rank = len(batch_shape) + + def _build(dim: int, coord_prefix: tuple[int, ...]) -> Tensor: + if dim == rank: + lhs_coord = _project_broadcast_coord(coord_prefix, lhs_batch) + rhs_coord = _project_broadcast_coord(coord_prefix, rhs_batch) + lhs_slice = _index_batch_prefix(g, lhs, lhs_coord) + rhs_slice = _index_batch_prefix(g, rhs, rhs_coord) + out = _matmul_with_quantized_rhs_legalization( + g, + lhs_slice, + rhs_slice, + output_dtype=output_dtype, + ) + return g.reshape(out, (1,) * rank + output_matrix_shape) + + pieces = [_build(dim + 1, coord_prefix + (index_value,)) for index_value in range(int(batch_shape[dim]))] + if len(pieces) == 1: + return pieces[0] + return g.cat(pieces, axis=dim) + + return _build(0, ()) + + +def _should_lower_gemma4_decoder_attention_without_kernel(ir: IRGraph, node: IRNode) -> bool: + if os.environ.get("CACTUS_GEMMA4_DECODER_MANUAL_ATTENTION", "0") != "1": + return False + family = str(ir.meta.get("adapter_family") or ir.meta.get("family") or "").strip().lower() + component = str(ir.meta.get("component", "") or "").strip().lower() + if family != "gemma4" or component != "decoder": + return False + return node.op in {"attention", "scaled_dot_product_attention"} + + +def _lower_gemma4_decoder_attention_without_kernel( + g: Graph, + ir: IRGraph, + env: dict[str, Any], + node: IRNode, +) -> Tensor: + query = _attention_tensor(env, node.inputs[0]) + key = _attention_tensor(env, node.inputs[1]) + value = _attention_tensor(env, node.inputs[2]) + + key_transposed = g.permute(key, (0, 1, 3, 2)) + scores = _lower_static_batched_matmul(g, query, key_transposed, output_dtype=Graph.FP16) + if scores is None: + raise NotImplementedError("Gemma4 decoder attention requires static batched matmul support") + + scale = float(node.attrs.get("scale", 1.0)) + if scale != 1.0: + scores = g.scalar_multiply(_ensure_scalar_math_tensor(g, scores), scale) + + additive_mask: Tensor | None = None + if len(node.inputs) > 3: + mask_tensor = _ensure_fp16_tensor(g, _tensor(env, node.inputs[3])) + if bool(node.attrs.get("additive_mask", False)): + additive_mask = mask_tensor + else: + additive_mask = _lower_binary_op(g, mask_tensor, 1.0, "subtract") + additive_mask = g.scalar_multiply(_ensure_scalar_math_tensor(g, additive_mask), 1.0e4) + else: + query_shape = tuple(int(dim) for dim in query.shape) + key_shape = tuple(int(dim) for dim in key.shape) + if len(query_shape) >= 4 and len(key_shape) >= 4: + query_seq = int(query_shape[-2]) + key_seq = int(key_shape[-2]) + additive_mask = _materialize_gemma4_attention_mask( + g, + query_seq=query_seq, + key_seq=key_seq, + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + dtype=torch.float16, + ) + + if additive_mask is not None: + scores = _lower_binary_op(g, scores, additive_mask, "add") + + probs = g.softmax(scores, axis=-1) + probs = _ensure_tensor_dtype(g, probs, value.dtype) + out = _lower_static_batched_matmul(g, probs, value, output_dtype=value.dtype) + if out is None: + raise NotImplementedError("Gemma4 decoder attention output requires static batched matmul support") + + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else out.dtype + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return out + + +def _materialize_gemma4_attention_mask( + g: Graph, + *, + query_seq: int, + key_seq: int, + is_causal: bool, + window_size: int, + dtype: torch.dtype, +) -> Tensor | None: + if query_seq <= 0 or key_seq <= 0: + return None + if not is_causal and window_size <= 0: + return None + + q_index = np.arange(query_seq, dtype=np.int32)[:, None] + k_index = np.arange(key_seq, dtype=np.int32)[None, :] + allowed = np.ones((query_seq, key_seq), dtype=np.bool_) + + if is_causal: + allowed &= k_index <= q_index + if window_size > 0 and window_size < key_seq: + allowed &= k_index >= (q_index - (window_size - 1)) + + mask = np.where(allowed, 0.0, -1.0e4).astype(np.float16) + tensor = torch.from_numpy(mask.reshape(1, 1, query_seq, key_seq)).to(dtype=dtype) + return _materialize_constant_tensor(g, tensor) + + +def _legalize_matmul_inputs( + g: Graph, + lhs: Tensor, + rhs: Tensor, + node: IRNode, +) -> tuple[Tensor, Tensor, tuple[int, ...]] | None: + lhs_shape = tuple(lhs.shape) + rhs_shape = tuple(rhs.shape) + if len(lhs_shape) <= 2 and len(rhs_shape) <= 2: + return None + + if len(lhs_shape) > 2 and len(rhs_shape) == 2 and lhs_shape[-1] == rhs_shape[0]: + leading = 1 + for dim in lhs_shape[:-1]: + leading *= int(dim) + lhs_2d = g.reshape(lhs, (leading, int(lhs_shape[-1]))) + output_shape = lhs_shape[:-1] + (int(rhs_shape[1]),) + return lhs_2d, rhs, output_shape + + # Cactus matmul is 2D-only. Legalize the narrow case where both operands have + # only singleton leading dims, e.g. rotary helper matmuls: + # (1, M, K) @ (1, K, N) -> reshape to (M, K) @ (K, N) -> reshape back. + if any(dim != 1 for dim in lhs_shape[:-2]): + return None + if any(dim != 1 for dim in rhs_shape[:-2]): + return None + + lhs_2d_shape = lhs_shape[-2:] + rhs_2d_shape = rhs_shape[-2:] + if lhs_2d_shape[-1] != rhs_2d_shape[0]: + return None + + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + output_shape = lhs_shape[:-2] + (lhs_2d_shape[0], rhs_2d_shape[1]) + output_shape = tuple(int(v) for v in output_shape) + + lhs_2d = g.reshape(lhs, lhs_2d_shape) + rhs_2d = g.reshape(rhs, rhs_2d_shape) + return lhs_2d, rhs_2d, output_shape + + +def _resolve_expand_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: + if len(requested_shape) < len(input_shape): + raise NotImplementedError(f"expand cannot reduce rank: {input_shape} -> {requested_shape}") + + padded_input = (1,) * (len(requested_shape) - len(input_shape)) + input_shape + resolved: list[int] = [] + for in_dim, req_dim in zip(padded_input, requested_shape): + if req_dim == -1: + resolved.append(in_dim) + continue + if req_dim < -1: + raise NotImplementedError(f"invalid expand dimension: {req_dim}") + resolved.append(int(req_dim)) + return tuple(resolved) + + +def _lower_repeat(g: Graph, x: Tensor, repeats: tuple[int, ...]) -> Tensor: + if not repeats: + return x + + input_shape = tuple(int(dim) for dim in x.shape) + if len(repeats) < len(input_shape): + raise NotImplementedError(f"repeat cannot reduce rank: {input_shape} with repeats={repeats}") + + if len(repeats) > len(input_shape): + padded_shape = (1,) * (len(repeats) - len(input_shape)) + input_shape + x = g.reshape(x, padded_shape) + + result = x + for axis, factor in enumerate(repeats): + if factor < 0: + raise NotImplementedError(f"repeat does not support negative factors: {repeats}") + if factor == 0: + raise NotImplementedError(f"repeat does not support zero factors: {repeats}") + if factor == 1: + continue + result = g.cat([result] * factor, axis=axis) + return result + + +def _resolve_reshape_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: + resolved = [int(v) for v in requested_shape] + unknown_indices = [idx for idx, dim in enumerate(resolved) if dim == -1] + if len(unknown_indices) > 1: + raise NotImplementedError(f"reshape with multiple inferred dimensions is unsupported: {requested_shape}") + if not unknown_indices: + return tuple(resolved) + + input_elements = 1 + for dim in input_shape: + input_elements *= int(dim) + + known_elements = 1 + for dim in resolved: + if dim != -1: + known_elements *= int(dim) + + if known_elements == 0 or input_elements % known_elements != 0: + raise NotImplementedError(f"cannot infer reshape target {requested_shape} from input shape {input_shape}") + + resolved[unknown_indices[0]] = input_elements // known_elements + return tuple(resolved) + + +def _match_gqa_expand_alias(ir: IRGraph, expand_node: IRNode) -> str | None: + target_shape = tuple(int(v) for v in expand_node.attrs["shape"]) + if len(target_shape) != 5: + return None + + current_value_id = expand_node.inputs[0] + base_value_id: str | None = None + + while True: + producer_id = ir.values[current_value_id].producer + if producer_id is None: + return None + producer = ir.nodes[producer_id] + if producer.op == "slice": + axis = int(producer.attrs.get("axis", -1)) + input_shape = ir.values[producer.inputs[0]].shape + if input_shape is None: + return None + normalized_axis = _normalize_dim(axis, len(input_shape)) + start = int(producer.attrs.get("start", 0)) + end = int(producer.attrs.get("end", input_shape[normalized_axis])) + if start != 0 or end < input_shape[normalized_axis]: + return None + current_value_id = producer.inputs[0] + continue + if producer.op == "unsqueeze": + base_value_id = producer.inputs[0] + base_shape = ir.values[base_value_id].shape + if base_shape is None: + return None + unsqueezed_dim = _normalize_dim(int(producer.attrs.get("dim", 0)), len(base_shape) + 1) + if unsqueezed_dim != 2: + return None + break + if producer.op in {"reshape", "view"}: + base_value_id = producer.inputs[0] + base_shape = ir.values[base_value_id].shape + view_shape = ir.values[current_value_id].shape + if base_shape is None or view_shape is None: + return None + if len(base_shape) != 4 or len(view_shape) != 5: + return None + if tuple(int(v) for v in view_shape) != ( + int(base_shape[0]), + int(base_shape[1]), + 1, + int(base_shape[2]), + int(base_shape[3]), + ): + return None + break + return None + + if base_value_id is None: + return None + base_shape = ir.values[base_value_id].shape + if base_shape is None or len(base_shape) != 4: + return None + + expected_target = ( + int(base_shape[0]), + int(base_shape[1]), + int(target_shape[2]), + int(base_shape[2]), + int(base_shape[3]), + ) + if target_shape != expected_target: + return None + if int(target_shape[2]) <= 1: + return None + + return base_value_id + + +def _map_ir_dtype(dtype: str) -> int: + if dtype == "bf16": + return Graph.FP32 + if dtype == "fp16": + return Graph.FP16 + if dtype in ("fp32", "fp64"): + return Graph.FP32 + if dtype == "int8": + return Graph.INT8 + if dtype in ("int32", "int64", "bool"): + return Graph.FP32 + + raise NotImplementedError(f"unsupported IR dtype: {dtype}") + + +def _map_ir_or_torch_dtype(dtype: Any) -> int: + if dtype is None: + raise NotImplementedError("missing dtype for precision_cast") + if isinstance(dtype, str): + if dtype.startswith("torch."): + return _map_torch_dtype(getattr(torch, dtype.split(".", 1)[1])) + return _map_ir_dtype(dtype) + return _map_torch_dtype(dtype) + + +def _map_torch_dtype(dtype: Any) -> int: + if dtype == torch.bfloat16: + return Graph.FP32 + if dtype == torch.float16: + return Graph.FP16 + if dtype == torch.float32: + return Graph.FP32 + if dtype == torch.float64: + return Graph.FP32 + if dtype == torch.int8: + return Graph.INT8 + if dtype in (torch.int16, torch.int32, torch.int64, torch.bool): + return Graph.FP32 + raise NotImplementedError(f"unsupported torch dtype: {dtype}") + + +def _materialize_constant_tensor(g: Graph, tensor_value: torch.Tensor) -> Tensor: + graph_dtype = _map_torch_dtype(tensor_value.dtype) + materialized = tensor_value.detach().cpu() + if graph_dtype == Graph.FP32 and materialized.dtype not in (torch.float32, torch.float64): + materialized = materialized.to(torch.float32) + elif graph_dtype == Graph.FP16 and materialized.dtype != torch.float16: + materialized = materialized.to(torch.float16) + elif graph_dtype == Graph.INT8 and materialized.dtype != torch.int8: + materialized = materialized.to(torch.int8) + + tensor = g.input(shape=tuple(materialized.shape), dtype=graph_dtype) + g.set_input(tensor, materialized, dtype=graph_dtype) + materialized_constants = getattr(g, "_transpile_materialized_constants", None) + if isinstance(materialized_constants, list): + materialized_constants.append(tensor) + return tensor + + +def _materialize_constant_torch_dtype(dtype: Any) -> torch.dtype: + if dtype is None: + return torch.float32 + if isinstance(dtype, str): + if dtype.startswith("torch."): + return getattr(torch, dtype.split(".", 1)[1]) + if dtype in ("fp16", "bf16"): + return torch.float16 + if dtype in ("fp32", "fp64", "int16", "int32", "int64", "bool"): + return torch.float32 + if dtype == "int8": + return torch.int8 + if isinstance(dtype, torch.dtype): + return dtype + raise NotImplementedError(f"unsupported dtype for constant materialization: {dtype}") + + +def _torch_dtype_for_graph_dtype(dtype: int) -> torch.dtype: + if dtype == Graph.FP16: + return torch.float16 + if dtype == Graph.FP32: + return torch.float32 + if dtype == Graph.INT8: + return torch.int8 + raise NotImplementedError(f"unsupported graph dtype for constant materialization: {dtype}") diff --git a/python/cactus/transpile/media_limits.py b/python/cactus/transpile/media_limits.py new file mode 100644 index 000000000..b413a5c7a --- /dev/null +++ b/python/cactus/transpile/media_limits.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import os + +_DEFAULT_IMAGE_SIZE = 256 + + +def static_image_size() -> int: + raw = os.environ.get("CACTUS_TRANSPILER_IMAGE_SIZE", str(_DEFAULT_IMAGE_SIZE)) + try: + return max(1, int(raw)) + except (TypeError, ValueError): + return _DEFAULT_IMAGE_SIZE + + +def resize_static_image(image: object) -> object: + size = static_image_size() + target = (size, size) + if getattr(image, "size", None) == target or not hasattr(image, "resize"): + return image + try: + from PIL import Image # type: ignore + + resample = Image.Resampling.BILINEAR + except AttributeError: # pragma: no cover + resample = Image.BILINEAR # type: ignore[name-defined] + return image.resize(target, resample=resample) diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py new file mode 100644 index 000000000..1cf61d3c5 --- /dev/null +++ b/python/cactus/transpile/model_adapters.py @@ -0,0 +1,3836 @@ +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import inspect +import struct +import os +from pathlib import Path +from typing import Callable + +import numpy as np +import torch +import torch.nn.functional as F + +from cactus.transpile.component_pipeline import ComponentModuleSpec +from cactus.convert.cactus_adapters.tensor_io import CACTUS_MAGIC +from cactus.convert.cactus_adapters.tensor_io import align_offset + + +_GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE = 1.0 / 64.0 +try: + from transformers.models.gemma4.modeling_gemma4 import create_bidirectional_mask as _GEMMA4_CREATE_BIDIRECTIONAL_MASK # type: ignore +except Exception: + try: + from transformers.masking_utils import create_bidirectional_mask as _GEMMA4_CREATE_BIDIRECTIONAL_MASK # type: ignore + except Exception: + _GEMMA4_CREATE_BIDIRECTIONAL_MASK = None + + +@dataclass +class CanonicalizedModel: + module: torch.nn.Module + task: str + family: str + input_names: tuple[str, ...] = () + + +@dataclass(frozen=True) +class _Gemma4NativeMergeSegment: + kind: str + input_start: int + length: int + feature_start: int = 0 + + +@dataclass(frozen=True) +class _Gemma4NativeMergePlan: + segments: tuple[_Gemma4NativeMergeSegment, ...] + pli_token_ids: tuple[int, ...] + + +def _model_name_or_path(model: torch.nn.Module) -> str: + value = getattr(model, "name_or_path", None) + if isinstance(value, str) and value: + return value + config = getattr(model, "config", None) + value = getattr(config, "_name_or_path", None) + if isinstance(value, str) and value: + return value + return "" + + +def _transpile_graph_meta(model: torch.nn.Module, *, adapter_family: str, adapter_type: str, input_names: tuple[str, ...]) -> dict[str, object]: + return { + "adapter_family": adapter_family, + "adapter_type": adapter_type, + "model_name_or_path": _model_name_or_path(model), + "input_names": input_names, + } + + +def _extract_tensor_output(output: object, *, preferred_field: str | None = None) -> torch.Tensor: + if preferred_field is not None: + value = getattr(output, preferred_field, None) + if isinstance(value, torch.Tensor): + return value + + if isinstance(output, torch.Tensor): + return output + if isinstance(output, tuple) and output and isinstance(output[0], torch.Tensor): + return output[0] + + for field_name in ("last_hidden_state", "logits"): + value = getattr(output, field_name, None) + if isinstance(value, torch.Tensor): + return value + + raise TypeError(f"could not extract tensor output from {type(output).__name__}") + + +def _gemma4_get_placeholder_masks( + get_placeholder_mask: Callable[..., object], + *, + token_type_ids: torch.Tensor | None, + input_ids: torch.Tensor, + inputs_embeds: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + kwargs: dict[str, torch.Tensor] = { + "input_ids": input_ids, + "inputs_embeds": inputs_embeds, + } + try: + signature = inspect.signature(get_placeholder_mask) + except (TypeError, ValueError): + signature = None + if token_type_ids is not None and signature is not None: + parameters = signature.parameters + accepts_kwargs = any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in parameters.values() + ) + if accepts_kwargs or "token_type_ids" in parameters: + kwargs["token_type_ids"] = token_type_ids + + result = get_placeholder_mask(**kwargs) + if not isinstance(result, tuple) or len(result) != 3: + raise TypeError( + "Gemma4 get_placeholder_mask must return " + "(text_mask, image_mask, audio_mask)" + ) + text_mask, image_mask, audio_mask = result + if not ( + isinstance(text_mask, torch.Tensor) + and isinstance(image_mask, torch.Tensor) + and isinstance(audio_mask, torch.Tensor) + ): + raise TypeError("Gemma4 get_placeholder_mask returned non-tensor masks") + return text_mask, image_mask, audio_mask + + +def _module_or_config_attr(module: object, name: str, default: object | None = None) -> object: + value = getattr(module, name, None) + if value is not None: + return value + config = getattr(module, "config", None) + value = getattr(config, name, None) + if value is not None: + return value + return default + + +def _gemma4_get_per_layer_inputs( + backbone: torch.nn.Module, + input_ids: torch.Tensor, + inputs_embeds: torch.Tensor, +) -> torch.Tensor: + get_per_layer_inputs = getattr(backbone, "get_per_layer_inputs") + try: + signature = inspect.signature(get_per_layer_inputs) + except (TypeError, ValueError): + signature = None + if signature is not None and "inputs_embeds" not in signature.parameters: + return get_per_layer_inputs(input_ids) + return get_per_layer_inputs(input_ids, inputs_embeds) + + +def _select_last_active_token(hidden_or_logits: torch.Tensor, attention_mask: torch.Tensor | None) -> torch.Tensor: + if attention_mask is None or hidden_or_logits.ndim < 3: + return hidden_or_logits[:, -1:, ...] + + if attention_mask.ndim != 2: + raise ValueError(f"expected 2D attention mask, got shape {tuple(attention_mask.shape)}") + if attention_mask.shape[1] != hidden_or_logits.shape[1]: + raise ValueError( + "attention mask / hidden sequence length mismatch: " + f"{tuple(attention_mask.shape)} vs {tuple(hidden_or_logits.shape)}" + ) + + trailing_zero = attention_mask[:, :1] - attention_mask[:, :1] + shifted_mask = torch.cat((attention_mask[:, 1:], trailing_zero), dim=1) + last_active_mask = torch.logical_and(attention_mask != 0, shifted_mask == 0) + expanded_mask = last_active_mask.to(dtype=hidden_or_logits.dtype) + for _ in range(hidden_or_logits.ndim - 2): + expanded_mask = expanded_mask.unsqueeze(-1) + return (hidden_or_logits * expanded_mask).sum(dim=1, keepdim=True) + + +def _select_last_non_pad_token( + hidden_or_logits: torch.Tensor, + input_ids: torch.Tensor | None, + *, + pad_token_id: int | None, +) -> torch.Tensor: + if hidden_or_logits.ndim < 3: + return hidden_or_logits + if not isinstance(input_ids, torch.Tensor) or input_ids.ndim != 2: + return hidden_or_logits[:, -1:, ...] + if input_ids.shape[1] != hidden_or_logits.shape[1]: + raise ValueError( + "input id / hidden sequence length mismatch: " + f"{tuple(input_ids.shape)} vs {tuple(hidden_or_logits.shape)}" + ) + if pad_token_id is None: + return hidden_or_logits[:, -1:, ...] + attention_mask = (input_ids != int(pad_token_id)).to(dtype=torch.int64) + return _select_last_active_token(hidden_or_logits, attention_mask) + + +def _resolve_model_pad_token_id(model: torch.nn.Module) -> int | None: + config = getattr(model, "config", None) + for attr_name in ("pad_token_id", "eos_token_id", "bos_token_id"): + value = getattr(config, attr_name, None) + if value is not None: + return int(value) + generation_config = getattr(model, "generation_config", None) + for attr_name in ("pad_token_id", "eos_token_id", "bos_token_id"): + value = getattr(generation_config, attr_name, None) + if value is not None: + return int(value) + return None + + +def _filter_supported_kwargs(module: torch.nn.Module, kwargs: dict[str, object]) -> dict[str, object]: + try: + signature = inspect.signature(module.forward) + except (TypeError, ValueError): + return kwargs + accepted = signature.parameters + supports_var_kwargs = any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in accepted.values() + ) + if supports_var_kwargs: + return kwargs + return {name: value for name, value in kwargs.items() if name in accepted} + + +def _module_floating_dtype(module: torch.nn.Module) -> torch.dtype | None: + for parameter in module.parameters(): + if parameter.is_floating_point(): + return parameter.dtype + for buffer in module.buffers(): + if buffer.is_floating_point(): + return buffer.dtype + return None + + +def _module_device(module: torch.nn.Module) -> torch.device | None: + for parameter in module.parameters(): + return parameter.device + for buffer in module.buffers(): + return buffer.device + return None + + +def _unique_modules(*candidates: object) -> tuple[torch.nn.Module, ...]: + modules: list[torch.nn.Module] = [] + seen: set[int] = set() + for candidate in candidates: + if not isinstance(candidate, torch.nn.Module): + continue + candidate_id = id(candidate) + if candidate_id in seen: + continue + seen.add(candidate_id) + modules.append(candidate) + return tuple(modules) + + +def _gemma4_vision_modules(multimodal_backbone: torch.nn.Module) -> tuple[torch.nn.Module, ...]: + return _unique_modules( + getattr(multimodal_backbone, "vision_tower", None), + getattr(multimodal_backbone, "embed_vision", None), + ) + + +def _gemma4_audio_modules(multimodal_backbone: torch.nn.Module) -> tuple[torch.nn.Module, ...]: + return _unique_modules( + getattr(multimodal_backbone, "audio_tower", None), + getattr(multimodal_backbone, "embed_audio", None), + ) + + +def _gemma4_text_config(multimodal_backbone: torch.nn.Module) -> object: + config = getattr(multimodal_backbone, "config", None) + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + try: + return get_text_config() + except Exception: + pass + return config + + +def _gemma4_special_token_ids(multimodal_backbone: torch.nn.Module) -> tuple[int, int, int]: + config = getattr(multimodal_backbone, "config", None) + text_config = _gemma4_text_config(multimodal_backbone) + image_token_id = int(getattr(config, "image_token_id", 0) or 0) + audio_token_id = int(getattr(config, "audio_token_id", 0) or 0) + pad_token_id = getattr(text_config, "pad_token_id", None) + if pad_token_id is None: + pad_token_id = getattr(config, "pad_token_id", 0) + return image_token_id, audio_token_id, int(pad_token_id or 0) + + +@contextmanager +def _temporary_cpu_float32_modules(modules: tuple[torch.nn.Module, ...]): + promoted: list[tuple[torch.nn.Module, torch.dtype]] = [] + for module in modules: + device = _module_device(module) + dtype = _module_floating_dtype(module) + if device is None or device.type != "cpu" or dtype is None or dtype == torch.float32: + continue + promoted.append((module, dtype)) + module.to(dtype=torch.float32) + try: + yield + finally: + for module, dtype in reversed(promoted): + module.to(dtype=dtype) + + +def _torch_is_compiling() -> bool: + dynamo = getattr(torch, "_dynamo", None) + is_compiling = getattr(dynamo, "is_compiling", None) + if callable(is_compiling): + try: + return bool(is_compiling()) + except Exception: + return False + return False + + +def _gemma4_cpu_safe_text_mlp_enabled(layer: torch.nn.Module, hidden_states: torch.Tensor) -> bool: + if hidden_states.device.type != "cpu" or hidden_states.dtype != torch.float16: + return False + if bool(getattr(layer, "enable_moe_block", False)): + return False + mlp = getattr(layer, "mlp", None) + if mlp is None: + return False + for attr_name in ("gate_proj", "up_proj", "down_proj", "act_fn"): + if not hasattr(mlp, attr_name): + return False + return True + + +def _gemma4_cpu_safe_text_mlp_forward(mlp: torch.nn.Module, hidden_states: torch.Tensor) -> torch.Tensor: + gate = F.linear( + hidden_states, + mlp.gate_proj.weight, + mlp.gate_proj.bias, + ) + up = F.linear( + hidden_states, + mlp.up_proj.weight, + mlp.up_proj.bias, + ) + # Gemma4 text MLPs can overflow in FP16 at the gated product. + # Scaling one branch here keeps the product and down-projection finite. + # The following post-feedforward RMSNorm is scale-invariant, so this + # constant factor cancels back out in the normalized output. + activated = mlp.act_fn(gate) * _GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE + return F.linear( + activated * up, + mlp.down_proj.weight, + mlp.down_proj.bias, + ) + + +def _gemma4_text_attention_forward( + attn: torch.nn.Module, + hidden_states: torch.Tensor, + *, + position_embeddings: torch.Tensor | tuple[torch.Tensor, torch.Tensor] | None, + attention_mask: torch.Tensor | None, + position_ids: torch.LongTensor | None, + past_key_values: object | None, + use_cache: bool, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None, +) -> torch.Tensor: + if shared_kv_states is None: + attn_out, _ = attn( + hidden_states=hidden_states, + position_embeddings=position_embeddings, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + ) + return attn_out + + from transformers.models.gemma4.modeling_gemma4 import ALL_ATTENTION_FUNCTIONS # type: ignore + from transformers.models.gemma4.modeling_gemma4 import apply_rotary_pos_emb # type: ignore + from transformers.models.gemma4.modeling_gemma4 import eager_attention_forward # type: ignore + + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, attn.head_dim) + if position_embeddings is None: + raise ValueError("Gemma4 text attention requires position embeddings") + cos, sin = position_embeddings + + query_states = attn.q_proj(hidden_states).view(hidden_shape) + query_states = attn.q_norm(query_states) + query_states = apply_rotary_pos_emb(query_states, cos, sin, unsqueeze_dim=2) + query_states = query_states.transpose(1, 2) + + kv_shared_layer_index = getattr(attn, "kv_shared_layer_index", None) + if bool(getattr(attn, "is_kv_shared_layer", False)) and kv_shared_layer_index in shared_kv_states: + key_states, value_states = shared_kv_states[int(kv_shared_layer_index)] + key_states = key_states.to(query_states.device) + value_states = value_states.to(query_states.device) + else: + key_states = attn.k_proj(hidden_states).view(hidden_shape) + value_states = attn.v_proj(hidden_states).view(hidden_shape) if attn.v_proj is not None else key_states + + key_states = attn.k_norm(key_states) + key_states = apply_rotary_pos_emb(key_states, cos, sin, unsqueeze_dim=2) + key_states = key_states.transpose(1, 2) + + value_states = attn.v_norm(value_states) + value_states = value_states.transpose(1, 2) + + if bool(getattr(attn, "store_full_length_kv", False)): + shared_kv_states[int(attn.layer_idx)] = (key_states, value_states) + + attention_interface = eager_attention_forward + if getattr(attn.config, "_attn_implementation", "eager") != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[attn.config._attn_implementation] + + attn_output, _ = attention_interface( + attn, + query_states, + key_states, + value_states, + attention_mask, + dropout=attn.attention_dropout if attn.training else 0.0, + scaling=attn.scaling, + sliding_window=attn.sliding_window, + ) + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + return attn.o_proj(attn_output) + + +def _gemma4_text_decoder_layer_forward( + layer: torch.nn.Module, + hidden_states: torch.Tensor, + *, + position_embeddings: torch.Tensor | tuple[torch.Tensor, torch.Tensor] | None = None, + per_layer_input: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: object | None = None, + use_cache: bool = False, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, +) -> torch.Tensor: + if not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): + return layer( + hidden_states, + position_embeddings=position_embeddings, + per_layer_input=per_layer_input, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + ) + + residual = hidden_states + + hidden_states = layer.input_layernorm(hidden_states) + hidden_states = _gemma4_text_attention_forward( + layer.self_attn, + hidden_states, + position_embeddings=position_embeddings, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + shared_kv_states=shared_kv_states, + ) + hidden_states = layer.post_attention_layernorm(hidden_states) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = layer.pre_feedforward_layernorm(hidden_states) + mlp_out = _gemma4_cpu_safe_text_mlp_forward(layer.mlp, hidden_states) + hidden_states = layer.post_feedforward_layernorm(mlp_out) + hidden_states = residual.float() + hidden_states + hidden_states = hidden_states.to(dtype=residual.dtype) + + if getattr(layer, "hidden_size_per_layer_input", 0): + residual = hidden_states + hidden_states = layer.per_layer_input_gate(hidden_states) + hidden_states = layer.act_fn(hidden_states) + if per_layer_input is None: + raise ValueError("Gemma4 layer expected per_layer_input but none was provided") + hidden_states = torch.multiply(hidden_states, per_layer_input) + hidden_states = layer.per_layer_projection(hidden_states) + hidden_states = layer.post_per_layer_input_norm(hidden_states) + hidden_states = residual + hidden_states + + hidden_states = hidden_states * layer.layer_scalar + return hidden_states + + +def _gemma4_text_backbone_forward( + backbone: torch.nn.Module, + *, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor | None, + causal_mask_mapping: dict[str, torch.Tensor], + position_ids: torch.LongTensor, +) -> torch.Tensor: + hidden_states = inputs_embeds + layer_types = tuple(dict.fromkeys(getattr(backbone.config, "layer_types", ()))) + position_embeddings = { + layer_type: backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in layer_types + } + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + + config_layer_types = tuple(getattr(backbone.config, "layer_types", ())) + for layer_index, decoder_layer in enumerate(backbone.layers[: backbone.config.num_hidden_layers]): + layer_per_input = None + if per_layer_inputs is not None: + layer_per_input = per_layer_inputs[:, :, decoder_layer.layer_idx, :] + attention_type = getattr( + decoder_layer, + "attention_type", + config_layer_types[layer_index] if layer_index < len(config_layer_types) else "full_attention", + ) + hidden_states = _gemma4_text_decoder_layer_forward( + decoder_layer, + hidden_states, + per_layer_input=layer_per_input, + attention_mask=causal_mask_mapping[attention_type], + position_embeddings=position_embeddings[attention_type], + position_ids=position_ids, + past_key_values=None, + use_cache=False, + shared_kv_states=shared_kv_states, + ) + + return backbone.norm(hidden_states) + + +def _gemma4_strip_audio_padding(audio_output: object) -> torch.Tensor: + audio_features = getattr(audio_output, "pooler_output", None) + audio_mask_from_encoder = getattr(audio_output, "audio_mel_mask", None) + if not isinstance(audio_features, torch.Tensor): + raise TypeError("Gemma4 audio output did not expose tensor pooler_output") + if not isinstance(audio_mask_from_encoder, torch.Tensor): + return audio_features + all_real_tokens: list[torch.Tensor] = [] + for encodings, padding_mask in zip(audio_features, audio_mask_from_encoder, strict=True): + all_real_tokens.append(encodings[~padding_mask]) + return torch.cat(all_real_tokens, dim=0).unsqueeze(0) + + +def _gemma4_rms_norm_no_scale(hidden_states: torch.Tensor, *, eps: float) -> torch.Tensor: + hidden_states_f32 = hidden_states.float() + mean_squared = hidden_states_f32.pow(2).mean(-1, keepdim=True) + eps + return hidden_states_f32 * torch.pow(mean_squared, -0.5) + + +def _gemma4_rms_norm(hidden_states: torch.Tensor, weight: torch.Tensor, *, eps: float) -> torch.Tensor: + normed = _gemma4_rms_norm_no_scale(hidden_states, eps=eps) + return normed * weight.to(device=normed.device, dtype=normed.dtype) + + +def _gemma4_layer_norm_eps(multimodal_backbone: torch.nn.Module) -> float: + config = getattr(multimodal_backbone, "config", None) + text_config = None + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + try: + text_config = get_text_config() + except Exception: + text_config = None + for candidate in (text_config, config): + value = getattr(candidate, "rms_norm_eps", None) + if value is not None: + return float(value) + value = getattr(candidate, "layer_norm_eps", None) + if value is not None: + return float(value) + return 1e-6 + + +def _gemma4_load_cactus_fp_tensor(path: Path) -> torch.Tensor: + with path.open("rb") as handle: + header = handle.read(84) + if len(header) < 84 or header[:4] != CACTUS_MAGIC: + raise RuntimeError(f"Gemma4 Cactus tensor is missing a valid header: {path}") + + alignment = max(1, int(struct.unpack_from(" 0) + precision = int(struct.unpack_from(" 0 else 0 + data_offset = ( + align_offset(scales_offset + scales_bytes, alignment) + if scales_bytes > 0 + else aligned_header + ) + data_count = byte_size // np.dtype(dtype).itemsize + array = np.memmap(path, mode="r", dtype=dtype, offset=data_offset, shape=(data_count,)) + tensor = torch.from_numpy(np.array(array, copy=True)) + if shape: + tensor = tensor.reshape(shape) + return tensor + + +def _gemma4_load_vision_post_proj_norm(weights_dir: str | Path | None) -> torch.Tensor | None: + if weights_dir is None: + return None + path = Path(weights_dir).expanduser() / "embed_vision_post_proj_norm.weights" + if not path.exists(): + return None + tensor = _gemma4_load_cactus_fp_tensor(path) + if tensor.ndim != 1: + raise RuntimeError(f"Gemma4 vision post-projection norm must be 1D, got shape={tuple(tensor.shape)}") + return tensor.float() + + +def _gemma4_can_use_native_like_vision_features(multimodal_backbone: torch.nn.Module) -> bool: + vision_tower = getattr(multimodal_backbone, "vision_tower", None) + embed_vision = getattr(multimodal_backbone, "embed_vision", None) + return ( + isinstance(vision_tower, torch.nn.Module) + and isinstance(embed_vision, torch.nn.Module) + and hasattr(vision_tower, "patch_embedder") + and hasattr(vision_tower, "encoder") + and hasattr(embed_vision, "embedding_projection") + ) + + +def _gemma4_can_use_native_like_audio_features(multimodal_backbone: torch.nn.Module) -> bool: + audio_tower = getattr(multimodal_backbone, "audio_tower", None) + embed_audio = getattr(multimodal_backbone, "embed_audio", None) + layers = getattr(audio_tower, "layers", None) if isinstance(audio_tower, torch.nn.Module) else None + conformer = getattr(audio_tower, "conformer", None) if isinstance(audio_tower, torch.nn.Module) else None + return ( + isinstance(audio_tower, torch.nn.Module) + and isinstance(embed_audio, torch.nn.Module) + and hasattr(audio_tower, "subsample_conv_projection") + and (isinstance(layers, torch.nn.ModuleList) or isinstance(conformer, torch.nn.ModuleList)) + and hasattr(embed_audio, "embedding_projection") + ) + + +def _gemma4_pool_vision_hidden_native_like( + vision_tower: torch.nn.Module, + hidden_states: torch.Tensor, + pixel_position_ids: torch.Tensor, + *, + image_soft_token_counts: tuple[int, ...] | None = None, + image_pool_shapes: tuple[tuple[int, int, int], ...] | None = None, +) -> torch.Tensor: + output_length = int(_module_or_config_attr(vision_tower, "default_output_length", 280) or 280) + pooling_kernel_size = int(_module_or_config_attr(vision_tower, "pooling_kernel_size", 3) or 3) + padding_positions = (pixel_position_ids == -1).all(dim=-1) + pooled_batches: list[torch.Tensor] = [] + for row_idx, (hidden_row, position_row, padding_row) in enumerate( + zip(hidden_states, pixel_position_ids, padding_positions, strict=True) + ): + if image_pool_shapes is not None and row_idx < len(image_pool_shapes): + grid_h, grid_w, pooled_count = image_pool_shapes[row_idx] + valid_patch_count = int(grid_h) * int(grid_w) + channels = int(hidden_row.shape[-1]) + valid_hidden = hidden_row[:valid_patch_count].float() + pooled = valid_hidden.reshape( + int(grid_h) // pooling_kernel_size, + pooling_kernel_size, + int(grid_w) // pooling_kernel_size, + pooling_kernel_size, + channels, + ).mean(dim=(1, 3)) + pooled_batches.append(pooled.reshape(int(pooled_count), channels)) + continue + + # Avoid boolean advanced indexing here. Gemma4 image padding is not + # guaranteed to be prefix-shaped, and the generic lowerer optimizes + # some masks as prefix slices. Zero-weighting padded patches preserves + # native pooling semantics while staying easy to lower. + clamped_positions = position_row.clamp(min=0) + max_x = clamped_positions[:, 0].max() + 1 + kernel_positions = torch.div(clamped_positions, pooling_kernel_size, rounding_mode="floor") + kernel_indices = kernel_positions[:, 0] + torch.div( + max_x, + pooling_kernel_size, + rounding_mode="floor", + ) * kernel_positions[:, 1] + valid_patch_weights = torch.logical_not(padding_row).float().unsqueeze(-1) + weights = ( + F.one_hot(kernel_indices.long(), output_length).float() + * valid_patch_weights + / float(pooling_kernel_size**2) + ) + pooled_full = weights.transpose(0, 1) @ hidden_row.float() + if image_soft_token_counts is not None and row_idx < len(image_soft_token_counts): + pooled_batches.append(pooled_full[: int(image_soft_token_counts[row_idx])]) + else: + valid_bins = torch.logical_not((weights == 0).all(dim=0)) + pooled_batches.append(pooled_full[valid_bins]) + return torch.cat(pooled_batches, dim=0) + + +def _gemma4_vision_encoder_hidden_states( + vision_encoder: torch.nn.Module, + inputs_embeds: torch.Tensor, + attention_mask: torch.Tensor, + pixel_position_ids: torch.Tensor | None, +) -> torch.Tensor: + config = getattr(vision_encoder, "config", None) + hidden_states = inputs_embeds + rotary_emb = getattr(vision_encoder, "rotary_emb") + layers = getattr(vision_encoder, "layers") + num_layers = int(getattr(config, "num_hidden_layers", len(layers))) + layer_types = tuple(str(value) for value in (getattr(config, "layer_types", ()) or ())) + if layer_types and all(hasattr(rotary_emb, f"{layer_type}_inv_freq") for layer_type in layer_types): + # Newer Gemma4 builds use per-layer vision RoPE tables keyed by the + # layer attention type instead of a single vision RoPE table. + attention_mask_4d = (attention_mask.unsqueeze(-1) * attention_mask.unsqueeze(1)).unsqueeze(1) + position_embeddings_by_type = { + layer_type: rotary_emb(hidden_states, pixel_position_ids, layer_type) + for layer_type in layer_types + } + for decoder_layer in layers[:num_layers]: + layer_type = str(getattr(decoder_layer, "attention_type", layer_types[0])) + hidden_states = decoder_layer( + hidden_states, + attention_mask=attention_mask_4d, + position_embeddings=position_embeddings_by_type[layer_type], + position_ids=pixel_position_ids, + ) + return hidden_states + + if _GEMMA4_CREATE_BIDIRECTIONAL_MASK is None: + raise RuntimeError("Gemma4 bidirectional mask helper is unavailable in this transformers install") + attention_mask = _GEMMA4_CREATE_BIDIRECTIONAL_MASK( + config=config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + ) + position_embeddings = rotary_emb(hidden_states, pixel_position_ids) + for decoder_layer in layers[:num_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_embeddings=position_embeddings, + position_ids=pixel_position_ids, + ) + return hidden_states + + +def _gemma4_convert_audio_mask_to_blocked_static( + mask_4d: torch.Tensor, + *, + chunk_size: int, + left_context: int, + right_context: int, +) -> torch.Tensor: + """Build Gemma4's blocked local audio mask without tensor index expand. + + Hugging Face's helper uses an expanded gather-index tensor. The v2 graph + runtime does not have a first-class expand op, so we express the same fixed + chunk extraction as static slices and cats. + """ + _, _, seq_len, _ = mask_4d.shape + seq_len_int = int(seq_len) + num_blocks = (seq_len_int + int(chunk_size) - 1) // int(chunk_size) + padded_seq_len = num_blocks * int(chunk_size) + pad_amount = padded_seq_len - seq_len_int + if pad_amount: + mask_4d = F.pad(mask_4d, (0, pad_amount, 0, pad_amount), value=False) + mask_4d = F.pad(mask_4d, (int(left_context), int(right_context)), value=False) + + context_size = int(chunk_size) + int(left_context) + int(right_context) + blocks: list[torch.Tensor] = [] + for block_idx in range(num_blocks): + start = block_idx * int(chunk_size) + block = mask_4d[ + :, + :, + start : start + int(chunk_size), + start : start + context_size, + ] + blocks.append(block.unsqueeze(2)) + return torch.cat(blocks, dim=2) + + +def _gemma4_compute_native_like_image_features( + multimodal_backbone: torch.nn.Module, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor | None, + *, + post_proj_norm_weight: torch.Tensor | None = None, + image_soft_token_counts: tuple[int, ...] | None = None, + image_pool_shapes: tuple[tuple[int, int, int], ...] | None = None, +) -> torch.Tensor: + if pixel_position_ids is None: + raise TypeError("Gemma4 native-like vision feature path requires pixel_position_ids") + vision_tower = getattr(multimodal_backbone, "vision_tower", None) + embed_vision = getattr(multimodal_backbone, "embed_vision", None) + if not isinstance(vision_tower, torch.nn.Module) or not isinstance(embed_vision, torch.nn.Module): + raise TypeError("Gemma4 multimodal backbone is missing native-like vision modules") + + padding_positions = (pixel_position_ids == -1).all(dim=-1) + vision_inputs = vision_tower.patch_embedder( + pixel_values, + pixel_position_ids, + padding_positions, + ) + vision_hidden = _gemma4_vision_encoder_hidden_states( + vision_tower.encoder, + vision_inputs, + ~padding_positions, + pixel_position_ids, + ) + pooled_hidden = _gemma4_pool_vision_hidden_native_like( + vision_tower, + vision_hidden, + pixel_position_ids, + image_soft_token_counts=image_soft_token_counts, + image_pool_shapes=image_pool_shapes, + ) + projection = getattr(embed_vision, "embedding_projection", None) + if not isinstance(projection, torch.nn.Linear): + raise TypeError("Gemma4 vision embedder is missing embedding_projection") + projected = F.linear( + pooled_hidden, + projection.weight.float(), + None if projection.bias is None else projection.bias.float(), + ) + if post_proj_norm_weight is not None and post_proj_norm_weight.numel() > 0: + return _gemma4_rms_norm( + projected, + post_proj_norm_weight, + eps=_gemma4_layer_norm_eps(multimodal_backbone), + ) + return _gemma4_rms_norm_no_scale(projected, eps=float(getattr(embed_vision, "eps", 1e-6))) + + +def _gemma4_compute_native_like_audio_features( + multimodal_backbone: torch.nn.Module, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, +) -> torch.Tensor: + audio_tower = getattr(multimodal_backbone, "audio_tower", None) + embed_audio = getattr(multimodal_backbone, "embed_audio", None) + if not isinstance(audio_tower, torch.nn.Module) or not isinstance(embed_audio, torch.nn.Module): + raise TypeError("Gemma4 multimodal backbone is missing native-like audio modules") + + config = getattr(audio_tower, "config", None) + if config is None: + raise TypeError("Gemma4 audio tower is missing config") + hidden_states, output_mask = audio_tower.subsample_conv_projection(input_features, input_features_mask) + + if not hasattr(audio_tower, "rel_pos_enc"): + # Newer Gemma4 audio towers keep relative-position handling inside each + # conformer block. Follow that native path with fixed-shape masks. + chunk_size = int(getattr(config, "conf_attention_chunk_size", getattr(config, "attention_chunk_size", 12))) + right_context = int(getattr(config, "conf_attention_context_right", getattr(config, "attention_context_right", 0))) + left_context = int(getattr(config, "conf_attention_context_left", getattr(config, "attention_context_left", 13))) + max_past_horizon = max(0, left_context - 1) + upper_diagonal = max_past_horizon + right_context + context_size = chunk_size + max_past_horizon + right_context + lower_causal_mask = torch.tril( + torch.ones((context_size, chunk_size), dtype=torch.bool, device=hidden_states.device), + diagonal=0, + ).T + upper_causal_mask = torch.tril( + torch.ones((chunk_size, context_size), dtype=torch.bool, device=hidden_states.device), + diagonal=upper_diagonal, + ) + causal_valid_mask = ( + torch.ones((chunk_size, context_size), dtype=torch.bool, device=hidden_states.device) + * lower_causal_mask + * upper_causal_mask + ) + + layers = getattr(audio_tower, "conformer", None) + if not isinstance(layers, torch.nn.ModuleList): + raise TypeError("Gemma4 audio tower is missing conformer layers") + num_layers = int(getattr(config, "num_hidden_layers", len(layers))) + for encoder_layer in layers[:num_layers]: + hidden_states = encoder_layer(hidden_states, output_mask, causal_valid_mask) + + reduction_factor = int(getattr(config, "conf_reduction_factor", 1) or 1) + if reduction_factor > 1: + hidden_states = hidden_states[:, ::reduction_factor] + if output_mask is not None: + output_mask = output_mask[:, ::reduction_factor] + + output_proj = getattr(audio_tower, "output_proj", None) + audio_encodings = output_proj(hidden_states) if isinstance(output_proj, torch.nn.Linear) else hidden_states + else: + position_embeddings = audio_tower.rel_pos_enc(hidden_states) + seq_len = hidden_states.shape[1] + query_positions = torch.arange(seq_len, device=hidden_states.device)[:, None] + key_positions = torch.arange(seq_len, device=hidden_states.device)[None, :] + distance = query_positions - key_positions + left_context = int(getattr(config, "attention_context_left", 13)) - 1 + right_context = int(getattr(config, "attention_context_right", 0)) + local_mask = ((distance >= 0) & (distance < left_context)) | ((distance < 0) & ((-distance) < right_context)) + attention_mask = local_mask.unsqueeze(0).unsqueeze(0) + if output_mask is not None: + attention_mask = attention_mask & output_mask[:, None, None, :].to(dtype=torch.bool) + attention_mask = _gemma4_convert_audio_mask_to_blocked_static( + attention_mask, + chunk_size=int(getattr(config, "attention_chunk_size", 12)), + left_context=left_context, + right_context=right_context, + ) + + layers = getattr(audio_tower, "layers", None) + if not isinstance(layers, torch.nn.ModuleList): + layers = getattr(audio_tower, "conformer", None) + if not isinstance(layers, torch.nn.ModuleList): + raise TypeError("Gemma4 audio tower is missing layers/conformer") + num_layers = int(getattr(config, "num_hidden_layers", len(layers))) + for encoder_layer in layers[:num_layers]: + hidden_states = encoder_layer( + hidden_states, + attention_mask=attention_mask, + position_embeddings=position_embeddings, + ) + + output_proj = getattr(audio_tower, "output_proj", None) + if not isinstance(output_proj, torch.nn.Linear): + raise TypeError("Gemma4 audio tower is missing output_proj") + audio_encodings = output_proj(hidden_states) + + projection = getattr(embed_audio, "embedding_projection", None) + if not isinstance(projection, torch.nn.Linear): + raise TypeError("Gemma4 audio embedder is missing embedding_projection") + normed = _gemma4_rms_norm_no_scale(audio_encodings, eps=float(getattr(embed_audio, "eps", 1e-6))) + projected = F.linear( + normed, + projection.weight.float(), + None if projection.bias is None else projection.bias.float(), + ) + projected = projected * (1.0 / 16.0) + # The transpiled bundle is shape-specialized from representative media. + # Native Gemma4 audio preprocessing emits an unpadded feature tensor for + # that media, so the post-subsampling sequence is already the real token + # sequence. Returning the dense sequence avoids dynamic boolean indexing in + # torch.export and matches the prompt's static audio soft-token count. + return projected + + +def _gemma4_feature_token_count(features: torch.Tensor | None) -> int: + if features is None or features.numel() == 0: + return 0 + if features.ndim == 3: + return int(features.shape[1]) + if features.ndim == 2: + return int(features.shape[0]) + if features.ndim >= 1: + return int(features.reshape(-1, features.shape[-1]).shape[0]) + return 0 + + +def _gemma4_static_image_soft_token_counts( + pixel_position_ids: torch.Tensor | None, + *, + pooling_kernel_size: int, +) -> tuple[int, ...] | None: + if pixel_position_ids is None or pixel_position_ids.ndim != 3: + return None + counts: list[int] = [] + for positions in pixel_position_ids.detach().cpu(): + valid = positions[(positions != -1).any(dim=-1)] + if valid.numel() == 0: + counts.append(0) + continue + max_x = int(valid[:, 0].max().item()) + 1 + max_y = int(valid[:, 1].max().item()) + 1 + counts.append((max_x // int(pooling_kernel_size)) * (max_y // int(pooling_kernel_size))) + return tuple(counts) + + +def _gemma4_static_image_pool_shapes( + pixel_position_ids: torch.Tensor | None, + *, + pooling_kernel_size: int, +) -> tuple[tuple[int, int, int], ...] | None: + if pixel_position_ids is None or pixel_position_ids.ndim != 3: + return None + shapes: list[tuple[int, int, int]] = [] + for positions in pixel_position_ids.detach().cpu(): + valid = positions[(positions != -1).any(dim=-1)] + if valid.numel() == 0: + shapes.append((0, 0, 0)) + continue + grid_w = int(valid[:, 0].max().item()) + 1 + grid_h = int(valid[:, 1].max().item()) + 1 + pooled_count = (grid_w // int(pooling_kernel_size)) * (grid_h // int(pooling_kernel_size)) + shapes.append((grid_h, grid_w, pooled_count)) + return tuple(shapes) + + +def _gemma4_build_native_merge_plan( + multimodal_backbone: torch.nn.Module, + input_ids: torch.Tensor, + *, + image_feature_count: int, + audio_feature_count: int, +) -> _Gemma4NativeMergePlan: + if input_ids.ndim != 2 or int(input_ids.shape[0]) != 1: + raise ValueError( + "Gemma4 native multimodal merge currently expects a static batch-1 input_ids tensor, " + f"got shape {tuple(input_ids.shape)}" + ) + + image_token_id, audio_token_id, pad_token_id = _gemma4_special_token_ids(multimodal_backbone) + token_ids = [int(token) for token in input_ids.detach().cpu().reshape(-1).tolist()] + segments: list[_Gemma4NativeMergeSegment] = [] + pli_token_ids: list[int] = [] + image_offset = 0 + audio_offset = 0 + index = 0 + + while index < len(token_ids): + token_id = token_ids[index] + is_image = image_token_id != 0 and token_id == image_token_id + is_audio = audio_token_id != 0 and token_id == audio_token_id + + if is_image or is_audio: + region_start = index + while index < len(token_ids) and token_ids[index] == token_id: + index += 1 + placeholder_count = index - region_start + if is_image: + insert_count = min(placeholder_count, max(0, image_feature_count - image_offset)) + if insert_count > 0: + segments.append( + _Gemma4NativeMergeSegment( + kind="image", + input_start=region_start, + length=insert_count, + feature_start=image_offset, + ) + ) + pli_token_ids.extend([pad_token_id] * insert_count) + image_offset += insert_count + else: + insert_count = min(placeholder_count, max(0, audio_feature_count - audio_offset)) + if insert_count > 0: + segments.append( + _Gemma4NativeMergeSegment( + kind="audio", + input_start=region_start, + length=insert_count, + feature_start=audio_offset, + ) + ) + pli_token_ids.extend([pad_token_id] * insert_count) + audio_offset += insert_count + continue + + text_start = index + while index < len(token_ids): + next_token = token_ids[index] + if image_token_id != 0 and next_token == image_token_id: + break + if audio_token_id != 0 and next_token == audio_token_id: + break + index += 1 + text_tokens = token_ids[text_start:index] + if text_tokens: + segments.append( + _Gemma4NativeMergeSegment( + kind="text", + input_start=text_start, + length=len(text_tokens), + ) + ) + pli_token_ids.extend(text_tokens) + + if not segments: + raise RuntimeError("Gemma4 native multimodal merge built no input segments") + return _Gemma4NativeMergePlan( + segments=tuple(segments), + pli_token_ids=tuple(pli_token_ids), + ) + + +def _gemma4_feature_sequence( + features: torch.Tensor, + *, + batch_size: int, + feature_dim: int, +) -> torch.Tensor: + if features.ndim == 3: + if int(features.shape[0]) == batch_size: + return features + if int(features.shape[0]) == 1: + return features.expand(batch_size, -1, -1) + if features.ndim == 2: + if int(features.shape[-1]) != feature_dim: + raise ValueError(f"Gemma4 feature dim mismatch: expected {feature_dim}, got {tuple(features.shape)}") + return features.unsqueeze(0).expand(batch_size, -1, -1) + if features.ndim >= 1 and int(features.shape[-1]) == feature_dim: + flattened = features.reshape(-1, feature_dim) + return flattened.unsqueeze(0).expand(batch_size, -1, -1) + raise ValueError(f"unsupported Gemma4 feature tensor shape: {tuple(features.shape)}") + + +def _gemma4_text_embedding_scale(embedding: torch.nn.Module, fallback_scale: float) -> float: + """Return the extra scale needed after calling the HF embedding module. + + Native Cactus scales raw token embedding weights by sqrt(hidden_dim). The + HF Gemma4 embedding module already applies that scale in forward(), so the + transpiler must not multiply a second time when it traces the HF module. + """ + for attr_name in ("scalar_embed_scale", "embed_scale"): + value = getattr(embedding, attr_name, None) + if isinstance(value, torch.Tensor): + try: + if value.numel() > 0 and abs(float(value.reshape(-1)[0].item()) - float(fallback_scale)) < 1e-3: + return 1.0 + except Exception: + continue + elif isinstance(value, (float, int)) and abs(float(value) - float(fallback_scale)) < 1e-3: + return 1.0 + return float(fallback_scale) + + +def _gemma4_apply_native_merge_plan( + model: torch.nn.Module, + *, + input_ids: torch.Tensor, + image_features: torch.Tensor, + audio_features: torch.Tensor, + merge_plan: _Gemma4NativeMergePlan, + pli_token_ids: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + embedding = model.get_input_embeddings() + if not isinstance(embedding, torch.nn.Module): + raise TypeError("Gemma4 model is missing input embeddings") + batch_size = int(input_ids.shape[0]) + feature_dim = int(getattr(embedding, "embedding_dim", 0) or image_features.shape[-1] or audio_features.shape[-1]) + target_dtype = _module_floating_dtype(embedding) or image_features.dtype + image_sequence = _gemma4_feature_sequence( + image_features, + batch_size=batch_size, + feature_dim=feature_dim, + ).to(device=input_ids.device, dtype=target_dtype) + audio_sequence = _gemma4_feature_sequence( + audio_features, + batch_size=batch_size, + feature_dim=feature_dim, + ).to(device=input_ids.device, dtype=target_dtype) + + embedded_segments: list[torch.Tensor] = [] + model_config = getattr(model, "config", None) + text_config = getattr(model_config, "text_config", None) + hidden_scale = float(getattr(model_config, "hidden_size", 0) or 0) + if hidden_scale <= 0.0: + hidden_scale = float(getattr(text_config, "hidden_size", 0) or 0) + if hidden_scale <= 0.0: + hidden_scale = float(feature_dim) + hidden_scale = float(hidden_scale) ** 0.5 + text_extra_scale = _gemma4_text_embedding_scale(embedding, hidden_scale) + for segment in merge_plan.segments: + if segment.kind == "text": + text_tokens = input_ids[:, segment.input_start : segment.input_start + segment.length] + text_embeds = embedding(text_tokens) + if text_extra_scale != 1.0: + text_embeds = text_embeds * text_extra_scale + embedded_segments.append(text_embeds) + elif segment.kind == "image": + embedded_segments.append( + image_sequence[:, segment.feature_start : segment.feature_start + segment.length, :] + ) + elif segment.kind == "audio": + embedded_segments.append( + audio_sequence[:, segment.feature_start : segment.feature_start + segment.length, :] + ) + else: + raise RuntimeError(f"unknown Gemma4 merge segment kind: {segment.kind!r}") + + inputs_embeds = torch.cat(embedded_segments, dim=1) + if pli_token_ids.numel() != inputs_embeds.shape[1]: + raise RuntimeError( + "Gemma4 native merge PLI token count mismatch: " + f"{int(pli_token_ids.numel())} vs {int(inputs_embeds.shape[1])}" + ) + pli_tokens = pli_token_ids.to(device=input_ids.device, dtype=input_ids.dtype).unsqueeze(0) + if batch_size != 1: + pli_tokens = pli_tokens.expand(batch_size, -1) + return inputs_embeds, pli_tokens + + +def _gemma4_remap_sequence_tensor( + tensor: torch.Tensor | None, + *, + merge_plan: _Gemma4NativeMergePlan, +) -> torch.Tensor | None: + if tensor is None: + return None + if tensor.ndim != 2: + raise ValueError(f"Gemma4 merge remap expects a rank-2 tensor, got shape {tuple(tensor.shape)}") + remapped_segments: list[torch.Tensor] = [] + for segment in merge_plan.segments: + remapped_segments.append( + tensor[:, segment.input_start : segment.input_start + segment.length] + ) + if not remapped_segments: + return tensor[:, :0] + return torch.cat(remapped_segments, dim=1) + + +def _gemma4_build_standard_causal_mask_mapping( + *, + create_causal_mask: Callable[..., torch.Tensor] | None, + create_sliding_window_causal_mask: Callable[..., torch.Tensor] | None, + config: object, + inputs_embeds: torch.Tensor, + attention_mask: torch.Tensor | None, + position_ids: torch.Tensor, +) -> dict[str, torch.Tensor]: + if not callable(create_causal_mask) or not callable(create_sliding_window_causal_mask): + raise RuntimeError("Gemma4 standard causal mask helpers are unavailable") + mask_kwargs = { + "config": config, + "inputs_embeds": inputs_embeds, + "attention_mask": attention_mask, + "past_key_values": None, + "position_ids": position_ids, + } + return { + "full_attention": create_causal_mask(**mask_kwargs), + "sliding_attention": create_sliding_window_causal_mask(**mask_kwargs), + } + + +def _infer_input_names(module: torch.nn.Module, *, preferred: tuple[str, ...]) -> tuple[str, ...]: + try: + signature = inspect.signature(module.forward) + except (TypeError, ValueError): + return preferred[:1] + + control_names = { + "self", + "return_dict", + "use_cache", + "past_key_values", + "cache_position", + "position_ids", + "labels", + "decoder_input_ids", + "decoder_attention_mask", + "output_attentions", + "output_hidden_states", + } + available = [ + name + for name, parameter in signature.parameters.items() + if name not in control_names + and parameter.kind in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + ] + matched = [name for name in preferred if name in available] + if matched: + return tuple(matched) + if available: + return tuple(available[: min(2, len(available))]) + return preferred[:1] + + +class BoundInputAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str, metadata_task: str): + super().__init__() + self.model = model + self.input_names = tuple(input_names) + self.family = family + self.metadata_task = metadata_task + + def _kwargs_from_bound_inputs(self, *bound_inputs: torch.Tensor | None) -> dict[str, torch.Tensor]: + provided = tuple(bound_inputs) + if len(self.input_names) > len(provided): + raise ValueError( + f"adapter expected at most {len(provided)} bound inputs, got {len(self.input_names)} names" + ) + kwargs: dict[str, torch.Tensor] = {} + for index, name in enumerate(self.input_names): + value = provided[index] + if value is None: + raise ValueError(f"missing required bound input {index} for {name}") + kwargs[name] = value + return kwargs + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family=self.family, + adapter_type=type(self).__name__, + input_names=self.input_names, + ), + "task": self.metadata_task, + } + } + + +class CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): + super().__init__() + self.model = model + self.backbone = getattr(model, "model", None) + self.lm_head = getattr(model, "lm_head", None) + self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model) + + def forward(self, input_ids: torch.Tensor): + backbone = self.backbone + lm_head = self.lm_head + if isinstance(backbone, torch.nn.Module) and isinstance(lm_head, torch.nn.Module): + backbone_kwargs: dict[str, object] = { + "input_ids": input_ids, + "attention_mask": (input_ids != int(self.pad_token_id)).long() + if self.pad_token_id is not None + else None, + "use_cache": False, + "return_dict": False, + } + outputs = backbone(**_filter_supported_kwargs(backbone, backbone_kwargs)) + hidden_states = _extract_tensor_output(outputs, preferred_field="last_hidden_state") + hidden_states = _select_last_non_pad_token( + hidden_states, + input_ids, + pad_token_id=self.pad_token_id, + ) + return lm_head(hidden_states) + + outputs = self.model( + input_ids=input_ids, + use_cache=False, + return_dict=False, + ) + logits = _extract_tensor_output(outputs, preferred_field="logits") + return _select_last_non_pad_token( + logits, + input_ids, + pad_token_id=self.pad_token_id, + ) + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="generic", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + } + } + + +class Lfm2CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): + super().__init__() + self.model = model + model_root = getattr(model, "model", None) + language_model = getattr(model_root, "language_model", None) + self.backbone = language_model if isinstance(language_model, torch.nn.Module) else model_root + self.lm_head = getattr(model, "lm_head", None) + self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model) + from transformers.models.lfm2.modeling_lfm2 import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward(self, input_ids: torch.Tensor): + backbone = self.backbone + lm_head = self.lm_head + if not isinstance(backbone, torch.nn.Module) or not isinstance(lm_head, torch.nn.Module): + raise TypeError("LFM2 causal logits adapter requires backbone and lm_head modules") + + inputs_embeds = backbone.embed_tokens(input_ids) + attention_mask = ( + (input_ids != int(self.pad_token_id)).to(dtype=torch.int64) + if self.pad_token_id is not None + else None + ) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + causal_mask = self._create_causal_mask( + config=backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=None, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = backbone.rotary_emb(hidden_states, position_ids=position_ids) + layer_types = tuple(getattr(backbone.config, "layer_types", ())) + linear_attention = attention_mask if inputs_embeds.shape[1] != 1 else None + + for layer_index, decoder_layer in enumerate(backbone.layers[: backbone.config.num_hidden_layers]): + layer_type = layer_types[layer_index] if layer_index < len(layer_types) else "full_attention" + layer_mask = causal_mask if layer_type == "full_attention" else linear_attention + hidden_states = decoder_layer( + hidden_states, + attention_mask=layer_mask, + position_embeddings=position_embeddings, + position_ids=position_ids, + past_key_values=None, + use_cache=False, + ) + + hidden_states = backbone.embedding_norm(hidden_states) + hidden_states = _select_last_non_pad_token( + hidden_states, + input_ids, + pad_token_id=self.pad_token_id, + ) + return lm_head(hidden_states) + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="lfm2", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + } + } + + +def _lfm2_vl_model_root(model: torch.nn.Module) -> torch.nn.Module: + root = getattr(model, "model", None) + if not isinstance(root, torch.nn.Module): + raise TypeError("LFM2-VL adapter requires a model.model module") + return root + + +def _lfm2_language_backbone(model: torch.nn.Module) -> torch.nn.Module: + root = getattr(model, "model", None) + language_model = getattr(root, "language_model", None) + backbone = language_model if isinstance(language_model, torch.nn.Module) else root + if not isinstance(backbone, torch.nn.Module): + raise TypeError("LFM2 adapter requires a language model backbone") + return backbone + + +class Lfm2VlVisionEncoderAdapter(torch.nn.Module): + def __init__( + self, + model: torch.nn.Module, + *, + pixel_attention_mask: torch.Tensor, + spatial_shapes: torch.Tensor, + weights_dir: str | None = None, + ): + super().__init__() + self.model = model + self.weights_dir = weights_dir + root = _lfm2_vl_model_root(model) + vision_tower = getattr(root, "vision_tower", None) + if not isinstance(vision_tower, torch.nn.Module): + raise TypeError("LFM2-VL model is missing a vision_tower module") + vision_model = getattr(vision_tower, "vision_model", None) + if not isinstance(vision_model, torch.nn.Module): + raise TypeError("LFM2-VL vision_tower is missing vision_model") + projector = getattr(root, "multi_modal_projector", None) + if not isinstance(projector, torch.nn.Module): + raise TypeError("LFM2-VL model is missing multi_modal_projector") + + self.vision_tower = vision_tower + self.vision_model = vision_model + self.embeddings = vision_model.embeddings + self.encoder = vision_model.encoder + self.post_layernorm = vision_model.post_layernorm + self.projector = projector + from transformers.models.siglip2.modeling_siglip2 import create_bidirectional_mask # type: ignore + + self._create_bidirectional_mask = create_bidirectional_mask + + static_spatial_shapes = [ + (int(shape[0]), int(shape[1])) + for shape in spatial_shapes.detach().cpu().tolist() + ] + self._static_spatial_shapes = tuple(static_spatial_shapes) + self._static_feature_lengths = tuple( + int(value) + for value in pixel_attention_mask.detach().cpu().to(dtype=torch.int64).sum(dim=1).tolist() + ) + with torch.no_grad(): + positional_embeddings = self.embeddings.position_embedding.weight.reshape( + self.embeddings.position_embedding_size, + self.embeddings.position_embedding_size, + -1, + ) + static_positional_embeddings = self.embeddings.resize_positional_embeddings( + positional_embeddings, + spatial_shapes.detach().cpu(), + max_length=int(pixel_attention_mask.shape[1]), + ).detach() + self.register_buffer("_static_positional_embeddings", static_positional_embeddings, persistent=False) + + def forward( + self, + pixel_values: torch.Tensor, + spatial_shapes: torch.Tensor, + pixel_attention_mask: torch.Tensor, + ) -> torch.Tensor: + target_dtype = self.embeddings.patch_embedding.weight.dtype + hidden_states = self.embeddings.patch_embedding(pixel_values.to(dtype=target_dtype)) + hidden_states = hidden_states + self._static_positional_embeddings.to( + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + encoder_attention_mask = self._create_bidirectional_mask( + config=self.vision_model.config, + inputs_embeds=hidden_states, + attention_mask=pixel_attention_mask, + ) + for encoder_layer in self.encoder.layers: + hidden_states = encoder_layer(hidden_states, encoder_attention_mask) + hidden_states = self.post_layernorm(hidden_states) + + image_features: list[torch.Tensor] = [] + for image_index, (feature_length, spatial_shape) in enumerate( + zip(self._static_feature_lengths, self._static_spatial_shapes, strict=True) + ): + feature_h, feature_w = spatial_shape + feature = hidden_states[image_index : image_index + 1, :feature_length, :] + feature = feature.reshape(1, feature_h, feature_w, -1) + image_embedding = self.projector(feature) + image_features.append(image_embedding.reshape(-1, image_embedding.shape[-1])) + if len(image_features) == 1: + return image_features[0] + return torch.cat(image_features, dim=0) + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="lfm2_vl", + adapter_type=type(self).__name__, + input_names=("pixel_values", "spatial_shapes", "pixel_attention_mask"), + ), + "weights_dir": self.weights_dir, + } + } + + +class Lfm2VlLMEncoderAdapter(torch.nn.Module): + def __init__( + self, + model: torch.nn.Module, + *, + input_ids: torch.Tensor, + weights_dir: str | None = None, + ): + super().__init__() + self.model = model + self.weights_dir = weights_dir + self.root = _lfm2_vl_model_root(model) + self.backbone = _lfm2_language_backbone(model) + image_token_id = int(getattr(self.root.config, "image_token_id")) + token_ids = [int(value) for value in input_ids.detach().cpu().reshape(-1).tolist()] + segments: list[tuple[int, int]] = [] + index = 0 + while index < len(token_ids): + if token_ids[index] != image_token_id: + index += 1 + continue + start = index + while index < len(token_ids) and token_ids[index] == image_token_id: + index += 1 + segments.append((start, index - start)) + self._static_image_segments = tuple(segments) + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + image_features: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + image_features = image_features.to(device=inputs_embeds.device, dtype=inputs_embeds.dtype) + if self._static_image_segments: + pieces: list[torch.Tensor] = [] + token_cursor = 0 + feature_cursor = 0 + for token_start, token_length in self._static_image_segments: + if token_start > token_cursor: + pieces.append(inputs_embeds[:, token_cursor:token_start, :]) + feature_end = feature_cursor + token_length + pieces.append(image_features[feature_cursor:feature_end, :].unsqueeze(0)) + feature_cursor = feature_end + token_cursor = token_start + token_length + if token_cursor < inputs_embeds.shape[1]: + pieces.append(inputs_embeds[:, token_cursor:, :]) + inputs_embeds = torch.cat(pieces, dim=1) if len(pieces) > 1 else pieces[0] + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + return inputs_embeds, attention_mask.to(dtype=torch.int64), position_ids + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="lfm2_vl", + adapter_type=type(self).__name__, + input_names=("input_ids", "attention_mask", "image_features"), + ), + "weights_dir": self.weights_dir, + } + } + + +class Lfm2VlDecoderAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__() + self.model = model + self.weights_dir = weights_dir + self.backbone = _lfm2_language_backbone(model) + self.lm_head = getattr(model, "lm_head", None) + if not isinstance(self.lm_head, torch.nn.Module): + raise TypeError("LFM2-VL decoder adapter requires an lm_head module") + from transformers.models.lfm2.modeling_lfm2 import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward( + self, + inputs_embeds: torch.Tensor, + attention_mask: torch.Tensor, + position_ids: torch.Tensor, + ) -> torch.Tensor: + causal_mask = self._create_causal_mask( + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=None, + position_ids=position_ids, + ) + hidden_states = inputs_embeds + position_embeddings = self.backbone.rotary_emb(hidden_states, position_ids=position_ids) + layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + linear_attention = attention_mask if inputs_embeds.shape[1] != 1 else None + + for layer_index, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + layer_type = layer_types[layer_index] if layer_index < len(layer_types) else "full_attention" + layer_mask = causal_mask if layer_type == "full_attention" else linear_attention + hidden_states = decoder_layer( + hidden_states, + attention_mask=layer_mask, + position_embeddings=position_embeddings, + position_ids=position_ids, + past_key_values=None, + use_cache=False, + ) + + hidden_states = self.backbone.embedding_norm(hidden_states) + return self.lm_head(hidden_states[:, -1:, :]) + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="lfm2_vl", + adapter_type=type(self).__name__, + input_names=("inputs_embeds", "attention_mask", "position_ids"), + ), + "weights_dir": self.weights_dir, + } + } + + +class Lfm2VlMultimodalCausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...]): + super().__init__() + self.model = model + self.input_names = tuple(input_names) + + def forward(self, *args: torch.Tensor) -> torch.Tensor: + kwargs = { + name: value + for name, value in zip(self.input_names, args, strict=True) + } + outputs = self.model( + **kwargs, + use_cache=False, + return_dict=True, + logits_to_keep=1, + ) + return _extract_tensor_output(outputs, preferred_field="logits") + + def get_transpile_metadata(self): + return { + "graph": _transpile_graph_meta( + self.model, + adapter_family="lfm2_vl", + adapter_type=type(self).__name__, + input_names=self.input_names, + ), + } + + +class GemmaCausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): + super().__init__() + self.model = model + self.backbone = model.model + self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model) + from transformers.models.gemma.modeling_gemma import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + causal_mask = self._create_causal_mask( + self.backbone.config, + inputs_embeds, + None, + past_key_values=None, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.backbone.rotary_emb(hidden_states, position_ids=position_ids) + checkpoints: list[torch.Tensor] = [] + + for decoder_layer in self.backbone.layers[: self.backbone.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=None, + use_cache=False, + position_embeddings=position_embeddings, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + hidden_states = _select_last_non_pad_token( + hidden_states, + input_ids, + pad_token_id=self.pad_token_id, + ) + checkpoints.append(hidden_states) + return _gemma4_apply_final_logit_softcapping(self.model, self.model.lm_head(hidden_states)), checkpoints + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + } + } + + +class Gemma3CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): + super().__init__() + self.model = model + self.backbone = model.model + self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model) + from transformers.models.gemma3.modeling_gemma3 import create_causal_mask # type: ignore + from transformers.models.gemma3.modeling_gemma3 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + causal_mask_mapping = { + "full_attention": self._create_causal_mask( + self.backbone.config, + inputs_embeds, + None, + None, + past_key_values=None, + position_ids=position_ids, + ), + "sliding_attention": self._create_sliding_window_causal_mask( + self.backbone.config, + inputs_embeds, + None, + None, + past_key_values=None, + position_ids=position_ids, + ), + } + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.config.layer_types + } + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], + position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], + position_ids=position_ids, + past_key_values=None, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma3", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + } + } + + +class Gemma4CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): + super().__init__() + self.model = model + model_backbone = model.model + self.backbone = getattr(model_backbone, "language_model", model_backbone) + self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model) + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + per_layer_inputs = None + if self.backbone.hidden_size_per_layer_input: + per_layer_inputs = _gemma4_get_per_layer_inputs(self.backbone, input_ids, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.unique_layer_types + } + shared_kv_states: dict[str, torch.Tensor] = {} + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + per_layer_input = per_layer_inputs[:, :, i, :] if per_layer_inputs is not None else None + hidden_states = decoder_layer( + hidden_states, + per_layer_input, + shared_kv_states=shared_kv_states, + position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], + attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], + position_ids=position_ids, + past_key_values=None, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def debug_first_block(self, input_ids: torch.Tensor) -> dict[str, torch.Tensor]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + per_layer_inputs = None + if self.backbone.hidden_size_per_layer_input: + per_layer_inputs = _gemma4_get_per_layer_inputs(self.backbone, input_ids, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.unique_layer_types + } + shared_kv_states: dict[str, torch.Tensor] = {} + layer = self.backbone.layers[0] + layer_type = self.backbone.config.layer_types[0] + per_layer_input = per_layer_inputs[:, :, 0, :] if per_layer_inputs is not None else None + + checkpoints: dict[str, torch.Tensor] = {} + + residual = hidden_states + normed = layer.input_layernorm(hidden_states) + checkpoints["pre_attn_norm"] = normed + + attn_out = layer.self_attn( + normed, + position_embeddings=position_embeddings[layer_type], + attention_mask=causal_mask_mapping[layer_type], + position_ids=position_ids, + past_key_values=None, + shared_kv_states=shared_kv_states, + ) + if isinstance(attn_out, tuple): + attn_out = attn_out[0] + checkpoints["attn_o_proj"] = attn_out + + post_attn_norm = layer.post_attention_layernorm(attn_out) + checkpoints["post_attn_norm"] = post_attn_norm + + after_attention = residual + post_attn_norm + checkpoints["after_attention_residual"] = after_attention + + pre_ffn_norm = layer.pre_feedforward_layernorm(after_attention) + checkpoints["pre_ffn_norm"] = pre_ffn_norm + + mlp_out = layer.mlp(pre_ffn_norm) + checkpoints["mlp_down"] = mlp_out + + post_ffn_norm = layer.post_feedforward_layernorm(mlp_out) + checkpoints["post_ffn_norm"] = post_ffn_norm + + after_ffn = after_attention + post_ffn_norm + checkpoints["after_ffn_residual"] = after_ffn + + if per_layer_input is not None: + gated = layer.per_layer_input_gate(after_ffn) + gated = layer.act_fn(gated) + projected = gated * per_layer_input + per_layer_proj = layer.per_layer_projection(projected) + checkpoints["per_layer_input_proj"] = per_layer_proj + post_per_layer_input_norm = layer.post_per_layer_input_norm(per_layer_proj) + checkpoints["post_per_layer_input_norm"] = post_per_layer_input_norm + after_ffn = after_ffn + post_per_layer_input_norm + + layer_scalar = getattr(layer, "layer_scalar", None) + if layer_scalar is not None: + after_ffn = after_ffn * layer_scalar + checkpoints["layer_scalar_out"] = after_ffn + return checkpoints + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + } + } + + +def _gemma4_apply_final_logit_softcapping(model: torch.nn.Module, logits: torch.Tensor) -> torch.Tensor: + config = getattr(model, "config", None) + text_config = None + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + try: + text_config = get_text_config() + except Exception: + text_config = None + cap = getattr(text_config, "final_logit_softcapping", None) + if cap is None: + cap = getattr(config, "final_logit_softcapping", None) + if cap is None: + return logits + cap_value = float(cap) + if cap_value <= 0.0: + return logits + return torch.tanh(logits / cap_value) * cap_value + + +class Gemma4MultimodalCausalLMLogitsAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], weights_dir: str | None = None): + super().__init__( + model, + input_names=input_names, + family="gemma4", + metadata_task="multimodal_causal_lm_logits", + ) + model_backbone = model.model + self.multimodal_backbone = model_backbone + self.backbone = getattr(model_backbone, "language_model", model_backbone) + self.last_token_logits_only = False + self._use_cached_multimodal_features = False + self._native_merge_plan: _Gemma4NativeMergePlan | None = None + self.register_buffer("_cached_image_features", torch.empty(0), persistent=False) + self.register_buffer("_cached_audio_features", torch.empty(0), persistent=False) + self.register_buffer("_native_merge_pli_token_ids", torch.empty(0, dtype=torch.long), persistent=False) + vision_post_proj_norm = _gemma4_load_vision_post_proj_norm(weights_dir) + if vision_post_proj_norm is None: + vision_post_proj_norm = torch.empty(0) + self.register_buffer("_cactus_vision_post_proj_norm", vision_post_proj_norm, persistent=False) + self._capture_cpu_float32_text_modules: list[tuple[torch.nn.Module, torch.dtype]] = [] + self._create_causal_mask_mapping = None + self._create_masks_for_generate = None + self._create_causal_mask = None + self._create_sliding_window_causal_mask = None + try: + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_masks_for_generate # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_causal_mask_mapping = create_causal_mask_mapping + self._create_masks_for_generate = create_masks_for_generate + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + except Exception: + pass + + def _apply_final_logit_softcapping(self, logits: torch.Tensor) -> torch.Tensor: + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def _capture_text_modules(self) -> tuple[torch.nn.Module, ...]: + return _unique_modules( + self.backbone, + getattr(self.model, "get_input_embeddings", lambda: None)(), + getattr(self.model, "lm_head", None), + ) + + def _compute_image_features( + self, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor | None, + get_image_features: Callable[..., object], + ) -> torch.Tensor: + multimodal_backbone = self.multimodal_backbone + if _gemma4_can_use_native_like_vision_features(multimodal_backbone): + return _gemma4_compute_native_like_image_features( + multimodal_backbone, + pixel_values, + pixel_position_ids, + post_proj_norm_weight=self._cactus_vision_post_proj_norm, + ) + vision_tower = getattr(multimodal_backbone, "vision_tower", None) + embed_vision = getattr(multimodal_backbone, "embed_vision", None) + vision_modules = _gemma4_vision_modules(multimodal_backbone) + if ( + pixel_values.device.type != "cpu" + or pixel_values.dtype != torch.float16 + or len(vision_modules) != 2 + or _torch_is_compiling() + ): + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + vision_dtype = _module_floating_dtype(vision_tower) + embed_dtype = _module_floating_dtype(embed_vision) + if vision_dtype != torch.float16 and embed_dtype != torch.float16: + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + # Gemma4's CPU float16 vision path can emit non-finite soft tokens; upcast only + # the static image feature extraction path and restore the original module dtypes. + with _temporary_cpu_float32_modules(vision_modules): + return get_image_features( + pixel_values.float(), + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + def _compute_audio_features( + self, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, + get_audio_features: Callable[..., object], + ) -> torch.Tensor: + multimodal_backbone = self.multimodal_backbone + if _gemma4_can_use_native_like_audio_features(multimodal_backbone): + return _gemma4_compute_native_like_audio_features( + multimodal_backbone, + input_features, + input_features_mask, + ) + audio_modules = _gemma4_audio_modules(multimodal_backbone) + if ( + input_features.device.type != "cpu" + or input_features.dtype != torch.float16 + or len(audio_modules) != 2 + or _torch_is_compiling() + ): + audio_output = get_audio_features(input_features, input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + if all(_module_floating_dtype(module) != torch.float16 for module in audio_modules): + audio_output = get_audio_features(input_features, input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + with _temporary_cpu_float32_modules(audio_modules): + audio_output = get_audio_features(input_features.float(), input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + def prepare_cpu_float32_capture(self) -> None: + self._capture_cpu_float32_text_modules.clear() + if os.environ.get("CACTUS_GEMMA4_CAPTURE_FP32") != "1": + return + for module in self._capture_text_modules(): + device = _module_device(module) + dtype = _module_floating_dtype(module) + if device is None or device.type != "cpu" or dtype is None or dtype == torch.float32: + continue + self._capture_cpu_float32_text_modules.append((module, dtype)) + module.to(dtype=torch.float32) + + def restore_cpu_float32_capture(self) -> None: + for module, dtype in reversed(self._capture_cpu_float32_text_modules): + module.to(dtype=dtype) + self._capture_cpu_float32_text_modules.clear() + + def _resolve_image_features( + self, + *, + inputs_embeds: torch.Tensor, + pixel_values: torch.Tensor | None, + pixel_position_ids: torch.Tensor | None, + get_image_features: Callable[..., object], + ) -> torch.Tensor | None: + if pixel_values is None: + return None + if self._use_cached_multimodal_features and self._cached_image_features.numel() > 0: + image_features = self._cached_image_features + else: + image_features = self._compute_image_features( + pixel_values, + pixel_position_ids, + get_image_features, + ) + return image_features.to(inputs_embeds.device, inputs_embeds.dtype) + + def _resolve_audio_features( + self, + *, + inputs_embeds: torch.Tensor, + input_features: torch.Tensor | None, + input_features_mask: torch.Tensor | None, + get_audio_features: Callable[..., object], + ) -> torch.Tensor | None: + if input_features is None or input_features_mask is None: + return None + if self._use_cached_multimodal_features and self._cached_audio_features.numel() > 0: + audio_features = self._cached_audio_features + else: + audio_features = self._compute_audio_features( + input_features, + input_features_mask, + get_audio_features, + ) + return audio_features.to(inputs_embeds.device, inputs_embeds.dtype) + + def _prepare_text_backbone_inputs( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor | None, + pixel_values: torch.Tensor | None, + pixel_position_ids: torch.Tensor | None, + input_features: torch.Tensor | None, + input_features_mask: torch.Tensor | None, + get_placeholder_mask: Callable[..., object], + get_image_features: Callable[..., object], + get_audio_features: Callable[..., object], + ) -> tuple[torch.Tensor, torch.Tensor | None, dict[str, torch.Tensor], torch.LongTensor]: + inputs_embeds = self.model.get_input_embeddings()(input_ids) + + image_features = self._resolve_image_features( + inputs_embeds=inputs_embeds, + pixel_values=pixel_values, + pixel_position_ids=pixel_position_ids, + get_image_features=get_image_features, + ) + audio_features = self._resolve_audio_features( + inputs_embeds=inputs_embeds, + input_features=input_features, + input_features_mask=input_features_mask, + get_audio_features=get_audio_features, + ) + + if image_features is not None and audio_features is not None and self._native_merge_plan is not None: + inputs_embeds, per_layer_inputs_tokens = _gemma4_apply_native_merge_plan( + self.model, + input_ids=input_ids, + image_features=image_features, + audio_features=audio_features, + merge_plan=self._native_merge_plan, + pli_token_ids=self._native_merge_pli_token_ids, + ) + attention_mask = _gemma4_remap_sequence_tensor( + attention_mask, + merge_plan=self._native_merge_plan, + ) + token_type_ids = _gemma4_remap_sequence_tensor( + token_type_ids, + merge_plan=self._native_merge_plan, + ) + if token_type_ids is None: + raise RuntimeError("Gemma4 native merge requires token_type_ids for multimodal attention masking") + else: + text_mask, image_mask, audio_mask = _gemma4_get_placeholder_masks( + get_placeholder_mask, + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + if image_features is not None: + inputs_embeds = inputs_embeds.masked_scatter( + image_mask.unsqueeze(-1).expand_as(inputs_embeds), + image_features, + ) + if audio_features is not None: + inputs_embeds = inputs_embeds.masked_scatter( + audio_mask.unsqueeze(-1).expand_as(inputs_embeds), + audio_features, + ) + per_layer_inputs_tokens = input_ids * text_mask.to(dtype=input_ids.dtype) + + per_layer_inputs = None + text_config = _gemma4_text_config(self.multimodal_backbone) + if getattr(text_config, "hidden_size_per_layer_input", None): + per_layer_inputs = _gemma4_get_per_layer_inputs(self.backbone, per_layer_inputs_tokens, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + if self._native_merge_plan is not None: + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + elif getattr(text_config, "use_bidirectional_attention", None) == "vision": + causal_mask_mapping = self._create_causal_mask_mapping( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + token_type_ids, + pixel_values, + is_training=self.training, + ) + else: + causal_mask_mapping = self._create_masks_for_generate( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + ) + return inputs_embeds, per_layer_inputs, causal_mask_mapping, position_ids + + def _forward_hidden_states( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor | None, + pixel_values: torch.Tensor | None, + pixel_position_ids: torch.Tensor | None, + input_features: torch.Tensor | None, + input_features_mask: torch.Tensor | None, + get_placeholder_mask: Callable[..., object], + get_image_features: Callable[..., object], + get_audio_features: Callable[..., object], + ) -> torch.Tensor: + inputs_embeds, per_layer_inputs, causal_mask_mapping, position_ids = self._prepare_text_backbone_inputs( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + pixel_values=pixel_values, + pixel_position_ids=pixel_position_ids, + input_features=input_features, + input_features_mask=input_features_mask, + get_placeholder_mask=get_placeholder_mask, + get_image_features=get_image_features, + get_audio_features=get_audio_features, + ) + return _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + ) + + def prime_static_multimodal_features(self, *bound_inputs: torch.Tensor | None) -> None: + kwargs = self._kwargs_from_bound_inputs(*bound_inputs) + input_ids = kwargs["input_ids"] + token_type_ids = kwargs.get("token_type_ids") + pixel_values = kwargs.get("pixel_values") + pixel_position_ids = kwargs.get("pixel_position_ids") + input_features = kwargs.get("input_features") + input_features_mask = kwargs.get("input_features_mask") + + multimodal_backbone = self.multimodal_backbone + get_placeholder_mask = getattr(multimodal_backbone, "get_placeholder_mask", None) + get_image_features = getattr(multimodal_backbone, "get_image_features", None) + get_audio_features = getattr(multimodal_backbone, "get_audio_features", None) + if not callable(get_placeholder_mask): + return + + with torch.no_grad(): + inputs_embeds = self.model.get_input_embeddings()(input_ids) + _, image_mask, audio_mask = _gemma4_get_placeholder_masks( + get_placeholder_mask, + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + + if pixel_values is not None and callable(get_image_features) and image_mask.any(): + image_features = self._compute_image_features( + pixel_values, + pixel_position_ids, + get_image_features, + ) + self._cached_image_features = image_features.detach() + else: + self._cached_image_features = self._cached_image_features.new_empty(0) + + if ( + input_features is not None + and input_features_mask is not None + and callable(get_audio_features) + and audio_mask.any() + ): + self._cached_audio_features = self._compute_audio_features( + input_features, + input_features_mask, + get_audio_features, + ).detach() + else: + self._cached_audio_features = self._cached_audio_features.new_empty(0) + + if self._cached_image_features.numel() > 0 and self._cached_audio_features.numel() > 0: + plan = _gemma4_build_native_merge_plan( + self.multimodal_backbone, + input_ids, + image_feature_count=_gemma4_feature_token_count(self._cached_image_features), + audio_feature_count=_gemma4_feature_token_count(self._cached_audio_features), + ) + self._native_merge_plan = plan + self._native_merge_pli_token_ids = torch.tensor( + plan.pli_token_ids, + dtype=torch.long, + ) + else: + self._native_merge_plan = None + self._native_merge_pli_token_ids = self._native_merge_pli_token_ids.new_empty(0) + + self._use_cached_multimodal_features = True + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + kwargs = self._kwargs_from_bound_inputs(*bound_inputs) + input_ids = kwargs["input_ids"] + attention_mask = kwargs.get("attention_mask") + token_type_ids = kwargs.get("token_type_ids") + pixel_values = kwargs.get("pixel_values") + pixel_position_ids = kwargs.get("pixel_position_ids") + input_features = kwargs.get("input_features") + input_features_mask = kwargs.get("input_features_mask") + + multimodal_backbone = self.multimodal_backbone + get_placeholder_mask = getattr(multimodal_backbone, "get_placeholder_mask", None) + get_image_features = getattr(multimodal_backbone, "get_image_features", None) + get_audio_features = getattr(multimodal_backbone, "get_audio_features", None) + lm_head = getattr(self.model, "lm_head", None) + if ( + not callable(get_placeholder_mask) + or not callable(get_image_features) + or not callable(get_audio_features) + or not callable(self._create_causal_mask_mapping) + or not callable(self._create_masks_for_generate) + or not isinstance(lm_head, torch.nn.Module) + ): + outputs = self.model( + return_dict=True, + use_cache=False, + **kwargs, + ) + logits = _extract_tensor_output(outputs, preferred_field="logits") + if self.last_token_logits_only and logits.ndim >= 3: + return _select_last_active_token(logits, attention_mask) + return logits + + hidden_states = self._forward_hidden_states( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + pixel_values=pixel_values, + pixel_position_ids=pixel_position_ids, + input_features=input_features, + input_features_mask=input_features_mask, + get_placeholder_mask=get_placeholder_mask, + get_image_features=get_image_features, + get_audio_features=get_audio_features, + ) + if self.last_token_logits_only: + hidden_states = _select_last_active_token(hidden_states, attention_mask) + return self._apply_final_logit_softcapping(lm_head(hidden_states)) + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=type(self).__name__, + input_names=self.input_names, + ), + "task": self.metadata_task, + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + "last_token_logits_only": bool(self.last_token_logits_only), + } + } + + +_GEMMA4_DECODER_PIPELINE_IO_KEYS = ( + "inputs_embeds", + "per_layer_inputs", + "position_ids", +) + + +class _Gemma4MultimodalComponentBase(torch.nn.Module): + def __init__( + self, + model: torch.nn.Module, + *, + input_names: tuple[str, ...], + weights_dir: str | None = None, + native_merge_plan: _Gemma4NativeMergePlan | None = None, + native_image_soft_token_counts: tuple[int, ...] | None = None, + native_image_pool_shapes: tuple[tuple[int, int, int], ...] | None = None, + ): + super().__init__() + self.model = model + self.input_names = input_names + self.multimodal_backbone = model.model + model_backbone = model.model + self.backbone = getattr(model_backbone, "language_model", model_backbone) + self._native_merge_plan = native_merge_plan + self._native_image_soft_token_counts = native_image_soft_token_counts + self._native_image_pool_shapes = native_image_pool_shapes + self.register_buffer( + "_native_merge_pli_token_ids", + torch.tensor(native_merge_plan.pli_token_ids, dtype=torch.long) + if native_merge_plan is not None + else torch.empty(0, dtype=torch.long), + persistent=False, + ) + vision_post_proj_norm = _gemma4_load_vision_post_proj_norm(weights_dir) + if vision_post_proj_norm is None: + vision_post_proj_norm = torch.empty(0) + self.register_buffer("_cactus_vision_post_proj_norm", vision_post_proj_norm, persistent=False) + self._create_causal_mask_mapping = None + self._create_masks_for_generate = None + self._create_causal_mask = None + self._create_sliding_window_causal_mask = None + try: + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_masks_for_generate # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_causal_mask_mapping = create_causal_mask_mapping + self._create_masks_for_generate = create_masks_for_generate + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + except Exception: + pass + self._capture_modules: list[tuple[torch.nn.Module, torch.dtype]] = [] + + def _base_graph_meta(self, *, adapter_type: str, input_names: tuple[str, ...]) -> dict[str, object]: + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=adapter_type, + input_names=input_names, + ), + "task": "multimodal_causal_lm_logits", + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + } + + def _compute_image_features( + self, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor | None, + ) -> torch.Tensor: + if _gemma4_can_use_native_like_vision_features(self.multimodal_backbone): + return _gemma4_compute_native_like_image_features( + self.multimodal_backbone, + pixel_values, + pixel_position_ids, + post_proj_norm_weight=self._cactus_vision_post_proj_norm, + image_soft_token_counts=self._native_image_soft_token_counts, + image_pool_shapes=self._native_image_pool_shapes, + ) + get_image_features = getattr(self.multimodal_backbone, "get_image_features", None) + vision_tower = getattr(self.multimodal_backbone, "vision_tower", None) + embed_vision = getattr(self.multimodal_backbone, "embed_vision", None) + vision_modules = _gemma4_vision_modules(self.multimodal_backbone) + if not callable(get_image_features): + raise TypeError("Gemma4 multimodal backbone is missing get_image_features") + if ( + pixel_values.device.type != "cpu" + or pixel_values.dtype != torch.float16 + or len(vision_modules) != 2 + or _torch_is_compiling() + ): + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + vision_dtype = _module_floating_dtype(vision_tower) + embed_dtype = _module_floating_dtype(embed_vision) + if vision_dtype != torch.float16 and embed_dtype != torch.float16: + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + with _temporary_cpu_float32_modules(vision_modules): + return get_image_features( + pixel_values.float(), + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + def _compute_audio_features( + self, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, + ) -> torch.Tensor: + if _gemma4_can_use_native_like_audio_features(self.multimodal_backbone): + return _gemma4_compute_native_like_audio_features( + self.multimodal_backbone, + input_features, + input_features_mask, + ) + get_audio_features = getattr(self.multimodal_backbone, "get_audio_features", None) + if not callable(get_audio_features): + raise TypeError("Gemma4 multimodal backbone is missing get_audio_features") + audio_modules = _gemma4_audio_modules(self.multimodal_backbone) + if ( + input_features.device.type == "cpu" + and input_features.dtype == torch.float16 + and len(audio_modules) == 2 + and not _torch_is_compiling() + and any(_module_floating_dtype(module) == torch.float16 for module in audio_modules) + ): + with _temporary_cpu_float32_modules(audio_modules): + audio_output = get_audio_features(input_features.float(), input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + audio_output = get_audio_features(input_features, input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + def _modules_to_prepare_for_capture(self) -> tuple[torch.nn.Module, ...]: + return () + + def prepare_for_capture(self, **_: object) -> None: + self.restore_after_capture() + modules = self._modules_to_prepare_for_capture() + if not modules: + return + promoted: list[tuple[torch.nn.Module, torch.dtype]] = [] + for module in modules: + device = _module_device(module) + dtype = _module_floating_dtype(module) + if device is None or device.type != "cpu" or dtype is None or dtype == torch.float32: + continue + promoted.append((module, dtype)) + module.to(dtype=torch.float32) + self._capture_modules = promoted + + def restore_after_capture(self, **_: object) -> None: + for module, dtype in reversed(self._capture_modules): + module.to(dtype=dtype) + self._capture_modules.clear() + + def _prepare_decoder_inputs( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor, + image_features: torch.Tensor, + audio_features: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.LongTensor]: + if self._native_merge_plan is not None: + inputs_embeds, per_layer_inputs_tokens = _gemma4_apply_native_merge_plan( + self.model, + input_ids=input_ids, + image_features=image_features, + audio_features=audio_features, + merge_plan=self._native_merge_plan, + pli_token_ids=self._native_merge_pli_token_ids, + ) + attention_mask = _gemma4_remap_sequence_tensor( + attention_mask, + merge_plan=self._native_merge_plan, + ) + token_type_ids = _gemma4_remap_sequence_tensor( + token_type_ids, + merge_plan=self._native_merge_plan, + ) + if token_type_ids is None: + raise RuntimeError("Gemma4 native merge requires token_type_ids for multimodal attention masking") + else: + get_placeholder_mask = getattr(self.multimodal_backbone, "get_placeholder_mask", None) + if not callable(get_placeholder_mask): + raise TypeError("Gemma4 multimodal backbone is missing get_placeholder_mask") + + inputs_embeds = self.model.get_input_embeddings()(input_ids) + text_mask, image_mask, audio_mask = _gemma4_get_placeholder_masks( + get_placeholder_mask, + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + inputs_embeds = inputs_embeds.masked_scatter( + image_mask.unsqueeze(-1).expand_as(inputs_embeds), + image_features.to(inputs_embeds.device, inputs_embeds.dtype), + ) + inputs_embeds = inputs_embeds.masked_scatter( + audio_mask.unsqueeze(-1).expand_as(inputs_embeds), + audio_features.to(inputs_embeds.device, inputs_embeds.dtype), + ) + per_layer_inputs_tokens = input_ids * text_mask.to(dtype=input_ids.dtype) + + per_layer_inputs: torch.Tensor + text_config = _gemma4_text_config(self.multimodal_backbone) + if getattr(text_config, "hidden_size_per_layer_input", None): + per_layer_inputs = _gemma4_get_per_layer_inputs(self.backbone, per_layer_inputs_tokens, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + else: + per_layer_inputs = inputs_embeds.new_empty((inputs_embeds.shape[0], inputs_embeds.shape[1], 0, 0)) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + return ( + inputs_embeds, + per_layer_inputs, + position_ids, + ) + + def _prepare_text_decoder_step_inputs( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.LongTensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.LongTensor]: + embedding = self.model.get_input_embeddings() + if not isinstance(embedding, torch.nn.Module): + raise TypeError("Gemma4 model is missing input embeddings") + inputs_embeds = embedding(input_ids) + + model_config = getattr(self.model, "config", None) + text_config = getattr(model_config, "text_config", None) + hidden_scale = float(getattr(model_config, "hidden_size", 0) or 0) + if hidden_scale <= 0.0: + hidden_scale = float(getattr(text_config, "hidden_size", 0) or 0) + if hidden_scale <= 0.0: + hidden_scale = float(inputs_embeds.shape[-1]) + text_extra_scale = _gemma4_text_embedding_scale(embedding, hidden_scale ** 0.5) + if text_extra_scale != 1.0: + inputs_embeds = inputs_embeds * text_extra_scale + + text_config = _gemma4_text_config(self.multimodal_backbone) + if getattr(text_config, "hidden_size_per_layer_input", None): + per_layer_inputs = _gemma4_get_per_layer_inputs(self.backbone, input_ids, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + else: + per_layer_inputs = inputs_embeds.new_empty((inputs_embeds.shape[0], inputs_embeds.shape[1], 0, 0)) + return inputs_embeds, per_layer_inputs, position_ids + + +class Gemma4VisionEncoderAdapter(_Gemma4MultimodalComponentBase): + def __init__( + self, + model: torch.nn.Module, + *, + weights_dir: str | None = None, + native_image_soft_token_counts: tuple[int, ...] | None = None, + native_image_pool_shapes: tuple[tuple[int, int, int], ...] | None = None, + ): + super().__init__( + model, + input_names=("pixel_values", "pixel_position_ids"), + weights_dir=weights_dir, + native_image_soft_token_counts=native_image_soft_token_counts, + native_image_pool_shapes=native_image_pool_shapes, + ) + + def _modules_to_prepare_for_capture(self) -> tuple[torch.nn.Module, ...]: + return () + + def forward(self, pixel_values: torch.Tensor, pixel_position_ids: torch.Tensor | None) -> torch.Tensor: + return self._compute_image_features(pixel_values, pixel_position_ids) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("pixel_values", "pixel_position_ids"), + ), + } + + +class Gemma4AudioEncoderAdapter(_Gemma4MultimodalComponentBase): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__(model, input_names=("input_features", "input_features_mask"), weights_dir=weights_dir) + + def _modules_to_prepare_for_capture(self) -> tuple[torch.nn.Module, ...]: + return () + + def forward(self, input_features: torch.Tensor, input_features_mask: torch.Tensor) -> torch.Tensor: + return self._compute_audio_features(input_features, input_features_mask) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("input_features", "input_features_mask"), + ), + } + + +class Gemma4LMEncoderAdapter(_Gemma4MultimodalComponentBase): + def __init__( + self, + model: torch.nn.Module, + *, + weights_dir: str | None = None, + native_merge_plan: _Gemma4NativeMergePlan | None = None, + ): + super().__init__( + model, + input_names=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), + weights_dir=weights_dir, + native_merge_plan=native_merge_plan, + ) + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor, + image_features: torch.Tensor, + audio_features: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.LongTensor]: + return self._prepare_decoder_inputs( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + image_features=image_features, + audio_features=audio_features, + ) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), + ), + } + + +class Gemma4LMEncoderStepAdapter(_Gemma4MultimodalComponentBase): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__(model, input_names=("input_ids", "position_ids"), weights_dir=weights_dir) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.LongTensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.LongTensor]: + return self._prepare_text_decoder_step_inputs( + input_ids=input_ids, + position_ids=position_ids, + ) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("input_ids", "position_ids"), + ), + } + + +class Gemma4DecoderAdapter(_Gemma4MultimodalComponentBase): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__(model, input_names=_GEMMA4_DECODER_PIPELINE_IO_KEYS, weights_dir=weights_dir) + + def forward( + self, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor, + position_ids: torch.LongTensor, + ) -> torch.Tensor: + normalized_per_layer_inputs = per_layer_inputs + if normalized_per_layer_inputs.numel() == 0: + normalized_per_layer_inputs = None + attention_mask = torch.ones( + position_ids.shape, + dtype=torch.long, + device=position_ids.device, + ) + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + ) + logits = self.model.lm_head(hidden_states[:, -1:, :]) + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + ) + } + + +class Gemma4DecoderStepAdapter(Gemma4DecoderAdapter): + pass + + +def _build_gemma4_multimodal_component_specs( + model: torch.nn.Module, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None, + components: tuple[str, ...] | None = None, +) -> list[ComponentModuleSpec]: + pixel_values = named_tensors["pixel_values"] + pixel_position_ids = named_tensors.get("pixel_position_ids") + input_features = named_tensors["input_features"] + input_features_mask = named_tensors["input_features_mask"] + input_ids = named_tensors["input_ids"] + attention_mask = named_tensors.get("attention_mask") + token_type_ids = named_tensors["token_type_ids"] + + requested_components = tuple(components or ("vision_encoder", "audio_encoder", "lm_encoder", "decoder")) + requested_set = set(requested_components) + if not requested_set: + return [] + + expanded_components: list[str] = [] + + def _require(component: str) -> None: + if component not in expanded_components: + expanded_components.append(component) + + if "decoder" in requested_set: + _require("vision_encoder") + _require("audio_encoder") + _require("lm_encoder") + _require("decoder_prefill_chunk") + _require("lm_encoder_step") + _require("decoder_step") + if "decoder_prefill_chunk" in requested_set: + _require("vision_encoder") + _require("audio_encoder") + _require("lm_encoder") + _require("decoder_prefill_chunk") + elif "lm_encoder" in requested_set: + _require("vision_encoder") + _require("audio_encoder") + _require("lm_encoder") + else: + if "vision_encoder" in requested_set: + _require("vision_encoder") + if "audio_encoder" in requested_set: + _require("audio_encoder") + if "lm_encoder_step" in requested_set: + _require("lm_encoder_step") + if "decoder_step" in requested_set: + _require("lm_encoder_step") + _require("decoder_step") + + vision_tower = getattr(getattr(model, "model", model), "vision_tower", None) + pooling_kernel_size = int(_module_or_config_attr(vision_tower, "pooling_kernel_size", 3) or 3) + native_image_soft_token_counts = _gemma4_static_image_soft_token_counts( + pixel_position_ids, + pooling_kernel_size=pooling_kernel_size, + ) + native_image_pool_shapes = _gemma4_static_image_pool_shapes( + pixel_position_ids, + pooling_kernel_size=pooling_kernel_size, + ) + + vision_encoder = Gemma4VisionEncoderAdapter( + model, + weights_dir=weights_dir, + native_image_soft_token_counts=native_image_soft_token_counts, + native_image_pool_shapes=native_image_pool_shapes, + ).eval() + audio_encoder = Gemma4AudioEncoderAdapter(model, weights_dir=weights_dir).eval() + + image_features: torch.Tensor | None = None + audio_features: torch.Tensor | None = None + decoder_inputs: tuple[torch.Tensor, ...] | None = None + decoder_step_inputs: tuple[torch.Tensor, ...] | None = None + step_input_ids: torch.Tensor | None = None + step_position_ids: torch.Tensor | None = None + native_merge_plan: _Gemma4NativeMergePlan | None = None + + with torch.no_grad(): + if "vision_encoder" in expanded_components and ( + "lm_encoder" in expanded_components + or "decoder" in expanded_components + or "decoder_prefill_chunk" in expanded_components + ): + image_features = vision_encoder(pixel_values, pixel_position_ids) + if "audio_encoder" in expanded_components and ( + "lm_encoder" in expanded_components + or "decoder" in expanded_components + or "decoder_prefill_chunk" in expanded_components + ): + audio_features = audio_encoder(input_features, input_features_mask) + if "lm_encoder" in expanded_components or "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + if image_features is None: + image_features = vision_encoder(pixel_values, pixel_position_ids) + if audio_features is None: + audio_features = audio_encoder(input_features, input_features_mask) + native_merge_plan = _gemma4_build_native_merge_plan( + getattr(model, "model", model), + input_ids, + image_feature_count=_gemma4_feature_token_count(image_features), + audio_feature_count=_gemma4_feature_token_count(audio_features), + ) + lm_encoder = Gemma4LMEncoderAdapter( + model, + weights_dir=weights_dir, + native_merge_plan=native_merge_plan, + ).eval() + lm_encoder_step = Gemma4LMEncoderStepAdapter(model, weights_dir=weights_dir).eval() + decoder = Gemma4DecoderAdapter(model, weights_dir=weights_dir).eval() + decoder_step = Gemma4DecoderStepAdapter(model, weights_dir=weights_dir).eval() + + with torch.no_grad(): + if "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + decoder_inputs = lm_encoder( + input_ids, + attention_mask, + token_type_ids, + image_features, + audio_features, + ) + if "lm_encoder_step" in expanded_components or "decoder_step" in expanded_components: + step_input_ids = input_ids[:, -1:].contiguous() + step_position_ids = torch.full( + (int(input_ids.shape[0]), 1), + max(0, int(input_ids.shape[1]) - 1), + dtype=torch.long, + device=input_ids.device, + ) + decoder_step_inputs = lm_encoder_step(step_input_ids, step_position_ids) + + common_graph_meta = { + "weights_dir": weights_dir, + "task": "multimodal_causal_lm_logits", + "adapter_family": "gemma4", + } + specs: list[ComponentModuleSpec] = [] + if "vision_encoder" in expanded_components: + specs.append(ComponentModuleSpec( + component="vision_encoder", + module=vision_encoder, + example_inputs=(pixel_values, pixel_position_ids), + input_keys=("pixel_values", "pixel_position_ids"), + output_keys=("image_features",), + graph_meta={**common_graph_meta, "component": "vision_encoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "audio_encoder" in expanded_components: + specs.append(ComponentModuleSpec( + component="audio_encoder", + module=audio_encoder, + example_inputs=(input_features, input_features_mask), + input_keys=("input_features", "input_features_mask"), + output_keys=("audio_features",), + graph_meta={**common_graph_meta, "component": "audio_encoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "lm_encoder" in expanded_components: + if image_features is None or audio_features is None: + raise RuntimeError("Gemma4 lm_encoder spec requires precomputed image/audio features") + specs.append(ComponentModuleSpec( + component="lm_encoder", + module=lm_encoder, + example_inputs=(input_ids, attention_mask, token_type_ids, image_features, audio_features), + input_keys=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), + output_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + graph_meta={**common_graph_meta, "component": "lm_encoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + cache_seq_len = max(1024, int(input_ids.shape[1]) + 256) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "32") or "32")) + prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) + if "decoder" in expanded_components: + if decoder_inputs is None: + raise RuntimeError("Gemma4 decoder spec requires precomputed decoder inputs") + specs.append(ComponentModuleSpec( + component="decoder", + module=decoder, + example_inputs=tuple(decoder_inputs), + input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + output_keys=("logits",), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "decoder_prefill_chunk" in expanded_components: + if decoder_inputs is None: + raise RuntimeError("Gemma4 decoder_prefill_chunk spec requires precomputed decoder inputs") + chunk_inputs = tuple( + value[:, :prefill_chunk_size, ...].contiguous() + if value.ndim >= 2 + else value + for value in decoder_inputs + ) + specs.append(ComponentModuleSpec( + component="decoder_prefill_chunk", + module=decoder, + example_inputs=chunk_inputs, + input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_prefill_chunk", + "use_internal_kv_cache": True, + "max_cache_seq_len": cache_seq_len, + "cache_sink_size": 4, + "prefill_chunk_size": prefill_chunk_size, + }, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "lm_encoder_step" in expanded_components: + if step_input_ids is None or step_position_ids is None: + raise RuntimeError("Gemma4 lm_encoder_step spec requires step token inputs") + specs.append(ComponentModuleSpec( + component="lm_encoder_step", + module=lm_encoder_step, + example_inputs=(step_input_ids, step_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "decoder_step" in expanded_components: + if decoder_step_inputs is None: + raise RuntimeError("Gemma4 decoder_step spec requires precomputed step decoder inputs") + specs.append(ComponentModuleSpec( + component="decoder_step", + module=decoder_step, + example_inputs=tuple(decoder_step_inputs), + input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_step", + "use_internal_kv_cache": True, + "max_cache_seq_len": cache_seq_len, + "cache_sink_size": 4, + }, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + return specs + + +class Qwen35CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): + super().__init__() + self.model = model + self.backbone = model.model + self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model) + from transformers.models.qwen3_5.modeling_qwen3_5 import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + attention_mask = ( + (input_ids != int(self.pad_token_id)).to(dtype=torch.int64) + if self.pad_token_id is not None + else None + ) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1) + text_position_ids = position_ids[0] + multimodal_position_ids = position_ids[1:] + + causal_mask = self._create_causal_mask( + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=None, + position_ids=text_position_ids, + ) + linear_attn_mask = self.backbone._update_linear_attn_mask(attention_mask, None) + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask + hidden_states = decoder_layer( + hidden_states, + position_embeddings=position_embeddings, + attention_mask=layer_mask, + position_ids=text_position_ids, + past_key_values=None, + use_cache=False, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + hidden_states = _select_last_non_pad_token( + hidden_states, + input_ids, + pad_token_id=self.pad_token_id, + ) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="qwen3_5", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": layer_types, + } + } + + +class Qwen3CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): + super().__init__() + self.model = model + self.backbone = model.model + self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model) + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + seq_len = int(inputs_embeds.shape[1]) + allowed_positions = torch.tril( + torch.ones((seq_len, seq_len), dtype=torch.bool, device=inputs_embeds.device), + ).view(1, 1, seq_len, seq_len) + if self.pad_token_id is not None: + key_mask = (input_ids != int(self.pad_token_id)).view(input_ids.shape[0], 1, 1, seq_len) + allowed_positions = torch.logical_and(allowed_positions, key_mask) + allowed_values = torch.ones( + (1, 1, seq_len, seq_len), + dtype=inputs_embeds.dtype, + device=inputs_embeds.device, + ) * 0.0 + blocked_values = torch.ones( + (1, 1, seq_len, seq_len), + dtype=inputs_embeds.dtype, + device=inputs_embeds.device, + ) * torch.finfo(inputs_embeds.dtype).min + causal_mask = torch.where( + allowed_positions, + allowed_values, + blocked_values, + ) + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = self.backbone.rotary_emb(hidden_states, position_ids) + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + hidden_states = decoder_layer( + hidden_states, + position_embeddings=position_embeddings, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=None, + use_cache=False, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + hidden_states = _select_last_non_pad_token( + hidden_states, + input_ids, + pad_token_id=self.pad_token_id, + ) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="qwen3", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + } + } + + +class CTCLogitsAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): + super().__init__(model, input_names=input_names, family=family, metadata_task="ctc_logits") + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + outputs = self.model(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) + return _extract_tensor_output(outputs, preferred_field="logits") + + +class EncoderHiddenStatesAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): + encoder = None + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder = get_encoder() + if encoder is None: + encoder = getattr(model, "encoder", None) + if encoder is None: + model_attr = getattr(model, "model", None) + if model_attr is not None: + encoder = getattr(model_attr, "encoder", None) + if not isinstance(encoder, torch.nn.Module): + raise NotImplementedError(f"{type(model).__name__} does not expose an encoder module") + super().__init__(model, input_names=input_names, family=family, metadata_task="encoder_hidden_states") + self.encoder = encoder + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + outputs = self.encoder(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) + return _extract_tensor_output(outputs, preferred_field="last_hidden_state") + + +class WhisperEncoderComponentAdapter(torch.nn.Module): + def __init__(self, encoder: torch.nn.Module): + super().__init__() + self.encoder = encoder + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + hidden_states = F.gelu(self.encoder.conv1(input_features)) + hidden_states = F.gelu(self.encoder.conv2(hidden_states)) + hidden_states = hidden_states.permute(0, 2, 1) + + position_ids = torch.arange(hidden_states.shape[1], device=hidden_states.device) + positions = self.encoder.embed_positions(position_ids) + hidden_states = hidden_states + positions.to( + device=hidden_states.device, + dtype=hidden_states.dtype, + ).unsqueeze(0) + + for layer in self.encoder.layers: + residual = hidden_states + hidden_states = layer.self_attn_layer_norm(hidden_states) + hidden_states, _ = layer.self_attn( + hidden_states=hidden_states, + attention_mask=None, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = layer.final_layer_norm(hidden_states) + hidden_states = layer.activation_fn(layer.fc1(hidden_states)) + hidden_states = layer.fc2(hidden_states) + hidden_states = residual + hidden_states + + return self.encoder.layer_norm(hidden_states) + + +class WhisperDecoderComponentAdapter(torch.nn.Module): + def __init__(self, decoder: torch.nn.Module, proj_out: torch.nn.Module, *, pad_token_id: int | None): + super().__init__() + self.decoder = decoder + self.proj_out = proj_out + self.pad_token_id = pad_token_id + + @staticmethod + def _build_causal_mask(input_ids: torch.Tensor, *, dtype: torch.dtype) -> torch.Tensor: + seq_len = int(input_ids.shape[1]) + allowed = torch.tril( + torch.ones((seq_len, seq_len), dtype=torch.bool, device=input_ids.device), + ).view(1, 1, seq_len, seq_len) + allowed_values = torch.ones( + (1, 1, seq_len, seq_len), + dtype=dtype, + device=input_ids.device, + ) * 0.0 + blocked_values = torch.ones( + (1, 1, seq_len, seq_len), + dtype=dtype, + device=input_ids.device, + ) * torch.finfo(dtype).min + return torch.where(allowed, allowed_values, blocked_values) + + def forward(self, decoder_input_ids: torch.Tensor, encoder_hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.decoder.embed_tokens(decoder_input_ids) + position_ids = torch.arange( + hidden_states.shape[1], + device=hidden_states.device, + ).unsqueeze(0).expand(hidden_states.shape[0], -1) + positions = self.decoder.embed_positions( + decoder_input_ids, + past_key_values_length=0, + position_ids=position_ids, + ) + hidden_states = hidden_states + positions.to( + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + causal_mask = self._build_causal_mask( + decoder_input_ids, + dtype=hidden_states.dtype, + ) + for layer in self.decoder.layers: + hidden_states = layer( + hidden_states, + attention_mask=causal_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=None, + past_key_values=None, + use_cache=False, + ) + + hidden_states = self.decoder.layer_norm(hidden_states) + hidden_states = _select_last_non_pad_token( + hidden_states, + decoder_input_ids, + pad_token_id=self.pad_token_id, + ) + return self.proj_out(hidden_states) + + +def _family_key(model: torch.nn.Module) -> str: + explicit_family = getattr(model, "family", None) + if isinstance(explicit_family, str) and explicit_family: + return explicit_family.lower() + module_name = type(model).__module__ + if module_name.startswith("transformers.models.whisper."): + return "whisper" + if module_name.startswith("transformers.models.gemma4."): + return "gemma4" + if module_name.startswith("transformers.models.gemma3."): + return "gemma3" + if module_name.startswith("transformers.models.gemma."): + return "gemma" + if module_name.startswith("transformers.models.qwen3_5."): + return "qwen3_5" + if module_name.startswith("transformers.models.qwen3."): + return "qwen3" + if module_name.startswith("transformers.models.lfm2_vl."): + return "lfm2_vl" + if module_name.startswith("transformers.models.lfm2_moe."): + return "lfm2_moe" + if module_name.startswith("transformers.models.lfm2."): + return "lfm2" + return "generic" + + +def _build_whisper_seq2seq_component_specs( + model: torch.nn.Module, + *, + named_tensors: dict[str, torch.Tensor], + inputs_metadata: dict[str, object] | None = None, + weights_dir: str | None = None, +) -> list[ComponentModuleSpec]: + input_features = named_tensors.get("input_features") + if not isinstance(input_features, torch.Tensor): + raise RuntimeError("Whisper component transpile requires input_features") + + get_encoder = getattr(model, "get_encoder", None) + encoder = get_encoder() if callable(get_encoder) else None + if not isinstance(encoder, torch.nn.Module): + encoder = getattr(getattr(model, "model", None), "encoder", None) + decoder = getattr(getattr(model, "model", None), "decoder", None) + proj_out = getattr(model, "proj_out", None) + if not isinstance(encoder, torch.nn.Module) or not isinstance(decoder, torch.nn.Module): + raise RuntimeError(f"{type(model).__name__} does not expose Whisper encoder/decoder modules") + if not isinstance(proj_out, torch.nn.Module): + raise RuntimeError(f"{type(model).__name__} does not expose a Whisper projection head") + + metadata = dict(inputs_metadata or {}) + decoder_prompt_ids = metadata.get("decoder_input_ids") + if not isinstance(decoder_prompt_ids, list) or not decoder_prompt_ids: + decoder_start_token_id = getattr(getattr(model, "config", None), "decoder_start_token_id", None) + if not isinstance(decoder_start_token_id, int): + raise RuntimeError("Whisper component transpile requires decoder_input_ids metadata") + decoder_prompt_ids = [int(decoder_start_token_id)] + else: + decoder_prompt_ids = [int(value) for value in decoder_prompt_ids] + + target_token_count = int(metadata.get("target_token_count", len(decoder_prompt_ids)) or len(decoder_prompt_ids)) + target_token_count = max(target_token_count, len(decoder_prompt_ids)) + pad_token_id = int(metadata.get("pad_token_id", getattr(getattr(model, "config", None), "pad_token_id", 0)) or 0) + + encoder_adapter = WhisperEncoderComponentAdapter(encoder).eval() + decoder_adapter = WhisperDecoderComponentAdapter( + decoder, + proj_out, + pad_token_id=pad_token_id, + ).eval() + + with torch.no_grad(): + encoder_hidden_states = encoder_adapter(input_features) + + decoder_input_ids = torch.full( + (1, target_token_count), + pad_token_id, + dtype=torch.int64, + device=input_features.device, + ) + decoder_input_ids[:, : len(decoder_prompt_ids)] = torch.tensor( + decoder_prompt_ids, + dtype=torch.int64, + device=input_features.device, + ) + + common_graph_meta = { + **_transpile_graph_meta( + model, + adapter_family="whisper", + adapter_type="component_pipeline", + input_names=("input_features",), + ), + "task": "seq2seq_transcription", + "adapter_family": "whisper", + } + if weights_dir: + common_graph_meta["weights_dir"] = weights_dir + return [ + ComponentModuleSpec( + component="audio_encoder", + module=encoder_adapter, + example_inputs=(input_features,), + input_keys=("input_features",), + output_keys=("encoder_hidden_states",), + graph_meta={**common_graph_meta, "component": "audio_encoder"}, + metadata={"family": "whisper", "task": "seq2seq_transcription"}, + ), + ComponentModuleSpec( + component="decoder", + module=decoder_adapter, + example_inputs=(decoder_input_ids, encoder_hidden_states), + input_keys=("decoder_input_ids", "encoder_hidden_states"), + output_keys=("logits",), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata={"family": "whisper", "task": "seq2seq_transcription"}, + ), + ] + + +def _build_lfm2_vl_multimodal_component_specs( + model: torch.nn.Module, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None, + components: tuple[str, ...] | None = None, +) -> list[ComponentModuleSpec]: + input_ids = named_tensors["input_ids"] + attention_mask = named_tensors["attention_mask"] + pixel_values = named_tensors["pixel_values"] + spatial_shapes = named_tensors["spatial_shapes"] + pixel_attention_mask = named_tensors["pixel_attention_mask"] + + requested_components = tuple(components or ("vision_encoder", "lm_encoder", "decoder")) + requested_set = set(requested_components) + if not requested_set: + return [] + + expanded_components: list[str] = [] + + def _require(component: str) -> None: + if component not in expanded_components: + expanded_components.append(component) + + if "decoder" in requested_set: + _require("vision_encoder") + _require("lm_encoder") + _require("decoder") + elif "lm_encoder" in requested_set: + _require("vision_encoder") + _require("lm_encoder") + elif "vision_encoder" in requested_set: + _require("vision_encoder") + + vision_encoder = Lfm2VlVisionEncoderAdapter( + model, + pixel_attention_mask=pixel_attention_mask, + spatial_shapes=spatial_shapes, + weights_dir=weights_dir, + ).eval() + lm_encoder = Lfm2VlLMEncoderAdapter(model, input_ids=input_ids, weights_dir=weights_dir).eval() + decoder = Lfm2VlDecoderAdapter(model, weights_dir=weights_dir).eval() + + image_features: torch.Tensor | None = None + decoder_inputs: tuple[torch.Tensor, ...] | None = None + with torch.no_grad(): + if "vision_encoder" in expanded_components and ("lm_encoder" in expanded_components or "decoder" in expanded_components): + image_features = vision_encoder(pixel_values, spatial_shapes, pixel_attention_mask) + if "lm_encoder" in expanded_components or "decoder" in expanded_components: + if image_features is None: + image_features = vision_encoder(pixel_values, spatial_shapes, pixel_attention_mask) + decoder_inputs = lm_encoder(input_ids, attention_mask, image_features) + + common_graph_meta = { + "weights_dir": weights_dir, + "task": "multimodal_causal_lm_logits", + "adapter_family": "lfm2_vl", + } + specs: list[ComponentModuleSpec] = [] + if "vision_encoder" in expanded_components: + specs.append(ComponentModuleSpec( + component="vision_encoder", + module=vision_encoder, + example_inputs=(pixel_values, spatial_shapes, pixel_attention_mask), + input_keys=("pixel_values", "spatial_shapes", "pixel_attention_mask"), + output_keys=("image_features",), + graph_meta={**common_graph_meta, "component": "vision_encoder"}, + metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, + )) + if "lm_encoder" in expanded_components: + if image_features is None: + raise RuntimeError("LFM2-VL lm_encoder spec requires precomputed image features") + specs.append(ComponentModuleSpec( + component="lm_encoder", + module=lm_encoder, + example_inputs=(input_ids, attention_mask, image_features), + input_keys=("input_ids", "attention_mask", "image_features"), + output_keys=("inputs_embeds", "attention_mask", "position_ids"), + graph_meta={**common_graph_meta, "component": "lm_encoder"}, + metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, + )) + if "decoder" in expanded_components: + if decoder_inputs is None: + raise RuntimeError("LFM2-VL decoder spec requires precomputed decoder inputs") + specs.append(ComponentModuleSpec( + component="decoder", + module=decoder, + example_inputs=decoder_inputs, + input_keys=("inputs_embeds", "attention_mask", "position_ids"), + output_keys=("logits",), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, + )) + return specs + + +def build_component_module_specs( + model: torch.nn.Module, + *, + task: str, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, + inputs_metadata: dict[str, object] | None = None, + components: tuple[str, ...] | None = None, +) -> list[ComponentModuleSpec] | None: + family = _family_key(model) + if family == "gemma4" and task == "multimodal_causal_lm_logits": + return _build_gemma4_multimodal_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + components=components, + ) + if family == "lfm2_vl" and task == "multimodal_causal_lm_logits": + return _build_lfm2_vl_multimodal_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + components=components, + ) + if family == "parakeet_tdt" and task == "tdt_transcription": + from cactus.transpile.tdt_runtime import build_parakeet_tdt_component_specs + + return build_parakeet_tdt_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + ) + if family == "whisper" and task == "seq2seq_transcription": + return _build_whisper_seq2seq_component_specs( + model, + named_tensors=named_tensors, + inputs_metadata=inputs_metadata, + weights_dir=weights_dir, + ) + return None + + +def canonicalize_model_interface( + model: torch.nn.Module, + task: str = "causal_lm_logits", + *, + input_names: tuple[str, ...] | None = None, + weights_dir: str | None = None, + inputs_metadata: dict[str, object] | None = None, +) -> CanonicalizedModel: + family = _family_key(model) + adapter_factory: Callable[[torch.nn.Module], torch.nn.Module] + resolved_input_names = tuple(input_names or ()) + padding_token_id_value = None + if isinstance(inputs_metadata, dict): + raw_padding_token_id = inputs_metadata.get("padding_token_id") + if isinstance(raw_padding_token_id, int): + padding_token_id_value = int(raw_padding_token_id) + + if task == "causal_lm_logits": + if family == "gemma": + adapter_factory = lambda inner_model: GemmaCausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + pad_token_id=padding_token_id_value, + ) + elif family == "gemma4": + adapter_factory = lambda inner_model: Gemma4CausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + pad_token_id=padding_token_id_value, + ) + elif family == "gemma3": + adapter_factory = lambda inner_model: Gemma3CausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + pad_token_id=padding_token_id_value, + ) + elif family == "qwen3_5": + adapter_factory = lambda inner_model: Qwen35CausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + pad_token_id=padding_token_id_value, + ) + elif family == "qwen3": + adapter_factory = lambda inner_model: Qwen3CausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + pad_token_id=padding_token_id_value, + ) + elif family in {"lfm2", "lfm2_vl", "lfm2_moe"}: + adapter_factory = lambda inner_model: Lfm2CausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + pad_token_id=padding_token_id_value, + ) + else: + adapter_factory = lambda inner_model: CausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + pad_token_id=padding_token_id_value, + ) + resolved_input_names = ("input_ids",) + elif task == "multimodal_causal_lm_logits": + if family not in {"gemma4", "lfm2_vl"}: + raise NotImplementedError(f"{type(model).__name__} does not support task={task}") + if family == "lfm2_vl": + if not resolved_input_names: + resolved_input_names = ( + "input_ids", + "attention_mask", + "pixel_values", + "spatial_shapes", + "pixel_attention_mask", + ) + adapter_factory = lambda inner_model: Lfm2VlMultimodalCausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + ) + else: + if not resolved_input_names: + resolved_input_names = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", + ) + adapter_factory = lambda inner_model: Gemma4MultimodalCausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + weights_dir=weights_dir, + ) + elif task == "ctc_logits": + if not resolved_input_names: + resolved_input_names = _infer_input_names( + model, + preferred=("input_values", "input_features", "attention_mask"), + ) + adapter_factory = lambda inner_model: CTCLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + family=family, + ) + elif task == "encoder_hidden_states": + if not resolved_input_names: + resolved_input_names = _infer_input_names( + model, + preferred=("input_features", "input_values", "attention_mask"), + ) + adapter_factory = lambda inner_model: EncoderHiddenStatesAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + family=family, + ) + elif task == "audio_classification_logits": + if not resolved_input_names: + resolved_input_names = _infer_input_names( + model, + preferred=("input_values", "input_features", "attention_mask"), + ) + adapter_factory = lambda inner_model: AudioClassificationLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + family=family, + ) + else: + raise NotImplementedError(f"unsupported task={task}") + + return CanonicalizedModel( + module=adapter_factory(model).eval(), + task=task, + family=family, + input_names=resolved_input_names, + ) diff --git a/python/cactus/transpile/model_patterns.py b/python/cactus/transpile/model_patterns.py new file mode 100644 index 000000000..fa226e9b0 --- /dev/null +++ b/python/cactus/transpile/model_patterns.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GoldPattern: + name: str + semantic_ops: tuple[str, ...] + description: str + + +GOLD_PATTERNS: tuple[GoldPattern, ...] = ( + GoldPattern( + name="decoder_attention_gqa", + semantic_ops=("rms_norm", "rope", "attention", "attention_int8_hybrid"), + description=( + "Q/K/V projections from the same normalized hidden state; Q and K optionally get per-head RMSNorm, " + "Q/K are rotary-embedded, then Cactus attention consumes native Q-head and KV-head counts." + ), + ), + GoldPattern( + name="gated_mlp_gelu", + semantic_ops=("matmul", "gelu", "multiply"), + description="Two projection branches from the same normalized input: GELU(gate) * up, then a down projection.", + ), + GoldPattern( + name="gated_mlp_silu", + semantic_ops=("matmul", "silu", "multiply"), + description="Two projection branches from the same normalized input: SiLU(gate) * up, then a down projection.", + ), + GoldPattern( + name="decoder_block_post_attn_norm", + semantic_ops=("rms_norm", "attention", "add_clipped"), + description=( + "Pre-norm attention block with post-attention RMSNorm before the residual add, followed by a second " + "pre-FFN RMSNorm and a post-FFN RMSNorm before the final residual add." + ), + ), + GoldPattern( + name="decoder_block_simple_residual", + semantic_ops=("rms_norm", "attention", "add"), + description=( + "Pre-norm attention block with a direct residual add after attention, followed by a second RMSNorm and " + "a gated MLP branch with another direct residual add." + ), + ), + GoldPattern( + name="gemma4_partial_rope_attention", + semantic_ops=("rms_norm", "rope", "concat", "attention"), + description=( + "Gemma4 attention where only a prefix of the head dimension is rotary-embedded, optionally with shared " + "K/V heads and sliding-window attention." + ), + ), + GoldPattern( + name="sliding_window_attention_mask", + semantic_ops=("attention",), + description=( + "Exported PyTorch may build a boolean mask with diff/cumsum/comparisons, but the handwritten Cactus " + "models encode the same behavior directly as an attention `window_size`." + ), + ), +) diff --git a/python/cactus/transpile/model_profiles.py b/python/cactus/transpile/model_profiles.py new file mode 100644 index 000000000..06e7debb3 --- /dev/null +++ b/python/cactus/transpile/model_profiles.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from dataclasses import dataclass +import re +from typing import Callable + +import torch + + +@dataclass(frozen=True) +class ModelProfile: + family: str + model_types: tuple[str, ...] = () + transformer_module: str | None = None + multimodal_context_tokens: int | None = None + aliases: tuple[tuple[str, str], ...] = () + regex_aliases: tuple[tuple[str, str], ...] = () + + +GEMMA4_PROFILE = ModelProfile( + family="gemma4", + model_types=("gemma4",), + transformer_module="transformers.models.gemma4.modeling_gemma4", + multimodal_context_tokens=2048, +) + + +LFM2_VL_PROFILE = ModelProfile( + family="lfm2_vl", + model_types=("lfm2_vl",), + multimodal_context_tokens=512, +) + + +PARAKEET_TDT_PROFILE = ModelProfile( + family="parakeet_tdt", + model_types=("parakeet_tdt",), + aliases=( + ("decoder.prediction.embed.weight", "decoder.embedding.weight"), + ("joint.enc.weight", "encoder_projector.weight"), + ("joint.enc.bias", "encoder_projector.bias"), + ("joint.pred.weight", "decoder.decoder_projector.weight"), + ("joint.pred.bias", "decoder.decoder_projector.bias"), + ("joint.joint_net.2.weight", "joint.head.weight"), + ("joint.joint_net.2.bias", "joint.head.bias"), + ("encoder.pre_encode.out.weight", "encoder.subsampling.linear.weight"), + ("encoder.pre_encode.out.bias", "encoder.subsampling.linear.bias"), + ), + regex_aliases=( + ( + r"encoder\.layers\.(\d+)\.self_attn\.(q|k|v)_proj\.weight", + r"encoder.layers.\1.self_attn.linear_\2.weight", + ), + ( + r"encoder\.layers\.(\d+)\.self_attn\.o_proj\.weight", + r"encoder.layers.\1.self_attn.linear_out.weight", + ), + ( + r"encoder\.layers\.(\d+)\.self_attn\.relative_k_proj\.weight", + r"encoder.layers.\1.self_attn.linear_pos.weight", + ), + ( + r"encoder\.layers\.(\d+)\.self_attn\.bias_u", + r"encoder.layers.\1.self_attn.pos_bias_u", + ), + ( + r"encoder\.layers\.(\d+)\.self_attn\.bias_v", + r"encoder.layers.\1.self_attn.pos_bias_v", + ), + ( + r"encoder\.layers\.(\d+)\.conv\.norm\.(.+)", + r"encoder.layers.\1.conv.batch_norm.\2", + ), + ), +) + + +PROFILES: tuple[ModelProfile, ...] = ( + GEMMA4_PROFILE, + LFM2_VL_PROFILE, + PARAKEET_TDT_PROFILE, +) + + +def profile_for_model_type(model_type: str) -> ModelProfile | None: + normalized = str(model_type or "").strip().lower() + for profile in PROFILES: + if normalized in profile.model_types: + return profile + return None + + +def multimodal_context_tokens_for_model_type(model_type: str, default: int) -> int: + profile = profile_for_model_type(model_type) + if profile is not None and profile.multimodal_context_tokens is not None: + return max(0, int(profile.multimodal_context_tokens)) + return max(0, int(default)) + + +def add_tensor_aliases( + state_dict: dict[str, torch.Tensor], + profile: ModelProfile, + *, + derived_aliases: Callable[[dict[str, torch.Tensor]], None] | None = None, +) -> dict[str, torch.Tensor]: + def alias(target: str, source: str) -> None: + if target not in state_dict and source in state_dict: + state_dict[target] = state_dict[source] + + for target, source in profile.aliases: + alias(target, source) + + for source_key in list(state_dict): + for source_pattern, target_template in profile.regex_aliases: + match = re.fullmatch(source_pattern, source_key) + if match is None: + continue + alias(match.expand(target_template), source_key) + + if derived_aliases is not None: + derived_aliases(state_dict) + + return state_dict diff --git a/python/cactus/transpile/multimodal_runtime.py b/python/cactus/transpile/multimodal_runtime.py new file mode 100644 index 000000000..699885a49 --- /dev/null +++ b/python/cactus/transpile/multimodal_runtime.py @@ -0,0 +1,523 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import torch + +from cactus.transpile.audio_preprocess import audio_duration_limit_seconds +from cactus.transpile.audio_preprocess import prepare_native_gemma4_audio_features +from cactus.transpile.media_limits import resize_static_image +from cactus.transpile.runtime_support import ensure_transformers_supports_gemma4 +from cactus.transpile.runtime_support import patch_torch_flex_attention_compat +from cactus.transpile.runtime_support import patch_transformers_torchvision_probe +from cactus.transpile.runtime_support import PreparedInputs + + +def _resolve_audio_sample_rate(processor: object) -> int: + for attr_name in ("feature_extractor", "tokenizer"): + child = getattr(processor, attr_name, None) + sample_rate = getattr(child, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + sample_rate = getattr(processor, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + return 16000 + + +_GEMMA4_MULTIMODAL_INPUT_ORDER = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", +) + + +def _normalize_multimodal_prompt( + prompt: str, + *, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, +) -> str: + normalized = prompt.strip() + prefixes: list[str] = [] + + if image_token and num_images > 0: + image_count = normalized.count(image_token) + if image_count < num_images: + prefixes.append(" ".join(image_token for _ in range(num_images - image_count))) + if audio_token and num_audio_segments > 0: + audio_count = normalized.count(audio_token) + if audio_count < num_audio_segments: + prefixes.append(" ".join(audio_token for _ in range(num_audio_segments - audio_count))) + + if prefixes: + prefix = "\n".join(part for part in prefixes if part) + if normalized: + return f"{prefix}\n{normalized}" + return prefix + return normalized + + +def _build_gemma4_chat_prompt( + *, + prompt: str, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> str: + result = "" + normalized_system = system_prompt.strip() + if enable_thinking_if_supported or normalized_system: + result += "<|turn>system\n" + if enable_thinking_if_supported: + result += "<|think|>" + result += normalized_system + result += "\n" + + result += "<|turn>user\n" + if image_token and num_images > 0: + for _ in range(num_images): + result += f"\n\n{image_token}\n\n" + result += prompt + if audio_token and num_audio_segments > 0: + result += "".join(audio_token for _ in range(num_audio_segments)) + result += "\n" + result += "<|turn>model\n" + return result + + +def _gemma4_split_cactus_newline_token_merges(batch: object) -> None: + input_ids = batch.get("input_ids") if hasattr(batch, "get") else None + if not isinstance(input_ids, torch.Tensor) or input_ids.ndim != 2: + return + + expansions = { + 108: (107, 107), + 109: (107, 107, 107), + } + if not any(int(token) in expansions for token in input_ids.reshape(-1).tolist()): + return + + lengths: list[list[int]] = [] + max_len = 0 + for row in input_ids.detach().cpu().tolist(): + row_lengths = [len(expansions.get(int(token), (int(token),))) for token in row] + lengths.append(row_lengths) + max_len = max(max_len, sum(row_lengths)) + + for key in ("input_ids", "attention_mask", "token_type_ids"): + value = batch.get(key) if hasattr(batch, "get") else None + if not isinstance(value, torch.Tensor) or value.ndim != 2: + continue + expanded = torch.full( + (value.shape[0], max_len), + 0, + dtype=value.dtype, + device=value.device, + ) + for row_idx, row in enumerate(value.detach().cpu().tolist()): + out: list[int] = [] + for token_idx, item in enumerate(row): + if key == "input_ids": + out.extend(expansions.get(int(item), (int(item),))) + else: + out.extend([int(item)] * lengths[row_idx][token_idx]) + expanded[row_idx, : len(out)] = torch.tensor(out, dtype=value.dtype, device=value.device) + batch[key] = expanded + + +def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: + if not image_files: + return [] + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for --image-file: {exc}") from exc + + images: list[object] = [] + for image_file in image_files: + path = Path(image_file).resolve() + if not path.exists(): + raise RuntimeError(f"image_file does not exist: {path}") + with Image.open(path) as image: + images.append(resize_static_image(image.convert("RGB")).copy()) + return images + + +def _get_processor_image_attr(processor: object, name: str, default: object) -> object: + image_processor = getattr(processor, "image_processor", None) + if image_processor is not None and hasattr(image_processor, name): + return getattr(image_processor, name) + if isinstance(image_processor, dict) and name in image_processor: + return image_processor[name] + return default + + +def _image_channel_array(value: object, default: float) -> np.ndarray: + if isinstance(value, (list, tuple, np.ndarray)): + array = np.asarray(value, dtype=np.float32).reshape(-1) + if array.size >= 3: + return array[:3] + if array.size == 1: + return np.full((3,), float(array[0]), dtype=np.float32) + if isinstance(value, (int, float)): + return np.full((3,), float(value), dtype=np.float32) + return np.full((3,), float(default), dtype=np.float32) + + +def _prepare_gemma4_native_image_tensors( + processor: object, + image_files: tuple[str, ...], +) -> tuple[torch.Tensor, torch.Tensor] | None: + if not image_files: + return None + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for Gemma4 native image preprocessing: {exc}") from exc + + patch_size = int(_get_processor_image_attr(processor, "patch_size", 16)) + pooling_kernel_size = int(_get_processor_image_attr(processor, "pooling_kernel_size", 3)) + max_soft_tokens = int(_get_processor_image_attr(processor, "max_soft_tokens", 280)) + rescale_factor = float(_get_processor_image_attr(processor, "rescale_factor", 1.0 / 255.0)) + # Native Cactus Gemma4 uses its config defaults here, even though the HF + # processor advertises do_normalize=False. Match the C++ path exactly. + image_mean = np.full((3,), 0.5, dtype=np.float32) + image_std = np.full((3,), 0.5, dtype=np.float32) + max_patches = max_soft_tokens * pooling_kernel_size * pooling_kernel_size + side_multiple = pooling_kernel_size * patch_size + patch_dim = 3 * patch_size * patch_size + if patch_size <= 0 or pooling_kernel_size <= 0 or max_patches <= 0: + return None + + try: + resample_bilinear = Image.Resampling.BILINEAR + except AttributeError: # pragma: no cover + resample_bilinear = Image.BILINEAR + + pixel_batches: list[np.ndarray] = [] + position_batches: list[np.ndarray] = [] + for image_file in image_files: + path = Path(image_file).resolve() + with Image.open(path) as image: + rgb = image.convert("RGB") + width, height = rgb.size + target_pixels = float(max_patches * patch_size * patch_size) + factor = float(np.sqrt(target_pixels / max(1.0, float(width * height)))) + target_h = int(np.floor(factor * height / side_multiple)) * side_multiple + target_w = int(np.floor(factor * width / side_multiple)) * side_multiple + if target_h == 0: + target_h = side_multiple + if target_w == 0: + target_w = side_multiple + if (target_w, target_h) != rgb.size: + rgb = rgb.resize((target_w, target_h), resample=resample_bilinear) + array = np.asarray(rgb, dtype=np.float32) * rescale_factor + array = (array - image_mean.reshape(1, 1, 3)) / image_std.reshape(1, 1, 3) + + patch_h = target_h // patch_size + patch_w = target_w // patch_size + num_patches = patch_h * patch_w + if num_patches > max_patches: + raise RuntimeError( + f"Gemma4 native image preprocessing produced {num_patches} patches, " + f"but max_patches={max_patches}" + ) + chw = np.transpose(array, (2, 0, 1)) + patches = ( + chw.reshape(3, patch_h, patch_size, patch_w, patch_size) + .transpose(1, 3, 2, 4, 0) + .reshape(num_patches, patch_dim) + ) + + padded_patches = np.zeros((max_patches, patch_dim), dtype=np.float32) + padded_patches[:num_patches] = patches + positions = np.full((max_patches, 2), -1, dtype=np.int64) + valid_positions = np.zeros((num_patches, 2), dtype=np.int64) + for patch_y in range(patch_h): + row_start = patch_y * patch_w + valid_positions[row_start : row_start + patch_w, 0] = np.arange(patch_w, dtype=np.int64) + valid_positions[row_start : row_start + patch_w, 1] = patch_y + positions[:num_patches] = valid_positions + + pixel_batches.append(padded_patches) + position_batches.append(positions) + + return ( + torch.from_numpy(np.stack(pixel_batches, axis=0)), + torch.from_numpy(np.stack(position_batches, axis=0)), + ) + + +def _gemma4_image_soft_token_counts( + pixel_position_ids: torch.Tensor, + *, + pooling_kernel_size: int, +) -> list[int]: + counts: list[int] = [] + for positions in pixel_position_ids: + valid = positions[(positions != -1).any(dim=-1)] + if valid.numel() == 0: + counts.append(0) + continue + max_x = int(valid[:, 0].max().item()) + 1 + max_y = int(valid[:, 1].max().item()) + 1 + counts.append((max_x // pooling_kernel_size) * (max_y // pooling_kernel_size)) + return counts + + +def _gemma4_audio_soft_token_count(num_frames: int) -> int: + after_stage1 = (int(num_frames) + 1) // 2 + return (after_stage1 + 1) // 2 + + +def _resolve_gemma4_audio_mels(processor: object, default: int = 128) -> int: + for child_name in ("feature_extractor", "audio_processor", "audio_feature_extractor"): + child = getattr(processor, child_name, None) + if child is None: + continue + for attr_name in ("feature_size", "num_mel_bins", "n_mels", "input_feat_size"): + value = getattr(child, attr_name, None) + if isinstance(value, int) and value > 0: + return int(value) + for attr_name in ("feature_size", "num_mel_bins", "n_mels", "input_feat_size"): + value = getattr(processor, attr_name, None) + if isinstance(value, int) and value > 0: + return int(value) + return default + + +def _build_gemma4_native_chat_prompt( + *, + prompt: str, + image_soft_token_counts: tuple[int, ...], + audio_soft_token_count: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> str: + result = "" + normalized_system = system_prompt.strip() + if enable_thinking_if_supported or normalized_system: + result += "<|turn>system\n" + if enable_thinking_if_supported: + result += "<|think|>" + result += normalized_system + result += "\n" + + result += "<|turn>user\n" + for count in image_soft_token_counts: + if count > 0: + result += "\n\n<|image>" + result += "<|image|>" * int(count) + result += "\n\n" + result += prompt + if audio_soft_token_count > 0: + result += "<|audio>" + result += "<|audio|>" * int(audio_soft_token_count) + result += "" + result += "\n" + result += "<|turn>model\n" + return result + + +def _tokenize_gemma4_native_prompt( + processor: object, + prompt: str, + *, + image_token: str | None, + audio_token: str | None, +) -> dict[str, torch.Tensor]: + tokenizer = getattr(processor, "tokenizer", processor) + if not callable(tokenizer): + raise RuntimeError("Gemma4 native prompt path requires a tokenizer") + batch = tokenizer(prompt, return_tensors="pt", add_special_tokens=False) + input_ids = batch["input_ids"] + if "attention_mask" not in batch: + batch["attention_mask"] = torch.ones_like(input_ids) + + token_type_ids = torch.zeros_like(input_ids) + convert = getattr(tokenizer, "convert_tokens_to_ids", None) + if callable(convert): + if image_token: + image_id = convert(image_token) + if isinstance(image_id, int) and image_id >= 0: + token_type_ids = torch.where(input_ids == image_id, torch.ones_like(token_type_ids), token_type_ids) + if audio_token: + audio_id = convert(audio_token) + if isinstance(audio_id, int) and audio_id >= 0: + token_type_ids = torch.where(input_ids == audio_id, torch.full_like(token_type_ids, 2), token_type_ids) + batch["token_type_ids"] = token_type_ids + return {key: value for key, value in batch.items() if isinstance(value, torch.Tensor)} + + +def prepare_gemma4_multimodal_inputs( + processor: object | None, + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, + use_gemma4_chat_template: bool = False, +) -> PreparedInputs: + if processor is None: + raise RuntimeError("multimodal Gemma4 transpile requires an AutoProcessor with image and audio support") + + images = _load_image_inputs(image_files) + audio_waveforms: list[np.ndarray] = [] + sample_rate: int | None = None + if audio_file: + sample_rate = _resolve_audio_sample_rate(processor) + from cactus.transpile.audio_preprocess import load_audio_waveform + + audio_waveforms.append( + load_audio_waveform( + audio_file, + target_sample_rate=sample_rate, + max_seconds=audio_duration_limit_seconds(), + ) + ) + + image_token = getattr(processor, "image_token", None) + audio_token = getattr(processor, "audio_token", None) + processor_prompt = _normalize_multimodal_prompt( + prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + ) + normalized_prompt = processor_prompt + if use_gemma4_chat_template: + normalized_prompt = prompt.strip() + native_image_tensors = _prepare_gemma4_native_image_tensors(processor, image_files) + native_audio: torch.Tensor | None = None + native_audio_mask: torch.Tensor | None = None + native_audio_frames: int | None = None + if audio_file: + expected_mels = _resolve_gemma4_audio_mels(processor) + try: + native_audio, native_audio_mask, native_audio_frames = prepare_native_gemma4_audio_features( + audio_file, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + max_seconds=audio_duration_limit_seconds(), + pad_to_max_seconds=True, + ) + except Exception as exc: + print(f"note=falling back to processor gemma4 audio features: {exc}") + native_audio = None + native_audio_mask = None + native_audio_frames = None + + if native_image_tensors is not None and (not audio_file or native_audio is not None): + pixel_values, pixel_position_ids = native_image_tensors + pooling_kernel_size = int(_get_processor_image_attr(processor, "pooling_kernel_size", 3)) + image_soft_counts = tuple( + _gemma4_image_soft_token_counts( + pixel_position_ids, + pooling_kernel_size=pooling_kernel_size, + ) + ) + audio_soft_count = _gemma4_audio_soft_token_count(native_audio_frames or 0) if audio_file else 0 + processor_prompt = _build_gemma4_native_chat_prompt( + prompt=normalized_prompt, + image_soft_token_counts=image_soft_counts, + audio_soft_token_count=audio_soft_count, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + batch = _tokenize_gemma4_native_prompt( + processor, + processor_prompt, + image_token=image_token if isinstance(image_token, str) else None, + audio_token=audio_token if isinstance(audio_token, str) else None, + ) + batch["pixel_values"] = pixel_values + batch["pixel_position_ids"] = pixel_position_ids + if native_audio is not None and native_audio_mask is not None: + batch["input_features"] = native_audio + batch["input_features_mask"] = native_audio_mask + batch["native_audio_frames"] = int(native_audio_frames or native_audio.shape[1]) + else: + processor_prompt = _build_gemma4_chat_prompt( + prompt=normalized_prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + batch = processor( + text=processor_prompt, + images=images or None, + audio=audio_waveforms or None, + return_tensors="pt", + ) + _gemma4_split_cactus_newline_token_merges(batch) + if native_image_tensors is not None: + batch["pixel_values"], batch["pixel_position_ids"] = native_image_tensors + else: + batch = processor( + text=processor_prompt, + images=images or None, + audio=audio_waveforms or None, + return_tensors="pt", + ) + + ordered_keys = [ + key + for key in _GEMMA4_MULTIMODAL_INPUT_ORDER + if isinstance(batch.get(key), torch.Tensor) + ] + if not ordered_keys: + raise RuntimeError("Gemma4 multimodal processor did not return any tensor inputs") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + metadata: dict[str, object] = { + "prompt": normalized_prompt, + "processor_prompt": processor_prompt, + "image_files": [str(Path(path).resolve()) for path in image_files], + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + } + if audio_file: + metadata["audio_file"] = str(Path(audio_file).resolve()) + if sample_rate is not None: + metadata["sample_rate"] = sample_rate + native_audio_frames = batch.get("native_audio_frames") + if isinstance(native_audio_frames, int): + metadata["native_audio_frames"] = native_audio_frames + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata=metadata, + ) + + +_patch_transformers_torchvision_probe = patch_transformers_torchvision_probe +_patch_torch_flex_attention_compat = patch_torch_flex_attention_compat +_prepare_gemma4_multimodal_inputs = prepare_gemma4_multimodal_inputs diff --git a/python/cactus/transpile/normalize.py b/python/cactus/transpile/normalize.py new file mode 100644 index 000000000..c7a0382a8 --- /dev/null +++ b/python/cactus/transpile/normalize.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import Any + +import torch + +from cactus.transpile.aten_ops import canonical_ir_op + + +def normalize_target(target: Any) -> str: + return canonical_ir_op(target) + + +def dtype_to_ir(dtype: Any | None) -> str | None: + if dtype is None: + return None + + dtype_map = { + torch.float16: "fp16", + torch.float32: "fp32", + torch.float64: "fp64", + torch.bfloat16: "bf16", + torch.int8: "int8", + torch.int16: "int16", + torch.int32: "int32", + torch.int64: "int64", + torch.bool: "bool", + } + return dtype_map.get(dtype, str(dtype)) diff --git a/python/cactus/transpile/capture/ops.py b/python/cactus/transpile/ops.py similarity index 64% rename from python/cactus/transpile/capture/ops.py rename to python/cactus/transpile/ops.py index b0340e4ed..e1a870f8b 100644 --- a/python/cactus/transpile/capture/ops.py +++ b/python/cactus/transpile/ops.py @@ -24,7 +24,16 @@ class OpSchema: OpSchema("multiply", num_inputs=2, backend_op="multiply", is_fusible=True), OpSchema("divide", num_inputs=2, backend_op="divide", is_fusible=True), OpSchema("not_equal", num_inputs=2, backend_op="not_equal"), + OpSchema("equal", num_inputs=2), + OpSchema("greater", num_inputs=2), + OpSchema("greater_equal", num_inputs=2), + OpSchema("less", num_inputs=2), + OpSchema("less_equal", num_inputs=2), + OpSchema("logical_and", num_inputs=2), + OpSchema("logical_or", num_inputs=2), + OpSchema("logical_not", num_inputs=1), OpSchema("abs", num_inputs=1, backend_op="abs", is_fusible=True), + OpSchema("clamp", num_inputs=1, attrs=("min", "max")), OpSchema("pow", num_inputs=1, attrs=("exponent",), backend_op="pow"), OpSchema("negate", num_inputs=1, is_fusible=True), OpSchema("scalar_add", num_inputs=1, attrs=("value",), backend_op="scalar_add", is_fusible=True), @@ -32,8 +41,14 @@ class OpSchema: OpSchema("scalar_subtract_reverse", num_inputs=1, attrs=("value",)), OpSchema("scalar_multiply", num_inputs=1, attrs=("value",), backend_op="scalar_multiply", is_fusible=True), OpSchema("scalar_divide", num_inputs=1, attrs=("value",), backend_op="scalar_divide", is_fusible=True), + OpSchema("scalar_floor_divide", num_inputs=1, attrs=("value",), backend_op="scalar_floor_divide"), OpSchema("scalar_divide_reverse", num_inputs=1, attrs=("value",)), OpSchema("scalar_not_equal", num_inputs=1, attrs=("value",), backend_op="scalar_not_equal"), + OpSchema("scalar_equal", num_inputs=1, attrs=("value",)), + OpSchema("scalar_greater", num_inputs=1, attrs=("value",)), + OpSchema("scalar_greater_equal", num_inputs=1, attrs=("value",)), + OpSchema("scalar_less", num_inputs=1, attrs=("value",)), + OpSchema("scalar_less_equal", num_inputs=1, attrs=("value",)), OpSchema("scalar_exp", num_inputs=1, backend_op="scalar_exp", is_fusible=True), OpSchema("scalar_sqrt", num_inputs=1, backend_op="scalar_sqrt", is_fusible=True), OpSchema("scalar_log", num_inputs=1, backend_op="scalar_log", is_fusible=True), @@ -42,27 +57,43 @@ class OpSchema: OpSchema("precision_cast", num_inputs=1, attrs=("dtype",), backend_op="precision_cast"), OpSchema("view", num_inputs=1, attrs=("shape",), backend_op="reshape", is_layout=True), OpSchema("permute", num_inputs=1, attrs=("permutation",), backend_op="permute", is_layout=True), + OpSchema("squeeze", num_inputs=1, attrs=("dim",), is_layout=True), + OpSchema("movedim", num_inputs=1, attrs=("source", "destination"), is_layout=True), OpSchema("slice", num_inputs=1, attrs=("axis", "start", "end", "step"), backend_op="slice"), OpSchema("split_with_sizes", num_inputs=1, attrs=("sizes", "axis")), OpSchema("chunk", num_inputs=1, attrs=("chunks", "axis")), OpSchema("ones", num_inputs=0, attrs=("shape", "dtype")), + OpSchema("one_hot", num_inputs=1, attrs=("num_classes",)), + OpSchema("tril", num_inputs=1, attrs=("diagonal",)), + OpSchema("unfold", num_inputs=1, attrs=("dimension", "size", "step")), OpSchema("index", num_inputs=1, attrs=("axis", "index_value"), backend_op="index"), + OpSchema("advanced_index", num_inputs=None), + OpSchema( + "where", + num_inputs=None, + attrs=("true_is_scalar", "true_value", "false_is_scalar", "false_value"), + ), + OpSchema("masked_fill", num_inputs=2, attrs=("value",)), + OpSchema("masked_scatter", num_inputs=3), + OpSchema("getitem", num_inputs=1, attrs=("index",)), OpSchema("cat", num_inputs=None, attrs=("axis",), backend_op="cat"), OpSchema("matmul", num_inputs=2, backend_op="matmul"), OpSchema("linear", num_inputs=None, attrs=("has_bias",), backend_op="matmul"), OpSchema("addmm", num_inputs=3), - OpSchema("gather", num_inputs=2, backend_op="gather"), + OpSchema("gather", num_inputs=2, attrs=("axis",), backend_op="gather"), OpSchema("embedding", num_inputs=2, backend_op="embedding_from_tensor"), OpSchema("sum", num_inputs=1, attrs=("axis", "keepdim"), backend_op="sum", is_reduction=True), OpSchema("mean", num_inputs=1, attrs=("axis", "keepdim"), backend_op="mean", is_reduction=True), OpSchema("variance", num_inputs=1, attrs=("axis", "keepdim"), backend_op="variance", is_reduction=True), OpSchema("min", num_inputs=1, attrs=("axis", "keepdim"), backend_op="min", is_reduction=True), OpSchema("max", num_inputs=1, attrs=("axis", "keepdim"), backend_op="max", is_reduction=True), + OpSchema("cumsum", num_inputs=1, attrs=("axis",), backend_op="cumsum"), OpSchema("relu", num_inputs=1, backend_op="relu", is_fusible=True), OpSchema("silu", num_inputs=1, backend_op="silu", is_fusible=True), OpSchema("gelu", num_inputs=1, backend_op="gelu", is_fusible=True), OpSchema("gelu_erf", num_inputs=1, backend_op="gelu_erf", is_fusible=True), OpSchema("sigmoid", num_inputs=1, backend_op="sigmoid", is_fusible=True), + OpSchema("glu", num_inputs=1, attrs=("axis",), backend_op="glu"), OpSchema("tanh", num_inputs=1, backend_op="tanh", is_fusible=True), OpSchema("softplus", num_inputs=1), OpSchema("softmax", num_inputs=1, attrs=("axis",), backend_op="softmax"), @@ -70,21 +101,86 @@ class OpSchema: OpSchema("group_norm", num_inputs=3, attrs=("num_groups", "eps"), backend_op="group_norm"), OpSchema("batch_norm", num_inputs=5, attrs=("axis", "eps"), backend_op="batch_norm"), OpSchema("conv1d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv1d"), + OpSchema("conv2d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv2d"), OpSchema("pad", num_inputs=1, attrs=("pads", "value", "mode")), + OpSchema("lstm_cell", num_inputs=7, backend_op="lstm_cell"), # CUSTOM / FUSED OPS + OpSchema("dense_mlp_tq_fused", num_inputs=4, attrs=("product_scale",), backend_op="dense_mlp_tq_fused"), OpSchema("rms_norm", num_inputs=2, attrs=("eps",), backend_op="rms_norm"), OpSchema("rope", num_inputs=1, attrs=("theta", "position_offset"), backend_op="rope"), + OpSchema("rel_pos_bias", num_inputs=2, attrs=("scale",), backend_op="rel_pos_bias"), OpSchema( "attention", num_inputs=3, - attrs=("scale", "is_causal", "window_size", "mask", "additive_mask"), + attrs=( + "scale", + "is_causal", + "window_size", + "mask", + "additive_mask", + "qkv_layout", + "q_layout", + "k_layout", + "v_layout", + "output_layout", + ), backend_op="attention", ), OpSchema( "attention_block", num_inputs=None, - attrs=("scale", "is_causal", "window_size", "has_gate", "has_bias", "attention_output_shape"), + attrs=( + "scale", + "is_causal", + "window_size", + "has_mask", + "additive_mask", + "has_gate", + "has_bias", + "attention_output_shape", + "qkv_layout", + ), + is_fusible=True, + ), + OpSchema( + "self_attention_block", + num_inputs=None, + attrs=( + "scale", + "is_causal", + "window_size", + "has_mask", + "additive_mask", + "has_gate", + "has_bias", + "has_query_projection_bias", + "has_query_add", + "has_rel_query_add", + "has_key_projection_bias", + "has_value_projection_bias", + "has_rel_pos_bias", + "has_relative_key_projection_bias", + "query_shape", + "key_shape", + "value_shape", + "relative_key_shape", + "rel_pos_scale", + "attention_output_shape", + ), + is_fusible=True, + ), + OpSchema( + "conv_module", + num_inputs=None, + attrs=( + "eps", + "has_pointwise1_bias", + "has_depthwise_bias", + "has_pointwise2_bias", + "depthwise_kernel_size", + "depthwise_padding", + ), is_fusible=True, ), OpSchema( diff --git a/python/cactus/transpile/optimize_graph.py b/python/cactus/transpile/optimize_graph.py new file mode 100644 index 000000000..1afa19651 --- /dev/null +++ b/python/cactus/transpile/optimize_graph.py @@ -0,0 +1,1469 @@ +from __future__ import annotations + +from dataclasses import dataclass +import re + +import torch + +from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph +from cactus.transpile.canonicalize.utils import normalize_dtype_name +from cactus.transpile.canonicalize.utils import remove_node +from cactus.transpile.canonicalize.utils import rebuild_graph +from cactus.transpile.fusion import match_attention +from cactus.transpile.fusion import match_attention_block +from cactus.transpile.fusion import match_conv_module +from cactus.transpile.fusion import match_gated_deltanet +from cactus.transpile.fusion import match_gated_mlp +from cactus.transpile.fusion import match_lstm_cell +from cactus.transpile.fusion import match_rel_pos_bias +from cactus.transpile.fusion import match_rms_norm +from cactus.transpile.fusion import match_rope +from cactus.transpile.fusion import match_self_attention_block +from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_passthrough +from cactus.transpile.fusion.linear import match_linear +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.graph_ir import verify_ir +from cactus.transpile.model_patterns import GOLD_PATTERNS +from cactus.transpile.normalize import dtype_to_ir + + +@dataclass(frozen=True) +class DetectedPattern: + name: str + anchor_node_id: str + node_ids: tuple[str, ...] + value_ids: tuple[str, ...] + details: dict[str, object] + + +@dataclass(frozen=True) +class FusionConfig: + enable_gated_deltanet: bool = True + enable_lstm_cell: bool = True + enable_rms_norm: bool = True + enable_rope: bool = True + enable_rel_pos_bias: bool = True + enable_attention: bool = True + enable_self_attention_block: bool = True + enable_attention_block: bool = True + enable_conv_module: bool = True + enable_add_clipped: bool = True + enable_dense_mlp_tq_fused: bool = True + + +def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None) -> IRGraph: + config = config or FusionConfig() + verify_ir(graph) + canonicalize_exported_graph(graph) + enable_attention_block_fusions = not ( + _is_gemma4_graph(graph) or _is_whisper_seq2seq_decoder_graph(graph) + ) + + for _ in range(max_passes): + changed = False + if config.enable_gated_deltanet and fuse_gated_deltanet(graph): + changed = True + if config.enable_lstm_cell and fuse_lstm_cells(graph): + changed = True + if config.enable_rms_norm: + if fuse_rms_norm(graph): + changed = True + if fuse_rms_norm_scale_multiply(graph): + changed = True + if config.enable_rope and fuse_rope(graph): + changed = True + if config.enable_rel_pos_bias and fuse_rel_pos_bias(graph): + changed = True + if config.enable_attention and fuse_attention(graph): + changed = True + if normalize_attention_layouts(graph): + changed = True + if ( + enable_attention_block_fusions + and config.enable_self_attention_block + and fuse_self_attention_blocks(graph) + ): + changed = True + if ( + enable_attention_block_fusions + and config.enable_attention_block + and fuse_attention_blocks(graph) + ): + changed = True + if normalize_gemma4_decoder_attention_semantics(graph): + changed = True + if config.enable_dense_mlp_tq_fused and fuse_dense_mlp_tq(graph): + changed = True + if config.enable_conv_module and fuse_conv_modules(graph): + changed = True + if config.enable_add_clipped and fuse_add_clipped(graph): + changed = True + if not changed: + break + canonicalize_exported_graph(graph) + + annotate_gold_patterns(graph) + _prune_unused_inputs(graph) + verify_ir(graph) + return graph + + +def fuse_gated_deltanet(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_gated_deltanet(graph, node) + if match is None: + continue + + inputs = [ + match.normalized_input_value_id, + match.qkv_weight_value_id, + match.a_weight_value_id, + match.b_weight_value_id, + match.norm_weight_value_id, + ] + if match.z_weight_value_id is not None: + inputs.append(match.z_weight_value_id) + if match.dt_bias_value_id is not None: + inputs.append(match.dt_bias_value_id) + if match.a_log_value_id is not None: + inputs.append(match.a_log_value_id) + if match.conv_weight_value_id is not None: + inputs.append(match.conv_weight_value_id) + + node.op = f"gated_deltanet_{match.mode}" + node.inputs = inputs + node.attrs = { + "num_k_heads": int(match.num_k_heads), + "num_v_heads": int(match.num_v_heads), + "key_dim": int(match.key_dim), + "value_dim": int(match.value_dim), + "eps": float(match.eps), + "chunk_size": int(match.chunk_size), + "has_z": bool(match.z_weight_value_id is not None), + "has_dt_bias": bool(match.dt_bias_value_id is not None), + "has_a_log": bool(match.a_log_value_id is not None), + "has_conv": bool(match.conv_weight_value_id is not None), + } + node.kind = "semantic" + node.meta["gated_deltanet_mode"] = match.mode + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_lstm_cells(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "multiply": + continue + + match = match_lstm_cell(graph, node) + if match is None: + continue + + bias_hh_value_id = match.bias_hh_value_id + if bias_hh_value_id is None: + bias_hh_value_id = _materialize_zero_like_constant( + graph, + match.bias_ih_value_id, + suffix="lstm_bias_hh_zero", + ) + + node.op = "lstm_cell" + node.inputs = [ + match.x_value_id, + match.h_prev_value_id, + match.c_prev_value_id, + match.weight_ih_value_id, + match.weight_hh_value_id, + match.bias_ih_value_id, + bias_hh_value_id, + ] + node.outputs = [match.h_output_value_id, match.c_output_value_id] + node.attrs = {} + node.kind = "semantic" + node.meta["lstm_cell_nodes"] = match.node_ids + node.meta["lstm_cell_bias_hh_zero"] = bool(match.bias_hh_value_id is None) + + for other_node_id in match.node_ids: + if other_node_id == node.id: + continue + graph.nodes.pop(other_node_id, None) + if other_node_id in graph.order: + graph.order.remove(other_node_id) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def annotate_gold_patterns(graph: IRGraph) -> list[DetectedPattern]: + _clear_gold_pattern_annotations(graph) + patterns = [ + *_detect_gated_mlps(graph), + *_detect_decoder_attentions(graph), + *_detect_transformer_blocks(graph), + ] + + for pattern in patterns: + for node_id in pattern.node_ids: + node = graph.nodes.get(node_id) + if node is None: + continue + node.meta.setdefault("gold_patterns", []) + node.meta["gold_patterns"].append(pattern.name) + + anchor = graph.nodes.get(pattern.anchor_node_id) + if anchor is not None: + anchor.meta["gold_pattern_anchor"] = True + anchor.meta["gold_pattern_details"] = pattern.details + + graph.meta["gold_patterns_catalog"] = tuple(pattern.name for pattern in GOLD_PATTERNS) + graph.meta["detected_gold_patterns"] = patterns + return patterns + + +def summarize_detected_gold_patterns(graph: IRGraph) -> dict[str, int]: + summary: dict[str, int] = {} + for pattern in annotate_gold_patterns(graph): + summary[pattern.name] = summary.get(pattern.name, 0) + 1 + return summary + + +def fuse_rms_norm(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"multiply", "type_as", "precision_cast"}: + continue + + match = match_rms_norm(graph, node) + if match is None: + continue + input_value = graph.values.get(match.input_value_id) + if input_value is not None and normalize_dtype_name(input_value.dtype) == "fp32": + # Keep FP32 normalization paths unfused so they can execute with FP32 + # reductions and elementwise math instead of the FP16-only RMS kernel. + continue + + weight_value_id = match.weight_value_id + if weight_value_id is None: + if input_value is None or input_value.shape is None or not input_value.shape: + continue + hidden_dim = input_value.shape[-1] + if not isinstance(hidden_dim, int) or hidden_dim <= 0: + continue + weight_value_id = _materialize_ones_constant( + graph, + hidden_dim, + dtype=input_value.dtype, + suffix="rms_norm_ones", + ) + if float(match.weight_offset) != 0.0: + weight_value_id = _materialize_shifted_constant( + graph, + weight_value_id, + float(match.weight_offset), + suffix="rms_norm_scale", + ) + + node.op = "rms_norm" + node.inputs = [match.input_value_id, weight_value_id] + node.attrs = {"eps": float(match.eps)} + node.kind = "semantic" + node.meta["rms_weight_offset"] = float(match.weight_offset) + node.meta["rms_input_value_id"] = match.input_value_id + node.meta["rms_weight_value_id"] = weight_value_id + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rms_norm_scale_multiply(graph: IRGraph) -> bool: + if not _is_gemma4_graph(graph): + return False + + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "multiply" or len(node.inputs) != 2 or len(node.outputs) != 1: + continue + + rms_node: IRNode | None = None + rms_output_id: str | None = None + scale_value_id: str | None = None + for input_id, other_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + candidate = producer(graph, input_id) + if candidate is None or candidate.op != "rms_norm" or len(candidate.inputs) != 2: + continue + if not _is_materialized_ones_constant(graph, candidate.inputs[1]): + continue + rms_node = candidate + rms_output_id = input_id + scale_value_id = other_id + break + + if rms_node is None or rms_output_id is None or scale_value_id is None: + continue + if rms_node.outputs != [rms_output_id]: + continue + if _has_other_users(graph, rms_output_id, excluding=node.id) or rms_output_id in graph.outputs: + continue + + node.op = "rms_norm" + node.inputs = [rms_node.inputs[0], scale_value_id] + node.attrs = dict(rms_node.attrs) + node.kind = "semantic" + node.meta["rms_folded_scale_multiply"] = True + node.meta["rms_input_value_id"] = rms_node.inputs[0] + node.meta["rms_weight_value_id"] = scale_value_id + remove_node(graph, rms_node.id) + graph.values.pop(rms_output_id, None) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rope(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or not node.outputs: + continue + + match = match_rope(graph, node.outputs[0]) + if match is None or match.partial: + continue + + node.op = "rope" + node.inputs = [match.input_value_id] + node.attrs = { + "theta": float(match.theta), + "position_offset": int(match.position_offset), + } + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_attention(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: + continue + + match = match_attention(graph, node) + if match is None: + continue + + window_size = int(node.attrs.get("window_size", 0)) + if len(node.inputs) > 3: + mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) + if mask_info is not None: + node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) + if window_size == 0: + window_size = int(mask_info["window_size"]) + node.meta["window_size_source"] = "mask_pattern" + + if window_size != 0 and "window_size_source" not in node.meta: + node.meta["window_size_source"] = "import_attr" + + semantic_attrs = { + key: value + for key, value in node.attrs.items() + if key not in {"mask", "dropout_p", "scale", "is_causal"} + } + semantic_attrs.update( + { + "scale": float(node.attrs.get("scale", 0.0)), + "is_causal": bool(node.attrs.get("is_causal", True)), + "window_size": window_size, + } + ) + + if ( + node.op == "attention" + and node.inputs[:3] == [match.query_value_id, match.key_value_id, match.value_value_id] + and node.attrs == semantic_attrs + ): + continue + + node.op = "attention" + node.inputs = [match.query_value_id, match.key_value_id, match.value_value_id] + node.attrs = semantic_attrs + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rel_pos_bias(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_rel_pos_bias(graph, node) + if match is None: + continue + + node.op = "rel_pos_bias" + node.inputs = [match.query_value_id, match.relative_key_value_id] + node.attrs = {"scale": float(match.scale)} + node.kind = "semantic" + node.meta["rel_pos_bias_nodes"] = match.node_ids + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_attention_blocks(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_attention_block(graph, node) + if match is None: + continue + + qkv_inputs = [match.query_value_id, match.key_value_id, match.value_value_id] + qkv_layout = "bhsd" + native_qkv_inputs = [ + _strip_bhsd_to_bthd_permute_input(graph, value_id) + for value_id in qkv_inputs + ] + if all(value_id is not None for value_id in native_qkv_inputs): + qkv_inputs = [str(value_id) for value_id in native_qkv_inputs] + qkv_layout = "bthd" + + inputs = qkv_inputs + if match.mask_value_id is not None: + inputs.append(match.mask_value_id) + if match.gate_value_id is not None: + inputs.append(match.gate_value_id) + inputs.append(match.output_projection_weight_value_id) + if match.output_projection_bias_value_id is not None: + inputs.append(match.output_projection_bias_value_id) + + node.op = "attention_block" + node.inputs = inputs + node.attrs = { + "scale": float(match.scale), + "is_causal": bool(match.is_causal), + "window_size": int(match.window_size), + "has_mask": bool(match.mask_value_id is not None), + "additive_mask": bool(match.additive_mask), + "has_gate": bool(match.gate_value_id is not None), + "has_bias": bool(match.output_projection_bias_value_id is not None), + "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), + "qkv_layout": qkv_layout, + } + node.kind = "semantic" + node.meta["attention_block_source"] = match.attention_node_id + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def _strip_bhsd_to_bthd_permute_input(graph: IRGraph, value_id: str) -> str | None: + value = graph.values.get(value_id) + if value is None or value.producer is None: + return None + node = graph.nodes.get(value.producer) + if node is None or node.op != "permute" or len(node.inputs) != 1: + return None + permutation = tuple(int(dim) for dim in node.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + return None + source_id = strip_passthrough(graph, node.inputs[0]) + source_value = graph.values.get(source_id) + if source_value is None or source_value.shape is None or value.shape is None: + return None + source_shape = tuple(int(dim) for dim in source_value.shape) + output_shape = tuple(int(dim) for dim in value.shape) + if len(source_shape) != 4 or len(output_shape) != 4: + return None + if ( + output_shape[0] == source_shape[0] + and output_shape[1] == source_shape[2] + and output_shape[2] == source_shape[1] + and output_shape[3] == source_shape[3] + ): + return source_id + return None + + +def normalize_attention_layouts(graph: IRGraph) -> bool: + """Erase PyTorch BHSD<->BTHD layout round-trips around attention ops.""" + + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"attention", "scaled_dot_product_attention"}: + continue + if len(node.inputs) < 3 or len(node.outputs) != 1: + continue + + node_changed = False + layouts = ["bhsd", "bhsd", "bhsd"] + for index, value_id in enumerate(node.inputs[:3]): + native_value_id = _strip_bhsd_to_bthd_permute_input(graph, value_id) + if native_value_id is None: + continue + node.inputs[index] = str(native_value_id) + layouts[index] = "bthd" + node_changed = True + if node_changed: + node.attrs["q_layout"] = layouts[0] + node.attrs["k_layout"] = layouts[1] + node.attrs["v_layout"] = layouts[2] + changed = True + + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + output_users = list(output_value.users) if output_value is not None else [] + if len(output_users) != 1: + continue + output_user = graph.nodes.get(output_users[0]) + if output_user is None or output_user.op != "permute" or len(output_user.inputs) != 1 or len(output_user.outputs) != 1: + continue + permutation = tuple(int(dim) for dim in output_user.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + continue + + replacement_id = output_user.outputs[0] + replacement_value = graph.values.get(replacement_id) + if replacement_value is not None: + old_value = graph.values.get(output_id) + if old_value is not None: + old_value.shape = replacement_value.shape + old_value.dtype = replacement_value.dtype + old_value.meta.update(replacement_value.meta) + node.outputs = [replacement_id] + node.attrs["output_layout"] = "bthd" + remove_node(graph, output_user.id) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def normalize_gemma4_decoder_attention_semantics(graph: IRGraph) -> bool: + if not _is_gemma4_graph(graph): + return False + + changed = False + if _assign_gemma4_decoder_attention_hints_from_graph_meta(graph): + changed = True + sliding_window = _graph_sliding_window(graph) + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"attention", "scaled_dot_product_attention", "attention_block"}: + continue + layer_type = str(node.meta.get("attention_layer_type") or "").strip().lower() + if layer_type == "full_attention": + mask_input_index: int | None = None + if node.op == "attention_block": + if bool(node.attrs.get("has_mask", False)) and len(node.inputs) > 3: + mask_input_index = 3 + elif len(node.inputs) > 3: + mask_input_index = 3 + + if mask_input_index is not None: + mask_value_id = node.inputs[mask_input_index] + if _gemma4_can_elide_attention_mask(graph, mask_value_id): + del node.inputs[mask_input_index] + node.meta["gemma4_full_mask_elided"] = True + changed = True + + if node.op == "attention_block" and bool(node.attrs.get("has_mask", False)): + node.attrs["has_mask"] = False + changed = True + if not bool(node.attrs.get("is_causal", False)): + node.attrs["is_causal"] = True + changed = True + if "additive_mask" in node.attrs and not bool(node.attrs.get("additive_mask", False)): + node.attrs.pop("additive_mask", None) + changed = True + if int(node.attrs.get("window_size", 0)) == 0: + seq_len = 0 + attention_output_shape = node.attrs.get("attention_output_shape") + if isinstance(attention_output_shape, (list, tuple)) and len(attention_output_shape) >= 2: + try: + seq_len = int(attention_output_shape[1]) + except (TypeError, ValueError): + seq_len = 0 + if seq_len <= 0 and node.inputs: + query_value = graph.values.get(node.inputs[0]) + if query_value is not None and query_value.shape is not None and len(query_value.shape) >= 3: + seq_len = int(query_value.shape[2]) + if seq_len > 0: + # Gemma4 full-attention layers are semantically unwindowed, but using a + # window_size >= seq_len avoids the Apple Accelerate full-window fast path + # that has been unstable for transpiled grouped-query Gemma4 graphs. + node.attrs["window_size"] = seq_len + node.meta["gemma4_full_attention_window_compat"] = True + changed = True + continue + if layer_type != "sliding_attention": + continue + if bool(node.attrs.get("additive_mask", False)): + continue + + mask_input_index: int | None = None + if node.op == "attention_block": + if bool(node.attrs.get("has_mask", False)) and len(node.inputs) > 3: + mask_input_index = 3 + elif len(node.inputs) > 3: + mask_input_index = 3 + + if mask_input_index is not None: + mask_value_id = node.inputs[mask_input_index] + if _gemma4_can_elide_attention_mask(graph, mask_value_id): + mask_info = _extract_sliding_window_mask(graph, mask_value_id) + mask_input_name = _graph_input_name(graph, mask_value_id) + del node.inputs[mask_input_index] + node.meta["gemma4_sliding_mask_elided"] = True + if mask_info is not None: + node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) + if int(node.attrs.get("window_size", 0)) == 0 and int(mask_info["window_size"]) > 0: + node.attrs["window_size"] = int(mask_info["window_size"]) + node.meta.setdefault("window_size_source", "mask_pattern") + elif mask_input_name is not None: + node.meta["gemma4_elided_mask_input_name"] = mask_input_name + changed = True + + if node.op == "attention_block" and bool(node.attrs.get("has_mask", False)): + node.attrs["has_mask"] = False + changed = True + if not bool(node.attrs.get("is_causal", False)): + node.attrs["is_causal"] = True + changed = True + if ( + int(node.attrs.get("window_size", 0)) == 0 + and sliding_window is not None + and sliding_window > 0 + ): + node.attrs["window_size"] = int(sliding_window) + node.meta.setdefault("window_size_source", "layer_type_config") + changed = True + if "additive_mask" in node.attrs and not bool(node.attrs.get("additive_mask", False)): + node.attrs.pop("additive_mask", None) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bool: + component = str(graph.meta.get("component", "") or "").strip().lower() + if component != "decoder": + return False + + layer_types = _graph_layer_types(graph) + if not layer_types: + return False + + attention_nodes: list[IRNode] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"attention", "scaled_dot_product_attention", "attention_block"}: + continue + attention_nodes.append(node) + + if len(attention_nodes) != len(layer_types): + return False + + changed = False + for layer_index, (node, layer_type) in enumerate(zip(attention_nodes, layer_types, strict=True)): + if "attention_layer_type" not in node.meta: + node.meta["attention_layer_type"] = str(layer_type) + changed = True + if "attention_layer_index" not in node.meta: + node.meta["attention_layer_index"] = int(layer_index) + changed = True + return changed + + +def _gemma4_can_elide_attention_mask( + graph: IRGraph, + value_id: str, + *, + _visited: set[str] | None = None, +) -> bool: + mask_value = graph.values.get(value_id) + if mask_value is None: + return False + if mask_value.dtype in (None, "bool"): + return True + + input_name = _graph_input_name(graph, value_id) + if isinstance(input_name, str) and "attention_mask" in input_name: + return True + + if _visited is None: + _visited = set() + if value_id in _visited: + return False + _visited.add(value_id) + + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + return False + if node.op in { + "logical_and", + "logical_or", + "scalar_equal", + "scalar_not_equal", + "greater", + "greater_equal", + "less", + "less_equal", + "aten.__and__.Tensor", + }: + return True + if node.op in {"precision_cast", "view", "expand", "permute", "slice", "index", "where"}: + return any(_gemma4_can_elide_attention_mask(graph, input_id, _visited=_visited) for input_id in node.inputs) + return False + + +def fuse_self_attention_blocks(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_self_attention_block(graph, node) + if match is None: + continue + + inputs = [match.hidden_value_id, match.query_weight_value_id] + if match.query_projection_bias_value_id is not None: + inputs.append(match.query_projection_bias_value_id) + if match.query_add_value_id is not None: + inputs.append(match.query_add_value_id) + if match.rel_query_add_value_id is not None: + inputs.append(match.rel_query_add_value_id) + inputs.append(match.key_weight_value_id) + if match.key_projection_bias_value_id is not None: + inputs.append(match.key_projection_bias_value_id) + inputs.append(match.value_weight_value_id) + if match.value_projection_bias_value_id is not None: + inputs.append(match.value_projection_bias_value_id) + if match.mask_value_id is not None: + inputs.append(match.mask_value_id) + if match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None: + inputs.append(match.relative_key_input_value_id) + inputs.append(match.relative_key_weight_value_id) + if match.relative_key_projection_bias_value_id is not None: + inputs.append(match.relative_key_projection_bias_value_id) + if match.gate_value_id is not None: + inputs.append(match.gate_value_id) + inputs.append(match.output_projection_weight_value_id) + if match.output_projection_bias_value_id is not None: + inputs.append(match.output_projection_bias_value_id) + + node.op = "self_attention_block" + node.inputs = inputs + node.attrs = { + "scale": float(match.scale), + "is_causal": bool(match.is_causal), + "window_size": int(match.window_size), + "has_mask": bool(match.mask_value_id is not None), + "additive_mask": bool(match.additive_mask), + "has_gate": bool(match.gate_value_id is not None), + "has_bias": bool(match.output_projection_bias_value_id is not None), + "has_query_projection_bias": bool(match.query_projection_bias_value_id is not None), + "has_query_add": bool(match.query_add_value_id is not None), + "has_rel_query_add": bool(match.rel_query_add_value_id is not None), + "has_key_projection_bias": bool(match.key_projection_bias_value_id is not None), + "has_value_projection_bias": bool(match.value_projection_bias_value_id is not None), + "has_rel_pos_bias": bool(match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None), + "has_relative_key_projection_bias": bool(match.relative_key_projection_bias_value_id is not None), + "query_shape": tuple(int(dim) for dim in match.query_shape), + "key_shape": tuple(int(dim) for dim in match.key_shape), + "value_shape": tuple(int(dim) for dim in match.value_shape), + "relative_key_shape": tuple(int(dim) for dim in match.relative_key_shape) if match.relative_key_shape is not None else (), + "rel_pos_scale": float(match.rel_pos_scale) if match.rel_pos_scale is not None else 1.0, + "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), + } + node.kind = "semantic" + node.meta["self_attention_block_source"] = match.attention_node_id + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_add_clipped(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "add" or len(node.inputs) != 2: + continue + + lhs = strip_passthrough(graph, node.inputs[0]) + rhs = strip_passthrough(graph, node.inputs[1]) + if not (_looks_like_gemma_residual_add(graph, lhs, rhs) or _looks_like_gemma_residual_add(graph, rhs, lhs)): + continue + + node.op = "add_clipped" + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_dense_mlp_tq(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = _match_dense_mlp_tq(graph, node) + if match is None: + continue + + matched_node_ids = set(match["node_ids"]) + if not _matched_nodes_are_private(graph, matched_node_ids, keep_node_id=node.id): + continue + + node.op = "dense_mlp_tq_fused" + node.inputs = [ + str(match["input_value_id"]), + str(match["gate_weight_value_id"]), + str(match["up_weight_value_id"]), + str(match["down_weight_value_id"]), + ] + node.attrs = {"product_scale": float(match.get("product_scale") or 1.0)} + node.kind = "semantic" + node.meta["dense_mlp_tq_fused"] = True + node.meta["dense_mlp_tq_nodes"] = tuple(sorted(matched_node_ids)) + if match.get("product_scale") is not None: + node.meta["product_scale_from_export"] = float(match["product_scale"]) + + for other_node_id in matched_node_ids - {node.id}: + graph.nodes.pop(other_node_id, None) + if other_node_id in graph.order: + graph.order.remove(other_node_id) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def _match_dense_mlp_tq(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + down = match_linear(graph, node) + if down is None or down.bias_value_id is not None: + return None + if not _is_cq_weight_value(graph, down.weight_value_id): + return None + + mul_node = producer(graph, down.input_value_id) + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for activated_value_id, up_value_id in ((mul_node.inputs[0], mul_node.inputs[1]), (mul_node.inputs[1], mul_node.inputs[0])): + activation_node, product_scale, scale_node_id = _unwrap_gemma4_scaled_activation(graph, activated_value_id) + if activation_node is None or activation_node.op != "gelu" or len(activation_node.inputs) != 1: + continue + + gate_input_producer = producer(graph, strip_passthrough(graph, activation_node.inputs[0])) + up_producer = producer(graph, strip_passthrough(graph, up_value_id)) + if gate_input_producer is None or up_producer is None: + continue + + gate = match_linear(graph, gate_input_producer) + up = match_linear(graph, up_producer) + if gate is None or up is None: + continue + if gate.bias_value_id is not None or up.bias_value_id is not None: + continue + if strip_passthrough(graph, gate.input_value_id) != strip_passthrough(graph, up.input_value_id): + continue + if not _is_cq_weight_value(graph, gate.weight_value_id): + continue + if not _is_cq_weight_value(graph, up.weight_value_id): + continue + if not _dense_mlp_weight_shapes_match(graph, gate.weight_value_id, up.weight_value_id, down.weight_value_id): + continue + + node_ids = { + *gate.node_ids, + activation_node.id, + *up.node_ids, + mul_node.id, + *down.node_ids, + } + if scale_node_id is not None: + node_ids.add(scale_node_id) + + return { + "input_value_id": strip_passthrough(graph, gate.input_value_id), + "gate_weight_value_id": gate.weight_value_id, + "up_weight_value_id": up.weight_value_id, + "down_weight_value_id": down.weight_value_id, + "product_scale": product_scale, + "node_ids": tuple(sorted(node_ids)), + } + + return None + + +def _unwrap_gemma4_scaled_activation(graph: IRGraph, value_id: str) -> tuple[IRNode | None, float | None, str | None]: + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + return None, None, None + if node.op == "scalar_multiply" and len(node.inputs) == 1: + scale = float(node.attrs.get("value", 1.0)) + inner = producer(graph, strip_passthrough(graph, node.inputs[0])) + return inner, scale, node.id + return node, None, None + + +def _is_cq_weight_value(graph: IRGraph, value_id: str) -> bool: + value = graph.values.get(value_id) + if value is None: + return False + path = value.meta.get("path") if isinstance(value.meta, dict) else None + if isinstance(path, str) and ".cq" in path: + return True + return False + + +def _dense_mlp_weight_shapes_match( + graph: IRGraph, + gate_weight_value_id: str, + up_weight_value_id: str, + down_weight_value_id: str, +) -> bool: + gate = graph.values.get(gate_weight_value_id) + up = graph.values.get(up_weight_value_id) + down = graph.values.get(down_weight_value_id) + if gate is None or up is None or down is None: + return False + if gate.shape is None or up.shape is None or down.shape is None: + return False + if len(gate.shape) != 2 or len(up.shape) != 2 or len(down.shape) != 2: + return False + return ( + int(gate.shape[0]) == int(up.shape[0]) + and int(gate.shape[1]) == int(up.shape[1]) + and int(down.shape[1]) == int(gate.shape[0]) + ) + + +def _matched_nodes_are_private(graph: IRGraph, matched_node_ids: set[str], *, keep_node_id: str) -> bool: + for node_id in matched_node_ids: + if node_id == keep_node_id: + continue + node = graph.nodes.get(node_id) + if node is None: + continue + for output_id in node.outputs: + value = graph.values.get(output_id) + if value is None: + continue + for user_id in value.users: + if user_id not in matched_node_ids: + return False + return True + + +def fuse_conv_modules(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_conv_module(graph, node) + if match is None: + continue + + inputs = [ + match.input_value_id, + match.pointwise1_weight_value_id, + ] + if match.pointwise1_bias_value_id is not None: + inputs.append(match.pointwise1_bias_value_id) + inputs.append(match.depthwise_weight_value_id) + if match.depthwise_bias_value_id is not None: + inputs.append(match.depthwise_bias_value_id) + inputs.extend( + [ + match.batch_norm_weight_value_id, + match.batch_norm_bias_value_id, + match.batch_norm_running_mean_value_id, + match.batch_norm_running_var_value_id, + match.pointwise2_weight_value_id, + ] + ) + if match.pointwise2_bias_value_id is not None: + inputs.append(match.pointwise2_bias_value_id) + + node.op = "conv_module" + node.inputs = inputs + node.attrs = { + "eps": float(match.eps), + "has_pointwise1_bias": bool(match.pointwise1_bias_value_id is not None), + "has_depthwise_bias": bool(match.depthwise_bias_value_id is not None), + "has_pointwise2_bias": bool(match.pointwise2_bias_value_id is not None), + "depthwise_kernel_size": int(match.depthwise_kernel_size), + "depthwise_padding": int(match.depthwise_padding), + } + node.kind = "semantic" + node.meta["conv_module_nodes"] = match.node_ids + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def _materialize_shifted_constant(graph: IRGraph, value_id: str, offset: float, *, suffix: str) -> str: + base = graph.constants[value_id] + if not isinstance(base, torch.Tensor): + raise NotImplementedError(f"expected tensor constant for {value_id}, got {type(base).__name__}") + + new_value_id = f"{value_id}_{suffix}" + if new_value_id in graph.constants: + return new_value_id + + shifted = base.detach().cpu() + offset + graph.constants[new_value_id] = shifted + meta: dict[str, object] = {} + source_value = graph.values.get(value_id) + if _is_gemma4_graph(graph) and float(offset) == 1.0 and source_value is not None: + path = source_value.meta.get("path") + kind = source_value.meta.get("kind") + source_name = source_value.meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + meta = dict(source_value.meta) + meta["materialized_from_value_id"] = value_id + meta["materialized_by_op"] = "scalar_add" + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=tuple(shifted.shape), + dtype=dtype_to_ir(shifted.dtype), + producer=None, + users=[], + meta=meta, + ) + return new_value_id + + +def _materialize_zero_like_constant(graph: IRGraph, value_id: str, *, suffix: str) -> str: + base = graph.constants[value_id] + new_value_id = f"{value_id}_{suffix}" + if new_value_id in graph.constants: + return new_value_id + + if isinstance(base, torch.Tensor): + zero = torch.zeros_like(base.detach().cpu()) + else: + value = graph.values.get(value_id) + if value is None or value.shape is None or value.dtype is None: + raise NotImplementedError( + f"expected tensor constant metadata for {value_id}, got {type(base).__name__}" + ) + dtype_map = { + "bool": torch.bool, + "u8": torch.uint8, + "i8": torch.int8, + "i16": torch.int16, + "i32": torch.int32, + "i64": torch.int64, + "fp16": torch.float16, + "bf16": torch.bfloat16, + "fp32": torch.float32, + "fp64": torch.float64, + } + torch_dtype = dtype_map.get(str(value.dtype)) + if torch_dtype is None: + raise NotImplementedError(f"unsupported IR dtype for zero-like constant: {value.dtype}") + zero = torch.zeros(tuple(int(dim) for dim in value.shape), dtype=torch_dtype) + + graph.constants[new_value_id] = zero + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=tuple(zero.shape), + dtype=dtype_to_ir(zero.dtype), + producer=None, + users=[], + ) + return new_value_id + + +def _materialize_ones_constant(graph: IRGraph, size: int, *, dtype: str | None, suffix: str) -> str: + dtype_map = { + "fp16": torch.float16, + "bf16": torch.bfloat16, + "fp32": torch.float32, + "fp64": torch.float64, + } + torch_dtype = dtype_map.get(dtype, torch.float32) + new_value_id = f"c_{suffix}_{size}_{dtype or 'fp32'}" + if new_value_id in graph.constants: + return new_value_id + + value = torch.ones((size,), dtype=torch_dtype) + graph.constants[new_value_id] = value + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=(size,), + dtype=dtype_to_ir(value.dtype), + producer=None, + users=[], + ) + return new_value_id + + +def _is_gemma4_graph(graph: IRGraph) -> bool: + return str(graph.meta.get("adapter_family") or graph.meta.get("family") or "").lower() == "gemma4" + + +def _is_whisper_seq2seq_decoder_graph(graph: IRGraph) -> bool: + family = str(graph.meta.get("adapter_family") or graph.meta.get("family") or "").lower() + task = str(graph.meta.get("task") or "").lower() + component = str(graph.meta.get("component") or "").lower() + return family == "whisper" and task == "seq2seq_transcription" and component == "decoder" + + +def _prune_unused_inputs(graph: IRGraph) -> bool: + kept_inputs: list[str] = [] + changed = False + for value_id in graph.inputs: + value = graph.values.get(value_id) + if value_id in graph.outputs: + kept_inputs.append(value_id) + continue + if value is not None and value.users: + kept_inputs.append(value_id) + continue + changed = True + + if not changed: + return False + + graph.inputs = kept_inputs + rebuild_graph(graph) + return True + + +def _graph_input_name(graph: IRGraph, value_id: str) -> str | None: + input_names = _coerce_string_tuple(graph.meta.get("input_names")) + match = re.match(r"^v_args_(\d+)$", value_id) + if match is not None: + input_index = int(match.group(1)) + if input_index < len(input_names): + return input_names[input_index] + try: + input_index = list(graph.inputs).index(value_id) + except ValueError: + return None + if input_index < len(input_names): + return input_names[input_index] + return None + + +def _is_materialized_ones_constant(graph: IRGraph, value_id: str) -> bool: + value = graph.constants.get(value_id) + if not isinstance(value, torch.Tensor): + return False + if value.numel() == 0: + return False + return bool(torch.all(value.detach().cpu() == 1).item()) + + +def _has_other_users(graph: IRGraph, value_id: str, *, excluding: str) -> bool: + for other_id, other in graph.nodes.items(): + if other_id == excluding: + continue + if value_id in other.inputs: + return True + return False + + +def _clear_gold_pattern_annotations(graph: IRGraph) -> None: + for node in graph.nodes.values(): + node.meta.pop("gold_patterns", None) + node.meta.pop("gold_pattern_anchor", None) + node.meta.pop("gold_pattern_details", None) + + +def _detect_gated_mlps(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_gated_mlp(graph, node) + if match is None: + continue + + pattern_name = "gated_mlp_gelu" if match.activation == "gelu" else "gated_mlp_silu" + patterns.append( + DetectedPattern( + name=pattern_name, + anchor_node_id=node.id, + node_ids=match.node_ids, + value_ids=(match.input_value_id,), + details={ + "activation": match.activation, + "input_value_id": match.input_value_id, + "gate_weight_value_id": match.gate_weight_value_id, + "up_weight_value_id": match.up_weight_value_id, + "down_weight_value_id": match.down_weight_value_id, + }, + ) + ) + return patterns + + +def _detect_decoder_attentions(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: + continue + + match = match_attention(graph, node) + if match is None: + continue + + q_input, k_input, v_input = match.source_input_value_ids + pattern_name = "decoder_attention_gqa" if match.has_gqa_repeat else "decoder_attention" + + node_ids = set(match.node_ids) + details = { + "input_value_ids": (q_input, k_input, v_input), + "q_linear_weight_value_id": match.weight_value_ids[0], + "k_linear_weight_value_id": match.weight_value_ids[1], + "v_linear_weight_value_id": match.weight_value_ids[2], + "has_rope": bool(match.has_rope), + "has_qk_rms_norm": bool(match.has_qk_norm), + "has_gqa_repeat": bool(match.has_gqa_repeat), + "is_causal": bool(match.is_causal), + "scale": float(match.scale), + "window_size_hint": int(match.window_size), + } + + if len(node.inputs) > 3: + details["mask_value_id"] = node.inputs[3] + mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) + if mask_info is not None: + details["mask_pattern"] = "sliding_window_attention_mask" + details["window_size_hint"] = int(mask_info["window_size"]) + node_ids.update(mask_info["node_ids"]) + + patterns.append( + DetectedPattern( + name=pattern_name, + anchor_node_id=node.id, + node_ids=tuple(sorted(node_ids)), + value_ids=(q_input, k_input, v_input), + details=details, + ) + ) + return patterns + + +def _detect_transformer_blocks(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"add", "add_clipped"} or len(node.inputs) != 2: + continue + + rhs_pattern = _find_anchor_pattern(graph, node.inputs[1], {"gated_mlp_gelu", "gated_mlp_silu"}) + lhs_pattern = _find_anchor_pattern(graph, node.inputs[0], {"gated_mlp_gelu", "gated_mlp_silu"}) + mlp_pattern = rhs_pattern or lhs_pattern + if mlp_pattern is None: + continue + + residual_value_id = node.inputs[0] if rhs_pattern is not None else node.inputs[1] + attn_pattern = _find_anchor_pattern( + graph, + residual_value_id, + {"decoder_attention_gqa", "decoder_attention", "gemma4_partial_rope_attention"}, + ) + if attn_pattern is None: + continue + + node_ids = {node.id} + for candidate in graph.nodes.values(): + gold_patterns = candidate.meta.get("gold_patterns", []) + if attn_pattern in gold_patterns or mlp_pattern in gold_patterns: + node_ids.add(candidate.id) + + patterns.append( + DetectedPattern( + name="decoder_block_post_attn_norm" if node.op == "add_clipped" else "decoder_block_simple_residual", + anchor_node_id=node.id, + node_ids=tuple(sorted(node_ids)), + value_ids=(residual_value_id,), + details={ + "residual_value_id": residual_value_id, + "attention_pattern": attn_pattern, + "mlp_pattern": mlp_pattern, + "residual_op": node.op, + }, + ) + ) + return patterns + + +def _looks_like_gemma_residual_add(graph: IRGraph, residual_value_id: str, branch_value_id: str) -> bool: + branch_node = producer(graph, strip_passthrough(graph, branch_value_id)) + if branch_node is None or branch_node.op != "rms_norm": + return False + return strip_passthrough(graph, residual_value_id) != strip_passthrough(graph, branch_node.inputs[0]) + + +def _find_anchor_pattern(graph: IRGraph, value_id: str, names: set[str]) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + + for name in node.meta.get("gold_patterns", []): + if name in names and node.meta.get("gold_pattern_anchor"): + return name + + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _extract_sliding_window_mask(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node_ids: set[str] = set() + top_and = producer(graph, strip_passthrough(graph, value_id)) + if top_and is None or top_and.op not in {"aten.__and__.Tensor", "logical_and"}: + return None + node_ids.add(top_and.id) + + stack = list(top_and.inputs) + saw_diff = False + saw_cumsum = False + window_candidates: list[int] = [] + + while stack: + current = strip_passthrough(graph, stack.pop()) + node = producer(graph, current) + if node is None or node.id in node_ids: + continue + node_ids.add(node.id) + + if node.op in {"aten.diff.default", "diff"}: + saw_diff = True + elif node.op in {"aten.cumsum.default", "cumsum"}: + saw_cumsum = True + elif node.op == "scalar_subtract": + maybe_window = int(node.attrs.get("value", 0)) + if maybe_window > 0: + window_candidates.append(maybe_window) + + stack.extend(node.inputs) + + if not saw_diff or not saw_cumsum: + return None + + return { + "window_size": max((candidate for candidate in window_candidates if candidate > 1), default=0), + "node_ids": tuple(sorted(node_ids)), + } + + +def _graph_layer_types(graph: IRGraph) -> tuple[str, ...]: + layer_types = _coerce_string_tuple(graph.meta.get("layer_types")) + if layer_types: + return layer_types + + providers = graph.meta.get("transpile_metadata_providers") + if isinstance(providers, dict): + for provider_meta in providers.values(): + if not isinstance(provider_meta, dict): + continue + layer_types = _coerce_string_tuple(provider_meta.get("layer_types")) + if layer_types: + return layer_types + return () + + +def _graph_sliding_window(graph: IRGraph) -> int | None: + sliding_window = _coerce_optional_int(graph.meta.get("sliding_window")) + if sliding_window is not None: + return sliding_window + + providers = graph.meta.get("transpile_metadata_providers") + if isinstance(providers, dict): + for provider_meta in providers.values(): + if not isinstance(provider_meta, dict): + continue + sliding_window = _coerce_optional_int(provider_meta.get("sliding_window")) + if sliding_window is not None: + return sliding_window + return None + + +def _coerce_string_tuple(value: object) -> tuple[str, ...]: + if not isinstance(value, (tuple, list)): + return () + result: list[str] = [] + for item in value: + if not isinstance(item, str): + return () + result.append(item) + return tuple(result) + + +def _coerce_optional_int(value: object) -> int | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return int(value) + return None diff --git a/python/cactus/transpile/runtime_compat.py b/python/cactus/transpile/runtime_compat.py new file mode 100644 index 000000000..1b78b6016 --- /dev/null +++ b/python/cactus/transpile/runtime_compat.py @@ -0,0 +1,347 @@ +from __future__ import annotations + +import ctypes +import importlib +import sys +from typing import Any + + +class _MissingFFIFunction: + _cactus_missing_symbol = True + + def __init__(self, name: str): + self.__name__ = name + self.argtypes = None + self.restype = ctypes.c_int + + def __call__(self, *args: Any, **kwargs: Any) -> int: + raise RuntimeError(f"Cactus runtime is missing required symbol: {self.__name__}") + + +_ORIG_CDLL_GETATTR = ctypes.CDLL.__getattr__ + + +def _patched_cdll_getattr(self: ctypes.CDLL, name: str): + try: + return _ORIG_CDLL_GETATTR(self, name) + except AttributeError: + if not name.startswith("cactus_"): + raise + missing = _MissingFFIFunction(name) + setattr(self, name, missing) + return missing + + +def _load_runtime_modules(): + if "cactus.bindings.cactus" in sys.modules and "cactus.bindings.graph" in sys.modules: + return sys.modules["cactus.bindings.cactus"], sys.modules["cactus.bindings.graph"] + + ctypes.CDLL.__getattr__ = _patched_cdll_getattr + try: + cactus_module = importlib.import_module("cactus.bindings.cactus") + graph_module = importlib.import_module("cactus.bindings.graph") + finally: + ctypes.CDLL.__getattr__ = _ORIG_CDLL_GETATTR + return cactus_module, graph_module + + +def _patch_graph_runtime(graph_module, cactus_module) -> None: + Graph = graph_module.Graph + if getattr(Graph, "_transpile_runtime_compat_patched", False): + return + + _lib = cactus_module._lib + _err = cactus_module._err + cactus_node_t = cactus_module.cactus_node_t + + def _has_symbol(name: str) -> bool: + symbol = getattr(_lib, name, None) + return symbol is not None and not getattr(symbol, "_cactus_missing_symbol", False) + + def _ensure_compare_tensor(self, tensor): + tensor = self._ensure_tensor(tensor) + if int(tensor.dtype) == int(Graph.FP16): + return tensor + return self.precision_cast(tensor, Graph.FP16) + + def _ensure_scalar_tensor(self, tensor): + tensor = self._ensure_tensor(tensor) + if int(tensor.dtype) == int(Graph.FP16): + return tensor + return self.precision_cast(tensor, Graph.FP16) + + def _ensure_fp16_activation(self, tensor): + tensor = self._ensure_tensor(tensor) + if int(tensor.dtype) == int(Graph.FP32): + return self.precision_cast(tensor, Graph.FP16) + return tensor + + def _approx_nonzero_mask(self, tensor): + tensor = _ensure_scalar_tensor(self, tensor) + # Gemma4 compare paths in v2 only need stable 0/1-style masks for + # discrete values such as token-type ids and prebuilt boolean masks. + magnitude = self.abs(tensor) + shifted = self.scalar_add(magnitude, -0.5) + sharpened = self.scalar_multiply(shifted, 16.0) + return self.sigmoid(sharpened) + + orig_conv2d = Graph.conv2d + orig_not_equal = Graph.not_equal + orig_scalar_not_equal = Graph.scalar_not_equal + orig_transpose = Graph.transpose + orig_permute = Graph.permute + orig_scalar_add = Graph.scalar_add + orig_scalar_subtract = Graph.scalar_subtract + orig_scalar_multiply = Graph.scalar_multiply + orig_scalar_divide = Graph.scalar_divide + orig_scalar_exp = Graph.scalar_exp + orig_scalar_sqrt = Graph.scalar_sqrt + orig_scalar_cos = Graph.scalar_cos + orig_scalar_sin = Graph.scalar_sin + orig_scalar_log = Graph.scalar_log + orig_clamp = getattr(Graph, "clamp", None) + orig_add = Graph.add + orig_subtract = Graph.subtract + orig_multiply = Graph.multiply + orig_divide = Graph.divide + orig_concat = Graph.concat + orig_cat = Graph.cat + orig_abs = Graph.abs + orig_pow = Graph.pow + + def matmul(self, a, b, pretransposed_rhs=False, backend=Graph.CPU, output_dtype=None): + a = _ensure_fp16_activation(self, a) + b = _ensure_fp16_activation(self, b) + out = cactus_node_t() + rc = _lib.cactus_graph_matmul( + self.h, + cactus_node_t(a.id), + cactus_node_t(b.id), + ctypes.c_bool(bool(pretransposed_rhs)), + ctypes.c_int32(int(backend)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_matmul failed")) + result = self._tensor_from_node(out.value) + if output_dtype is not None and int(output_dtype) != int(result.dtype): + result = self.precision_cast(result, int(output_dtype)) + return result + + def gather(self, tensor, indices, axis=0): + tensor = self._ensure_tensor(tensor) + indices = self._ensure_tensor(indices) + if int(axis) != 0: + raise NotImplementedError( + f"transpiler runtime compatibility only supports gather(axis=0), got axis={axis}" + ) + out = cactus_node_t() + rc = _lib.cactus_graph_gather( + self.h, + cactus_node_t(tensor.id), + cactus_node_t(indices.id), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_gather failed") + return self._tensor_from_node(out.value) + + def conv2d(self, x, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): + def _pair(value: Any) -> tuple[int, int]: + if isinstance(value, (tuple, list)): + if len(value) != 2: + raise ValueError(f"expected pair for conv2d parameter, got {value!r}") + return int(value[0]), int(value[1]) + return int(value), int(value) + + stride_hw = _pair(stride) + padding_hw = _pair(padding) + dilation_hw = _pair(dilation) + groups_int = int(groups) + kernel_hw = tuple(int(dim) for dim in self._ensure_tensor(weight).shape[-2:]) + + if _has_symbol("cactus_graph_conv2d"): + return orig_conv2d( + self, + x, + weight, + bias=bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + ) + + if ( + kernel_hw == (3, 3) + and stride_hw == (2, 2) + and padding_hw == (1, 1) + and dilation_hw == (1, 1) + ): + if groups_int == 1: + return self.conv2d_k3s2p1(x, weight, bias=bias) + if len(weight.shape) >= 1 and groups_int == int(weight.shape[0]): + return self.conv2d_depthwise_k3s2p1(x, weight, bias=bias) + + if ( + kernel_hw == (3, 3) + and stride_hw == (1, 1) + and padding_hw == (1, 1) + and dilation_hw == (1, 1) + and groups_int == 1 + and _has_symbol("cactus_graph_conv2d_k3s1p1") + ): + return self.conv2d_k3s1p1(x, weight, bias=bias) + + if ( + kernel_hw == (1, 1) + and stride_hw == (1, 1) + and padding_hw == (0, 0) + and dilation_hw == (1, 1) + and groups_int == 1 + ): + return self.conv2d_pointwise_1x1(x, weight, bias=bias) + + raise NotImplementedError( + "v2 runtime does not expose generic conv2d for this configuration: " + f"kernel={kernel_hw} stride={stride_hw} padding={padding_hw} " + f"dilation={dilation_hw} groups={groups_int}" + ) + + def scalar_not_equal(self, x, value): + if _has_symbol("cactus_graph_scalar_not_equal"): + return orig_scalar_not_equal(self, x, value) + x = _ensure_compare_tensor(self, x) + delta = self.scalar_add(x, -float(value)) + return _approx_nonzero_mask(self, delta) + + def not_equal(self, a, b): + if _has_symbol("cactus_graph_not_equal"): + return orig_not_equal(self, a, b) + a = _ensure_compare_tensor(self, a) + b = _ensure_compare_tensor(self, b) + delta = self.subtract(a, b) + return _approx_nonzero_mask(self, delta) + + def transpose(self, x, backend=Graph.CPU): + return orig_transpose(self, _ensure_scalar_tensor(self, x), backend=backend) + + def permute(self, x, permutation, backend=Graph.CPU): + return orig_permute(self, _ensure_scalar_tensor(self, x), permutation, backend=backend) + + def scalar_add(self, x, value): + return orig_scalar_add(self, _ensure_scalar_tensor(self, x), value) + + def scalar_subtract(self, x, value): + return orig_scalar_subtract(self, _ensure_scalar_tensor(self, x), value) + + def scalar_multiply(self, x, value): + return orig_scalar_multiply(self, _ensure_scalar_tensor(self, x), value) + + def scalar_divide(self, x, value): + return orig_scalar_divide(self, _ensure_scalar_tensor(self, x), value) + + def scalar_exp(self, x): + return orig_scalar_exp(self, _ensure_scalar_tensor(self, x)) + + def scalar_sqrt(self, x): + return orig_scalar_sqrt(self, _ensure_scalar_tensor(self, x)) + + def scalar_cos(self, x): + return orig_scalar_cos(self, _ensure_scalar_tensor(self, x)) + + def scalar_sin(self, x): + return orig_scalar_sin(self, _ensure_scalar_tensor(self, x)) + + def scalar_log(self, x): + return orig_scalar_log(self, _ensure_scalar_tensor(self, x)) + + def clamp(self, x, lo, hi): + if orig_clamp is None: + raise RuntimeError("Cactus runtime is missing required symbol: cactus_graph_clamp") + return orig_clamp(self, _ensure_scalar_tensor(self, x), lo, hi) + + def add(self, a, b): + return orig_add(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def subtract(self, a, b): + return orig_subtract(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def multiply(self, a, b): + return orig_multiply(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def divide(self, a, b): + return orig_divide(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def concat(self, a, b, axis=0): + return orig_concat(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b), axis=axis) + + def cat(self, tensors, axis=0): + legalized = [_ensure_scalar_tensor(self, tensor) for tensor in tensors] + return orig_cat(self, legalized, axis=axis) + + def abs(self, x): + return orig_abs(self, _ensure_scalar_tensor(self, x)) + + def pow(self, x, exponent): + return orig_pow(self, _ensure_scalar_tensor(self, x), exponent) + + Graph.matmul = matmul + Graph.gather = gather + Graph.conv2d = conv2d + Graph.scalar_not_equal = scalar_not_equal + Graph.not_equal = not_equal + Graph.transpose = transpose + Graph.permute = permute + Graph.scalar_add = scalar_add + Graph.scalar_subtract = scalar_subtract + Graph.scalar_multiply = scalar_multiply + Graph.scalar_divide = scalar_divide + Graph.scalar_exp = scalar_exp + Graph.scalar_sqrt = scalar_sqrt + Graph.scalar_cos = scalar_cos + Graph.scalar_sin = scalar_sin + Graph.scalar_log = scalar_log + Graph.clamp = clamp + Graph.add = add + Graph.subtract = subtract + Graph.multiply = multiply + Graph.divide = divide + Graph.concat = concat + Graph.cat = cat + Graph.abs = abs + Graph.pow = pow + Graph._transpile_runtime_compat_patched = True + + +_cactus_module, _graph_module = _load_runtime_modules() + +if hasattr(_cactus_module, "_lib"): + _lib_obj = _cactus_module._lib + if hasattr(_lib_obj, "cactus_graph_matmul"): + _lib_obj.cactus_graph_matmul.argtypes = [ + ctypes.c_void_p, + _cactus_module.cactus_node_t, + _cactus_module.cactus_node_t, + ctypes.c_bool, + ctypes.c_int32, + ctypes.POINTER(_cactus_module.cactus_node_t), + ] + _lib_obj.cactus_graph_matmul.restype = ctypes.c_int + if hasattr(_lib_obj, "cactus_graph_gather"): + _lib_obj.cactus_graph_gather.argtypes = [ + ctypes.c_void_p, + _cactus_module.cactus_node_t, + _cactus_module.cactus_node_t, + ctypes.POINTER(_cactus_module.cactus_node_t), + ] + _lib_obj.cactus_graph_gather.restype = ctypes.c_int + +_patch_graph_runtime(_graph_module, _cactus_module) + +_lib = _cactus_module._lib +_err = _cactus_module._err +cactus_node_t = _cactus_module.cactus_node_t +cactus_tensor_info_t = _cactus_module.cactus_tensor_info_t +Graph = _graph_module.Graph +Tensor = _graph_module.Tensor diff --git a/python/cactus/transpile/runtime_support.py b/python/cactus/transpile/runtime_support.py new file mode 100644 index 000000000..8b966f77d --- /dev/null +++ b/python/cactus/transpile/runtime_support.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import builtins +import importlib.util +import sys +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +import torch + +from cactus.transpile.model_profiles import GEMMA4_PROFILE +from cactus.transpile.model_profiles import ModelProfile +from cactus.transpile.model_profiles import profile_for_model_type + + +_TORCHVISION_COMPAT_LIBRARIES: list[object] = [] + + +@dataclass +class PreparedInputs: + names: tuple[str, ...] + tensors: tuple[torch.Tensor, ...] + metadata: dict[str, object] + + +def _transformers_supports_model_module(module_name: str) -> bool: + try: + return importlib.util.find_spec(module_name) is not None + except Exception: + return False + + +def _candidate_external_site_packages() -> list[Path]: + candidates: list[Path] = [] + major_minor = f"python{sys.version_info.major}.{sys.version_info.minor}" + pyenv_versions = Path.home() / ".pyenv" / "versions" + if not pyenv_versions.exists(): + return candidates + for version_dir in sorted(pyenv_versions.iterdir(), reverse=True): + site_packages = version_dir / "lib" / major_minor / "site-packages" + if site_packages.exists(): + candidates.append(site_packages) + return candidates + + +def ensure_transformers_supports_profile(profile: ModelProfile | None) -> str | None: + target_module = None if profile is None else profile.transformer_module + if not target_module or _transformers_supports_model_module(target_module): + return None + + for site_packages in _candidate_external_site_packages(): + candidate = site_packages / Path(*target_module.split(".")) + if not candidate.with_suffix(".py").exists() and not candidate.is_dir(): + continue + sys.path.insert(0, str(site_packages)) + for module_name in list(sys.modules): + root_name = module_name.split(".", 1)[0] + if root_name in {"transformers", "huggingface_hub", "tokenizers"}: + del sys.modules[module_name] + if _transformers_supports_model_module(target_module): + return str(site_packages) + try: + sys.path.remove(str(site_packages)) + except ValueError: + pass + return None + + +def ensure_transformers_supports_model_type(model_type: str) -> str | None: + return ensure_transformers_supports_profile(profile_for_model_type(model_type)) + + +def ensure_transformers_supports_gemma4() -> str | None: + return ensure_transformers_supports_profile(GEMMA4_PROFILE) + + +def _patch_torchvision_missing_nms_op() -> str | None: + """Keep mismatched torch/torchvision installs importable for processors.""" + + try: + import torchvision # type: ignore # noqa: F401 + return None + except RuntimeError as exc: + if "operator torchvision::nms does not exist" not in str(exc): + return None + except Exception: + return None + + try: + import torchvision.extension as tv_extension # type: ignore + + if bool(getattr(tv_extension, "_HAS_OPS", False)): + return None + except Exception: + pass + + try: + library = torch.library.Library("torchvision", "DEF") + library.define("nms(Tensor dets, Tensor scores, float iou_threshold) -> Tensor") + _TORCHVISION_COMPAT_LIBRARIES.append(library) + return "defined missing torchvision::nms operator for torchvision import compatibility" + except Exception: + return None + + +def patch_transformers_torchvision_probe() -> str | None: + has_torchvision = importlib.util.find_spec("torchvision") is not None + has_lzma = importlib.util.find_spec("_lzma") is not None + + if not has_torchvision: + return None + + if has_lzma: + nms_patch_note = _patch_torchvision_missing_nms_op() + return nms_patch_note + + base_note: str | None = None + try: + import backports.lzma as backports_lzma # type: ignore + + sys.modules.setdefault("lzma", backports_lzma) + base_note = "using backports.lzma because this Python build is missing _lzma" + nms_patch_note = _patch_torchvision_missing_nms_op() + if nms_patch_note: + return f"{base_note}; {nms_patch_note}" + return base_note + except Exception: + pass + + class _InterpolationModeStub: + NEAREST = "nearest" + BILINEAR = "bilinear" + BICUBIC = "bicubic" + LANCZOS = "lanczos" + + class _TorchvisionFunctionalStub: + InterpolationMode = _InterpolationModeStub + + def __getattr__(self, name: str): + raise RuntimeError( + "torchvision functionality is unavailable because this Python build is missing _lzma; " + f"attempted to access torchvision.transforms.functional.{name}" + ) + + builtins.F = _TorchvisionFunctionalStub() + builtins.tvF = builtins.F + + import transformers.utils as tf_utils # type: ignore + import transformers.utils.import_utils as tf_import_utils # type: ignore + + @lru_cache + def _disabled() -> bool: + return False + + tf_import_utils.is_torchvision_available = _disabled + tf_import_utils.is_torchvision_v2_available = _disabled + tf_utils.is_torchvision_available = _disabled + tf_utils.is_torchvision_v2_available = _disabled + return "disabled torchvision import checks because this Python build is missing _lzma" + + +def patch_torch_flex_attention_compat() -> str | None: + try: + import torch.nn.attention.flex_attention as flex_attention # type: ignore + except Exception: + return None + + if hasattr(flex_attention, "AuxRequest"): + return None + + class _AuxRequest: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + flex_attention.AuxRequest = _AuxRequest # type: ignore[attr-defined] + return "installed torch flex_attention AuxRequest compatibility stub" diff --git a/python/cactus/transpile/tdt_runtime.py b/python/cactus/transpile/tdt_runtime.py new file mode 100644 index 000000000..bdb6f59fb --- /dev/null +++ b/python/cactus/transpile/tdt_runtime.py @@ -0,0 +1,850 @@ +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import replace +import json +from pathlib import Path +import re +from typing import Any + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from cactus.transpile.component_pipeline import ComponentModuleSpec +from cactus.transpile.audio_preprocess import prepare_native_parakeet_audio_features +from cactus.transpile.model_profiles import add_tensor_aliases +from cactus.transpile.model_profiles import PARAKEET_TDT_PROFILE + + +def _cfg_get(config: dict[str, Any], key: str, default: Any = None) -> Any: + value = config.get(key, default) + return default if value is None else value + + +def _load_tensor_state_dict(model_source: str) -> dict[str, torch.Tensor]: + root = Path(model_source) + safetensors_path = root / "model.safetensors" + if safetensors_path.exists(): + from safetensors.torch import load_file + + return _with_parakeet_tdt_aliases(dict(load_file(str(safetensors_path)))) + + bin_path = root / "pytorch_model.bin" + if bin_path.exists(): + loaded = torch.load(bin_path, map_location="cpu") + if isinstance(loaded, dict): + return _with_parakeet_tdt_aliases({ + str(key): value + for key, value in loaded.items() + if isinstance(value, torch.Tensor) + }) + raise RuntimeError(f"unsupported Parakeet TDT checkpoint format in {model_source}") + + +def _add_tdt_derived_aliases(state_dict: dict[str, torch.Tensor]) -> None: + def alias(target: str, source: str) -> None: + if target not in state_dict and source in state_dict: + state_dict[target] = state_dict[source] + + for index in range(4): + alias(f"decoder.prediction.dec_rnn.lstm.{index}.Wx", f"decoder.lstm.weight_ih_l{index}") + alias(f"decoder.prediction.dec_rnn.lstm.{index}.Wh", f"decoder.lstm.weight_hh_l{index}") + bias_key = f"decoder.prediction.dec_rnn.lstm.{index}.bias" + bias_ih = state_dict.get(f"decoder.lstm.bias_ih_l{index}") + bias_hh = state_dict.get(f"decoder.lstm.bias_hh_l{index}") + if bias_key not in state_dict and bias_ih is not None and bias_hh is not None: + state_dict[bias_key] = bias_ih + bias_hh + + for source_index, target_index in ((0, 0), (2, 2), (3, 3), (5, 5), (6, 6)): + alias( + f"encoder.pre_encode.conv.{target_index}.weight", + f"encoder.subsampling.layers.{source_index}.weight", + ) + alias( + f"encoder.pre_encode.conv.{target_index}.bias", + f"encoder.subsampling.layers.{source_index}.bias", + ) + alias("encoder.pre_encode.out.weight", "encoder.subsampling.linear.weight") + alias("encoder.pre_encode.out.bias", "encoder.subsampling.linear.bias") + + + +def _with_parakeet_tdt_aliases(state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + """Apply profile-declared aliases needed by the local TDT fallback runtime.""" + + add_tensor_aliases( + state_dict, + PARAKEET_TDT_PROFILE, + derived_aliases=_add_tdt_derived_aliases, + ) + return state_dict + + +@dataclass +class ParakeetTDTConfig: + model_source: str + sample_rate: int + num_mel_bins: int + hidden_dim: int + num_layers: int + attention_heads: int + attention_head_dim: int + attention_scale: float + ff_intermediate_dim: int + conv_kernel_size: int + subsampling_factor: int + subsampling_conv_channels: int + predictor_hidden_dim: int + predictor_num_layers: int + joint_dim: int + num_tdt_durations: int + tdt_durations: tuple[int, ...] + blank_id: int + vocabulary: tuple[str, ...] + encoder_hidden_act: str + + +def prepare_parakeet_tdt_audio_features( + audio_file: str | Path, + *, + expected_frames: int | None, + expected_mels: int, + torch_dtype: torch.dtype, +) -> tuple[torch.Tensor, int]: + return prepare_native_parakeet_audio_features( + audio_file, + expected_frames=expected_frames, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + + +def greedy_decode_parakeet_tdt_token_ids( + *, + config: ParakeetTDTConfig, + encoder_hidden_states: np.ndarray, + initial_states: tuple[np.ndarray, ...], + step, +) -> list[int]: + hidden = np.ascontiguousarray(np.asarray(encoder_hidden_states)) + if hidden.ndim != 3 or hidden.shape[0] != 1: + raise ValueError( + f"expected encoder_hidden_states with shape [1, T, D], got {tuple(hidden.shape)}" + ) + predictor_layers = int(config.predictor_num_layers) + if len(initial_states) != predictor_layers * 2: + raise ValueError( + f"expected {predictor_layers * 2} initial state tensors, got {len(initial_states)}" + ) + + states = tuple(np.ascontiguousarray(np.asarray(state)) for state in initial_states) + durations = [int(value) for value in config.tdt_durations] + if not durations and int(config.num_tdt_durations) > 0: + durations = list(range(int(config.num_tdt_durations))) + duration_class_count = max(len(durations), int(config.num_tdt_durations)) + last_token = _initial_parakeet_tdt_blank_id(config) + emitted: list[int] = [] + time_index = 0 + + while time_index < int(hidden.shape[1]): + frame = np.ascontiguousarray(hidden[:, time_index, :]) + advanced = False + symbols_added = 0 + + while symbols_added < 10: + logits, next_states = step(frame, last_token, states) + logits_array = np.asarray(logits, dtype=np.float32) + total_classes = int(logits_array.shape[-1]) + active_duration_count = duration_class_count + if active_duration_count <= 0 or active_duration_count >= total_classes: + active_duration_count = max(1, min(total_classes - 1, int(config.num_tdt_durations) or len(durations) or 1)) + if len(durations) != active_duration_count: + durations = list(range(active_duration_count)) + token_class_count = total_classes - active_duration_count + blank_id = _effective_parakeet_tdt_blank_id( + config, + token_class_count=token_class_count, + ) + if last_token < 0 or last_token >= token_class_count: + last_token = blank_id + token_scores = logits_array[:, :token_class_count] + duration_scores = logits_array[:, token_class_count:] + next_token = int(np.argmax(token_scores[0])) + duration_index = int(np.argmax(duration_scores[0])) + skip = int(durations[min(duration_index, len(durations) - 1)]) if durations else 1 + + if next_token != blank_id: + emitted.append(next_token) + last_token = next_token + states = tuple(np.ascontiguousarray(np.asarray(state).copy()) for state in next_states) + + symbols_added += 1 + + if skip > 0: + time_index += skip + advanced = True + break + if next_token == blank_id: + time_index += 1 + advanced = True + break + + if not advanced: + time_index += 1 + + return emitted + + +def _initial_parakeet_tdt_blank_id(config: ParakeetTDTConfig) -> int: + blank_id = int(config.blank_id) + vocab_size = len(config.vocabulary) + if vocab_size > 0 and blank_id == vocab_size - 1: + return vocab_size + return blank_id + + +def _effective_parakeet_tdt_blank_id( + config: ParakeetTDTConfig, + *, + token_class_count: int, +) -> int: + blank_id = int(config.blank_id) + vocab_size = len(config.vocabulary) + if vocab_size > 0 and blank_id == vocab_size - 1 and token_class_count == vocab_size + 1: + return token_class_count - 1 + if 0 <= blank_id < token_class_count: + return blank_id + if vocab_size > 0 and token_class_count == vocab_size + 1: + return token_class_count - 1 + return max(0, token_class_count - 1) + + +def load_parakeet_tdt_config(model_source: str) -> ParakeetTDTConfig: + root_path = Path(model_source) + config_path = root_path / "config.json" + if not config_path.exists(): + raise FileNotFoundError(f"missing config.json for Parakeet TDT: {config_path}") + root = json.loads(config_path.read_text()) + encoder = root.get("encoder") or root.get("encoder_config") or {} + decoder = root.get("decoder") or {} + prediction = decoder.get("prediction") or decoder.get("prednet") or {} + joint = root.get("joint") or {} + jointnet = joint.get("jointnet") or {} + model_defaults = root.get("model_defaults") or {} + preprocessor = root.get("preprocessor") or {} + + hidden_dim = int(_cfg_get(root, "hidden_dim", _cfg_get(encoder, "d_model", _cfg_get(encoder, "hidden_size", 0)))) + attention_heads = int(_cfg_get(encoder, "n_heads", _cfg_get(encoder, "num_attention_heads", 0))) + attention_head_dim = hidden_dim // max(attention_heads, 1) + tdt_durations = tuple( + int(value) + for value in _cfg_get(root, "tdt_durations", _cfg_get(root, "durations", _cfg_get(model_defaults, "tdt_durations", (0, 1, 2, 3, 4)))) + ) + vocabulary = tuple(str(value) for value in _cfg_get(joint, "vocabulary", ())) + if not vocabulary: + vocabulary = _load_parakeet_vocabulary(root_path) + decoder_vocab_size = int(_cfg_get(root, "vocab_size", _cfg_get(decoder, "vocab_size", len(vocabulary)))) + decoding = root.get("decoding") or {} + blank_id_raw = _cfg_get( + root, + "tdt_blank_id", + _cfg_get(root, "blank_token_id", _cfg_get(decoding, "blank_id", _cfg_get(decoder, "blank_id", None))), + ) + if blank_id_raw is None: + blank_id = decoder_vocab_size + else: + try: + blank_id = int(blank_id_raw) + except (TypeError, ValueError): + blank_id = decoder_vocab_size + if blank_id < 0: + blank_id = decoder_vocab_size + + return ParakeetTDTConfig( + model_source=model_source, + sample_rate=int(_cfg_get(preprocessor, "sample_rate", 16000)), + num_mel_bins=int(_cfg_get(root, "num_mel_bins", _cfg_get(preprocessor, "features", _cfg_get(encoder, "feat_in", _cfg_get(encoder, "num_mel_bins", 128))))), + hidden_dim=hidden_dim, + num_layers=int(_cfg_get(root, "num_layers", _cfg_get(encoder, "n_layers", _cfg_get(encoder, "num_hidden_layers", 0)))), + attention_heads=attention_heads, + attention_head_dim=attention_head_dim, + attention_scale=float(attention_head_dim ** -0.5 if attention_head_dim > 0 else 1.0), + ff_intermediate_dim=int( + _cfg_get( + root, + "ffn_intermediate_dim", + _cfg_get( + encoder, + "ffn_hidden_size", + _cfg_get(encoder, "intermediate_size", round(hidden_dim * float(_cfg_get(encoder, "ff_expansion_factor", 4.0)))), + ), + ) + ), + conv_kernel_size=int(_cfg_get(root, "conv_kernel_size", _cfg_get(encoder, "conv_kernel_size", 9))), + subsampling_factor=int(_cfg_get(root, "subsampling_factor", _cfg_get(encoder, "subsampling_factor", 8))), + subsampling_conv_channels=int( + _cfg_get(root, "subsampling_conv_channels", _cfg_get(encoder, "subsampling_conv_channels", 256)) + ), + predictor_hidden_dim=int( + _cfg_get(root, "predictor_hidden_dim", _cfg_get(root, "decoder_hidden_size", _cfg_get(prediction, "pred_hidden", _cfg_get(model_defaults, "pred_hidden", 640)))) + ), + predictor_num_layers=int( + _cfg_get(root, "predictor_num_layers", _cfg_get(root, "num_decoder_layers", _cfg_get(prediction, "pred_rnn_layers", 1))) + ), + joint_dim=int(_cfg_get(root, "tdt_joint_dim", _cfg_get(jointnet, "joint_hidden", _cfg_get(model_defaults, "joint_hidden", 640)))), + num_tdt_durations=int(_cfg_get(root, "tdt_num_durations", _cfg_get(model_defaults, "num_tdt_durations", len(tdt_durations)))), + tdt_durations=tdt_durations, + blank_id=blank_id, + vocabulary=vocabulary, + encoder_hidden_act=str(_cfg_get(root, "encoder_hidden_act", _cfg_get(encoder, "activation", _cfg_get(encoder, "hidden_act", "silu")))).lower(), + ) + + +def _load_parakeet_vocabulary(root: Path) -> tuple[str, ...]: + vocab_txt = root / "vocab.txt" + if vocab_txt.exists(): + pieces: list[str] = [] + for line in vocab_txt.read_text(encoding="utf-8").splitlines(): + if "\t" in line: + _, token = line.split("\t", 1) + pieces.append(token) + elif line: + pieces.append(line) + if pieces: + return tuple(pieces) + + tokenizer_json = root / "tokenizer.json" + if tokenizer_json.exists(): + try: + loaded = json.loads(tokenizer_json.read_text(encoding="utf-8")) + except Exception: + return () + model = loaded.get("model") if isinstance(loaded, dict) else None + vocab = model.get("vocab") if isinstance(model, dict) else None + if isinstance(vocab, dict): + ordered = sorted( + ((int(index), str(token)) for token, index in vocab.items()), + key=lambda item: item[0], + ) + return tuple(token for _, token in ordered) + return () + + +def _copy_linear_weight(linear: nn.Linear, weight: torch.Tensor, *, bias: torch.Tensor | None = None) -> None: + linear.weight.data.copy_(weight.to(dtype=linear.weight.dtype, device=linear.weight.device)) + if linear.bias is not None: + if bias is None: + linear.bias.data.zero_() + else: + linear.bias.data.copy_(bias.to(dtype=linear.bias.dtype, device=linear.bias.device)) + + +def _copy_conv2d_weight(conv: nn.Conv2d, weight: torch.Tensor, *, bias: torch.Tensor | None = None) -> None: + tensor = weight + if tuple(tensor.shape) != tuple(conv.weight.shape): + tensor = tensor.permute(0, 3, 1, 2).contiguous() + conv.weight.data.copy_(tensor.to(dtype=conv.weight.dtype, device=conv.weight.device)) + if conv.bias is not None: + if bias is None: + conv.bias.data.zero_() + else: + conv.bias.data.copy_(bias.to(dtype=conv.bias.dtype, device=conv.bias.device)) + + +def _copy_conv1d_weight(conv: nn.Conv1d, weight: torch.Tensor, *, bias: torch.Tensor | None = None) -> None: + tensor = weight + if tensor.ndim == 3 and tensor.shape[1] != conv.in_channels // conv.groups: + tensor = tensor.permute(0, 2, 1).contiguous() + conv.weight.data.copy_(tensor.to(dtype=conv.weight.dtype, device=conv.weight.device)) + if conv.bias is not None: + if bias is None: + conv.bias.data.zero_() + else: + conv.bias.data.copy_(bias.to(dtype=conv.bias.dtype, device=conv.bias.device)) + + +def _apply_activation(x: torch.Tensor, activation: str) -> torch.Tensor: + if "gelu" in activation: + return F.gelu(x) + if activation == "relu": + return F.relu(x) + return F.silu(x) + + +def _relative_position_embeddings(*, seq_len: int, hidden_dim: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + rel_positions = torch.arange(seq_len - 1, -seq_len, -1, device=device, dtype=torch.float32) + half_dim = hidden_dim // 2 + inv_freq = 1.0 / ( + 10000.0 + ** (torch.arange(0, half_dim, device=device, dtype=torch.float32) * 2.0 / float(max(hidden_dim, 1))) + ) + angles = rel_positions.unsqueeze(1) * inv_freq.unsqueeze(0) + sin = torch.sin(angles) + cos = torch.cos(angles) + embeddings = torch.stack((sin, cos), dim=-1).reshape(2 * seq_len - 1, half_dim * 2) + if embeddings.shape[1] < hidden_dim: + pad = torch.zeros( + (embeddings.shape[0], hidden_dim - embeddings.shape[1]), + device=device, + dtype=embeddings.dtype, + ) + embeddings = torch.cat((embeddings, pad), dim=1) + elif embeddings.shape[1] > hidden_dim: + embeddings = embeddings[:, :hidden_dim] + return embeddings.to(dtype=dtype) + + +def _relative_position_bias(query: torch.Tensor, relative_key: torch.Tensor, *, scale: float) -> torch.Tensor: + batch, heads, seq_len, _ = query.shape + scores = torch.matmul(query, relative_key.transpose(-1, -2)) + rel_index = ( + torch.arange(seq_len, device=query.device).view(seq_len, 1) + - torch.arange(seq_len, device=query.device).view(1, seq_len) + + (seq_len - 1) + ) + rel_index = rel_index.view(1, 1, seq_len, seq_len).expand(batch, heads, seq_len, seq_len) + gathered = scores.gather(-1, rel_index) + return gathered * float(scale) + + +class ParakeetTDTFeedForward(nn.Module): + def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.linear1 = nn.Linear(config.hidden_dim, config.ff_intermediate_dim, bias=False) + self.linear2 = nn.Linear(config.ff_intermediate_dim, config.hidden_dim, bias=False) + _copy_linear_weight(self.linear1, state_dict[f"{prefix}.linear1.weight"]) + _copy_linear_weight(self.linear2, state_dict[f"{prefix}.linear2.weight"]) + + def forward(self, x: torch.Tensor, *, activation: str) -> torch.Tensor: + return self.linear2(_apply_activation(self.linear1(x), activation)) + + +class ParakeetTDTSelfAttention(nn.Module): + def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.linear_q = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_k = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_v = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_out = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_pos = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + _copy_linear_weight(self.linear_q, state_dict[f"{prefix}.linear_q.weight"]) + _copy_linear_weight(self.linear_k, state_dict[f"{prefix}.linear_k.weight"]) + _copy_linear_weight(self.linear_v, state_dict[f"{prefix}.linear_v.weight"]) + _copy_linear_weight(self.linear_out, state_dict[f"{prefix}.linear_out.weight"]) + _copy_linear_weight(self.linear_pos, state_dict[f"{prefix}.linear_pos.weight"]) + self.pos_bias_u = nn.Parameter(state_dict[f"{prefix}.pos_bias_u"].clone()) + self.pos_bias_v = nn.Parameter(state_dict[f"{prefix}.pos_bias_v"].clone()) + self.config = config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + batch, seq_len, _ = x.shape + num_heads = self.config.attention_heads + head_dim = self.config.attention_head_dim + + q = self.linear_q(x).view(batch, seq_len, num_heads, head_dim) + k = self.linear_k(x).view(batch, seq_len, num_heads, head_dim) + v = self.linear_v(x).view(batch, seq_len, num_heads, head_dim) + + rel_pos = _relative_position_embeddings( + seq_len=seq_len, + hidden_dim=self.config.hidden_dim, + device=x.device, + dtype=x.dtype, + ) + rel_k = self.linear_pos(rel_pos).view(1, 2 * seq_len - 1, num_heads, head_dim) + + q_u = q + self.pos_bias_u.view(1, 1, num_heads, head_dim).to(dtype=x.dtype, device=x.device) + q_v = q + self.pos_bias_v.view(1, 1, num_heads, head_dim).to(dtype=x.dtype, device=x.device) + + q_u_heads = q_u.permute(0, 2, 1, 3) + q_v_heads = q_v.permute(0, 2, 1, 3) + k_heads = k.permute(0, 2, 1, 3) + v_heads = v.permute(0, 2, 1, 3) + rel_k_heads = rel_k.permute(0, 2, 1, 3) + + rel_bias = _relative_position_bias( + q_v_heads, + rel_k_heads, + scale=self.config.attention_scale, + ) + attn = F.scaled_dot_product_attention( + q_u_heads, + k_heads, + v_heads, + attn_mask=rel_bias, + dropout_p=0.0, + is_causal=False, + ) + attn = attn.permute(0, 2, 1, 3).reshape(batch, seq_len, self.config.hidden_dim) + return self.linear_out(attn) + + +class ParakeetTDTConformerConv(nn.Module): + def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + hidden_dim = config.hidden_dim + kernel = config.conv_kernel_size + self.pointwise_conv1 = nn.Conv1d(hidden_dim, hidden_dim * 2, kernel_size=1, bias=False) + self.depthwise_conv = nn.Conv1d( + hidden_dim, + hidden_dim, + kernel_size=kernel, + padding=kernel // 2, + groups=hidden_dim, + bias=False, + ) + self.batch_norm = nn.BatchNorm1d(hidden_dim, eps=1e-5, affine=True, track_running_stats=True) + self.pointwise_conv2 = nn.Conv1d(hidden_dim, hidden_dim, kernel_size=1, bias=False) + _copy_conv1d_weight(self.pointwise_conv1, state_dict[f"{prefix}.pointwise_conv1.weight"]) + _copy_conv1d_weight(self.depthwise_conv, state_dict[f"{prefix}.depthwise_conv.weight"]) + _copy_conv1d_weight(self.pointwise_conv2, state_dict[f"{prefix}.pointwise_conv2.weight"]) + self.batch_norm.weight.data.copy_(state_dict[f"{prefix}.batch_norm.weight"].to(dtype=self.batch_norm.weight.dtype)) + self.batch_norm.bias.data.copy_(state_dict[f"{prefix}.batch_norm.bias"].to(dtype=self.batch_norm.bias.dtype)) + self.batch_norm.running_mean.data.copy_( + state_dict[f"{prefix}.batch_norm.running_mean"].to(dtype=self.batch_norm.running_mean.dtype) + ) + self.batch_norm.running_var.data.copy_( + state_dict[f"{prefix}.batch_norm.running_var"].to(dtype=self.batch_norm.running_var.dtype) + ) + self.config = config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.transpose(1, 2) + x = self.pointwise_conv1(x) + x = F.glu(x, dim=1) + x = self.depthwise_conv(x) + x = self.batch_norm(x) + x = _apply_activation(x, self.config.encoder_hidden_act) + x = self.pointwise_conv2(x) + return x.transpose(1, 2) + + +class ParakeetTDTEncoderLayer(nn.Module): + def __init__(self, config: ParakeetTDTConfig, layer_index: int, state_dict: dict[str, torch.Tensor]): + super().__init__() + prefix = f"encoder.layers.{layer_index}" + self.feed_forward1 = ParakeetTDTFeedForward(config, f"{prefix}.feed_forward1", state_dict) + self.feed_forward2 = ParakeetTDTFeedForward(config, f"{prefix}.feed_forward2", state_dict) + self.self_attn = ParakeetTDTSelfAttention(config, f"{prefix}.self_attn", state_dict) + self.conv = ParakeetTDTConformerConv(config, f"{prefix}.conv", state_dict) + self.norm_feed_forward1 = nn.LayerNorm(config.hidden_dim) + self.norm_self_att = nn.LayerNorm(config.hidden_dim) + self.norm_conv = nn.LayerNorm(config.hidden_dim) + self.norm_feed_forward2 = nn.LayerNorm(config.hidden_dim) + self.norm_out = nn.LayerNorm(config.hidden_dim) + for name in ("norm_feed_forward1", "norm_self_att", "norm_conv", "norm_feed_forward2", "norm_out"): + module = getattr(self, name) + module.weight.data.copy_(state_dict[f"{prefix}.{name}.weight"].to(dtype=module.weight.dtype)) + module.bias.data.copy_(state_dict[f"{prefix}.{name}.bias"].to(dtype=module.bias.dtype)) + self.config = config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x + 0.5 * self.feed_forward1(self.norm_feed_forward1(x), activation=self.config.encoder_hidden_act) + x = x + self.self_attn(self.norm_self_att(x)) + x = x + self.conv(self.norm_conv(x)) + x = x + 0.5 * self.feed_forward2(self.norm_feed_forward2(x), activation=self.config.encoder_hidden_act) + return self.norm_out(x) + + +class ParakeetTDTPreEncode(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + channels = config.subsampling_conv_channels + self.conv = nn.ModuleList( + [ + nn.Conv2d(1, channels, kernel_size=3, stride=2, padding=1), + nn.Identity(), + nn.Conv2d(channels, channels, kernel_size=3, stride=2, padding=1, groups=channels), + nn.Conv2d(channels, channels, kernel_size=1, stride=1, padding=0), + nn.Identity(), + nn.Conv2d(channels, channels, kernel_size=3, stride=2, padding=1, groups=channels), + nn.Conv2d(channels, channels, kernel_size=1, stride=1, padding=0), + ] + ) + _copy_conv2d_weight(self.conv[0], state_dict["encoder.pre_encode.conv.0.weight"], bias=state_dict["encoder.pre_encode.conv.0.bias"]) + _copy_conv2d_weight(self.conv[2], state_dict["encoder.pre_encode.conv.2.weight"], bias=state_dict["encoder.pre_encode.conv.2.bias"]) + _copy_conv2d_weight(self.conv[3], state_dict["encoder.pre_encode.conv.3.weight"], bias=state_dict["encoder.pre_encode.conv.3.bias"]) + _copy_conv2d_weight(self.conv[5], state_dict["encoder.pre_encode.conv.5.weight"], bias=state_dict["encoder.pre_encode.conv.5.bias"]) + _copy_conv2d_weight(self.conv[6], state_dict["encoder.pre_encode.conv.6.weight"], bias=state_dict["encoder.pre_encode.conv.6.bias"]) + projected_width = config.num_mel_bins + for _ in range(3): + projected_width = (projected_width + 2 - 3) // 2 + 1 + self.out = nn.Linear(channels * projected_width, config.hidden_dim) + _copy_linear_weight( + self.out, + state_dict["encoder.pre_encode.out.weight"], + bias=state_dict["encoder.pre_encode.out.bias"], + ) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + if input_features.ndim != 3: + raise ValueError(f"expected input_features [batch, frames, mels], got {tuple(input_features.shape)}") + x = input_features.unsqueeze(1) + x = F.relu(self.conv[0](x)) + x = F.relu(self.conv[3](self.conv[2](x))) + x = F.relu(self.conv[6](self.conv[5](x))) + x = x.permute(0, 2, 1, 3).reshape(x.shape[0], x.shape[2], -1) + return self.out(x) + + +class ParakeetTDTEncoder(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.pre_encode = ParakeetTDTPreEncode(config, state_dict) + self.layers = nn.ModuleList( + [ParakeetTDTEncoderLayer(config, index, state_dict) for index in range(config.num_layers)] + ) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + x = self.pre_encode(input_features) + for layer in self.layers: + x = layer(x) + return x + + +class ParakeetTDTDecoderCell(nn.Module): + def __init__(self, hidden_dim: int, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.Wx = nn.Parameter(state_dict[f"{prefix}.Wx"].clone()) + self.Wh = nn.Parameter(state_dict[f"{prefix}.Wh"].clone()) + self.bias = nn.Parameter(state_dict[f"{prefix}.bias"].clone()) + self.hidden_dim = hidden_dim + + def forward(self, x: torch.Tensor, h: torch.Tensor, c: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + gates = F.linear(x, self.Wx) + F.linear(h, self.Wh) + self.bias + i, f, g, o = torch.chunk(gates, 4, dim=-1) + i = torch.sigmoid(i) + f = torch.sigmoid(f) + g = torch.tanh(g) + o = torch.sigmoid(o) + c_next = f * c + i * g + h_next = o * torch.tanh(c_next) + return h_next, c_next + + +class ParakeetTDTDecoderPrediction(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + vocab_size = max(config.blank_id + 1, state_dict["decoder.prediction.embed.weight"].shape[0]) + self.embed = nn.Embedding(vocab_size, config.predictor_hidden_dim) + self.embed.weight.data.copy_(state_dict["decoder.prediction.embed.weight"].to(dtype=self.embed.weight.dtype)) + self.dec_rnn = nn.Module() + self.dec_rnn.lstm = nn.ModuleList( + [ + ParakeetTDTDecoderCell( + config.predictor_hidden_dim, + f"decoder.prediction.dec_rnn.lstm.{index}", + state_dict, + ) + for index in range(config.predictor_num_layers) + ] + ) + + def forward( + self, + token_ids: torch.Tensor, + state_h: tuple[torch.Tensor, ...], + state_c: tuple[torch.Tensor, ...], + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...], tuple[torch.Tensor, ...]]: + x = self.embed(token_ids) + if x.ndim == 3 and x.shape[1] == 1: + x = x[:, 0, :] + next_h: list[torch.Tensor] = [] + next_c: list[torch.Tensor] = [] + for index, cell in enumerate(self.dec_rnn.lstm): + h_new, c_new = cell(x, state_h[index], state_c[index]) + next_h.append(h_new) + next_c.append(c_new) + x = h_new + return x, tuple(next_h), tuple(next_c) + + +class ParakeetTDTJoint(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.enc = nn.Linear(config.hidden_dim, config.joint_dim) + self.pred = nn.Linear(config.predictor_hidden_dim, config.joint_dim) + output_classes = int(state_dict["joint.joint_net.2.weight"].shape[0]) + self.joint_net = nn.Sequential( + nn.Identity(), + nn.ReLU(), + nn.Linear( + config.joint_dim, + output_classes, + ), + ) + _copy_linear_weight(self.enc, state_dict["joint.enc.weight"], bias=state_dict["joint.enc.bias"]) + _copy_linear_weight(self.pred, state_dict["joint.pred.weight"], bias=state_dict["joint.pred.bias"]) + _copy_linear_weight( + self.joint_net[2], + state_dict["joint.joint_net.2.weight"], + bias=state_dict["joint.joint_net.2.bias"], + ) + + def forward(self, encoder_frame: torch.Tensor, predictor_hidden: torch.Tensor) -> torch.Tensor: + return self.joint_net(F.relu(self.enc(encoder_frame) + self.pred(predictor_hidden))) + + +class ParakeetTDTDecoderStep(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.prediction = ParakeetTDTDecoderPrediction(config, state_dict) + self.joint = ParakeetTDTJoint(config, state_dict) + self.config = config + + def forward(self, encoder_frame: torch.Tensor, token_ids: torch.Tensor, *state_tensors: torch.Tensor) -> tuple[torch.Tensor, ...]: + if len(state_tensors) != self.config.predictor_num_layers * 2: + raise ValueError( + f"expected {self.config.predictor_num_layers * 2} predictor state tensors, got {len(state_tensors)}" + ) + state_h = tuple(state_tensors[0::2]) + state_c = tuple(state_tensors[1::2]) + predictor_hidden, next_h, next_c = self.prediction(token_ids, state_h, state_c) + logits = self.joint(encoder_frame, predictor_hidden) + outputs: list[torch.Tensor] = [logits] + for h, c in zip(next_h, next_c, strict=True): + outputs.append(h) + outputs.append(c) + return tuple(outputs) + + +class ParakeetTDTLocalModel(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.name_or_path = config.model_source + self.family = "parakeet_tdt" + self.config = config + self.encoder = ParakeetTDTEncoder(config, state_dict) + self.decoder = nn.Module() + self.decoder.prediction = ParakeetTDTDecoderPrediction(config, state_dict) + self.joint = ParakeetTDTJoint(config, state_dict) + self.decoder_step = ParakeetTDTDecoderStep(config, state_dict) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + return self.encoder(input_features) + + def initial_decoder_state(self, *, batch_size: int, device: torch.device, dtype: torch.dtype) -> tuple[torch.Tensor, ...]: + state: list[torch.Tensor] = [] + for _ in range(self.config.predictor_num_layers): + state.append(torch.zeros((batch_size, self.config.predictor_hidden_dim), device=device, dtype=dtype)) + state.append(torch.zeros((batch_size, self.config.predictor_hidden_dim), device=device, dtype=dtype)) + return tuple(state) + + def greedy_decode_token_ids(self, input_features: torch.Tensor) -> list[int]: + with torch.no_grad(): + encoder_hidden = self.encoder(input_features) + batch = int(encoder_hidden.shape[0]) + if batch != 1: + raise ValueError("Parakeet TDT local greedy decode currently expects batch size 1") + states = self.initial_decoder_state( + batch_size=batch, + device=encoder_hidden.device, + dtype=encoder_hidden.dtype, + ) + state_arrays = tuple(state.detach().cpu().numpy() for state in states) + encoder_hidden_np = encoder_hidden.detach().cpu().numpy() + + def _step( + frame: np.ndarray, + token_id: int, + state_values: tuple[np.ndarray, ...], + ) -> tuple[np.ndarray, tuple[np.ndarray, ...]]: + frame_tensor = torch.from_numpy(frame).to(device=encoder_hidden.device, dtype=encoder_hidden.dtype) + token_tensor = torch.tensor([[token_id]], device=encoder_hidden.device, dtype=torch.long) + state_tensors = tuple( + torch.from_numpy(value).to(device=encoder_hidden.device, dtype=encoder_hidden.dtype) + for value in state_values + ) + outputs = self.decoder_step(frame_tensor, token_tensor, *state_tensors) + logits = outputs[0].detach().cpu().numpy() + next_states = tuple(output.detach().cpu().numpy() for output in outputs[1:]) + return logits, next_states + + return greedy_decode_parakeet_tdt_token_ids( + config=self.config, + encoder_hidden_states=encoder_hidden_np, + initial_states=state_arrays, + step=_step, + ) + + def decode_token_ids(self, token_ids: list[int]) -> str: + pieces: list[str] = [] + for token_id in token_ids: + if token_id < 0 or token_id >= len(self.config.vocabulary): + continue + piece = self.config.vocabulary[token_id] + if piece.startswith("<|") and piece.endswith("|>"): + continue + pieces.append(piece) + text = "".join(pieces).replace("▁", " ") + return re.sub(r"\s+", " ", text).strip() + + +def load_tdt_local_model(model_source: str, *, torch_dtype: torch.dtype) -> ParakeetTDTLocalModel: + config = load_parakeet_tdt_config(model_source) + state_dict = _load_tensor_state_dict(model_source) + predictor_vocab_size = int(state_dict["decoder.prediction.embed.weight"].shape[0]) + if config.blank_id < 0 or config.blank_id >= predictor_vocab_size: + config = replace(config, blank_id=predictor_vocab_size - 1) + elif config.blank_id == len(config.vocabulary) - 1 and predictor_vocab_size == len(config.vocabulary) + 1: + config = replace(config, blank_id=predictor_vocab_size - 1) + model = ParakeetTDTLocalModel(config, state_dict).eval() + model.to(dtype=torch_dtype) + return model + + +def build_parakeet_tdt_component_specs( + model: ParakeetTDTLocalModel, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, +) -> list[ComponentModuleSpec]: + input_features = named_tensors["input_features"] + example_hidden = model.encoder(input_features) + batch_size = int(example_hidden.shape[0]) + initial_states = model.initial_decoder_state( + batch_size=batch_size, + device=example_hidden.device, + dtype=example_hidden.dtype, + ) + example_frame = example_hidden[:, :1, :].reshape(batch_size, example_hidden.shape[-1]) + example_token_id = torch.full( + (batch_size,), + int(model.config.blank_id), + device=example_hidden.device, + dtype=torch.long, + ) + + state_input_keys: list[str] = [] + state_output_keys: list[str] = [] + for index in range(model.config.predictor_num_layers): + state_input_keys.extend((f"state_h_{index}", f"state_c_{index}")) + state_output_keys.extend((f"state_h_{index}", f"state_c_{index}")) + + common_graph_meta = { + "weights_dir": weights_dir, + "task": "tdt_transcription", + "adapter_family": "parakeet_tdt", + } + return [ + ComponentModuleSpec( + component="audio_encoder", + module=model.encoder, + example_inputs=(input_features,), + input_keys=("input_features",), + output_keys=("encoder_hidden_states",), + graph_meta={**common_graph_meta, "component": "audio_encoder"}, + metadata={"family": "parakeet_tdt", "task": "tdt_transcription"}, + ), + ComponentModuleSpec( + component="decoder", + module=model.decoder_step, + example_inputs=(example_frame, example_token_id, *initial_states), + input_keys=("encoder_frame", "token_ids", *state_input_keys), + output_keys=("step_logits", *state_output_keys), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata={"family": "parakeet_tdt", "task": "tdt_transcription"}, + ), + ] diff --git a/python/cactus/transpile/weight_binding.py b/python/cactus/transpile/weight_binding.py new file mode 100644 index 000000000..a0f94a5ad --- /dev/null +++ b/python/cactus/transpile/weight_binding.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +from dataclasses import dataclass +import json +import os +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class WeightBinding: + path: str + kind: str # "weight" | "embedding" + source_name: str + + +_PROJECT_ROOT = Path(__file__).resolve().parents[3] +_TOKEN_EMBEDDING_OUTPUT_NAMES = { + "token_embeddings.weights", + "token_embeddings.cq2.weights", + "token_embeddings.cq3.weights", + "token_embeddings.cq4.weights", + "decoder_token_embeddings.weights", + "decoder_token_embeddings.cq2.weights", + "decoder_token_embeddings.cq3.weights", + "decoder_token_embeddings.cq4.weights", +} + + +def _candidate_model_dir_names(model_name_or_path: str) -> list[str]: + candidates: list[str] = [] + + def _add(name: str) -> None: + name = name.strip().lower() + if name and name not in candidates: + candidates.append(name) + + raw = model_name_or_path.strip() + if not raw: + return candidates + + path = Path(raw) + _add(path.name) + if "/" in raw: + _add(raw.split("/")[-1]) + + slug = raw.replace("/", "--") + _add(slug) + _add(slug.replace("--", "-")) + + for part in path.parts: + if not part.startswith("models--"): + continue + cache_name = part[len("models--") :] + _add(cache_name) + _add(cache_name.replace("--", "-")) + _add(cache_name.split("--")[-1]) + break + + return candidates + + +def _default_weights_dir_for_model_name(model_name_or_path: str) -> str | None: + if not model_name_or_path: + return None + for model_dir_name in _candidate_model_dir_names(model_name_or_path): + candidate = _PROJECT_ROOT / "weights" / model_dir_name + if candidate.exists(): + return str(candidate) + return None + + +def resolve_transpile_weights_dir(graph_meta: dict[str, object]) -> str | None: + explicit = graph_meta.get("weights_dir") + if isinstance(explicit, str) and explicit: + return explicit + + family = str(graph_meta.get("adapter_family", "")).upper() + family_env = f"CACTUS_TRANSPILER_WEIGHTS_DIR_{family}" + if family and family_env in os.environ and os.environ[family_env]: + return os.environ[family_env] + + generic = os.environ.get("CACTUS_TRANSPILER_WEIGHTS_DIR") + if generic: + return generic + + model_name_or_path = graph_meta.get("model_name_or_path") + if isinstance(model_name_or_path, str) and model_name_or_path: + return _default_weights_dir_for_model_name(model_name_or_path) + return None + + +def _manifest_source_aliases(name: str) -> tuple[str, ...]: + aliases: list[str] = [] + + def _add(candidate: str) -> None: + if candidate and candidate not in aliases: + aliases.append(candidate) + + def _add_with_wrappers(candidate: str) -> None: + _add(candidate) + for prefix in ( + "module.", + "module.model.", + "adapter.", + "adapter.model.", + ): + _add(f"{prefix}{candidate}") + + raw = name.strip() + if not raw: + return () + + _add_with_wrappers(raw) + + for prefix in ("encoder.", "decoder.", "model.encoder.", "model.decoder."): + if raw.startswith(prefix): + _add_with_wrappers(raw[len(prefix) :]) + + for prefix in ("model.language_model.", "language_model."): + if raw.startswith(prefix): + tail = raw[len(prefix) :] + _add_with_wrappers(tail) + _add_with_wrappers(f"backbone.{tail}") + _add_with_wrappers(f"model.{tail}") + + if raw.startswith("model."): + tail = raw[len("model.") :] + _add_with_wrappers(tail) + + if raw.startswith("backbone."): + tail = raw[len("backbone.") :] + _add_with_wrappers(tail) + _add_with_wrappers(f"model.{tail}") + _add_with_wrappers(f"model.language_model.{tail}") + + return tuple(aliases) + + +def _manifest_entry_kind(output_name: str, explicit_kind: Any, explicit_names: list[object]) -> str: + if isinstance(explicit_kind, str) and explicit_kind: + return explicit_kind + + output_text = output_name.lower() + explicit_text = " ".join(str(name).lower() for name in explicit_names if isinstance(name, str)) + if ( + output_name in _TOKEN_EMBEDDING_OUTPUT_NAMES + or "embedding" in output_text + or "token_embeddings" in output_text + or "embed_tokens.weight" in explicit_text + or "embed_positions.weight" in explicit_text + ): + return "embedding" + return "weight" + + +def _flatten_convert_manifest(root_manifest: dict[str, Any]) -> dict[str, object]: + flattened: dict[str, object] = {} + rows = root_manifest.get("weights") + if not isinstance(rows, list): + for name, entry in root_manifest.items(): + if not isinstance(name, str): + continue + for alias in _manifest_source_aliases(name): + flattened.setdefault(alias, entry) + return flattened + + for row in rows: + if not isinstance(row, dict): + continue + output_name = row.get("output_name") or row.get("filename") + if not isinstance(output_name, str) or not output_name: + continue + + explicit_names: list[object] = [ + row.get("source_name"), + row.get("hf_name"), + row.get("adapter_name"), + ] + source_names = row.get("source_names") + if isinstance(source_names, list): + explicit_names.extend(source_names) + + entry = { + "filename": output_name, + "kind": _manifest_entry_kind(output_name, row.get("kind"), explicit_names), + } + for source_name in explicit_names: + if not isinstance(source_name, str) or not source_name: + continue + for alias in _manifest_source_aliases(source_name): + flattened.setdefault(alias, entry) + return flattened + + +def _load_weights_manifest(root: Path) -> dict[str, object]: + manifest_path = root / "weights_manifest.json" + if not manifest_path.exists(): + return {} + try: + loaded_manifest = json.loads(manifest_path.read_text()) + except Exception: + return {} + if not isinstance(loaded_manifest, dict): + return {} + return _flatten_convert_manifest(loaded_manifest) + + +def _binding_from_manifest_entry(root: Path, source_name: str, entry: object) -> WeightBinding | None: + if not isinstance(entry, dict): + return None + filename = entry.get("filename") + kind = entry.get("kind", "weight") + if not isinstance(filename, str) or not isinstance(kind, str): + return None + candidate = root / filename + if not candidate.exists(): + return None + return WeightBinding(path=str(candidate), kind=kind, source_name=source_name) + + +def resolve_weight_binding(*, weights_dir: str | None, source_name: str) -> WeightBinding | None: + """Resolve an exported parameter to a converted Cactus tensor file. + + The resolver intentionally trusts only `weights_manifest.json`. It may add + wrapper-local aliases such as `module.` or `adapter.model.`, but it does not + guess old model-specific filenames from layer names. + """ + + if not weights_dir: + return None + root = Path(weights_dir) + if not root.exists(): + return None + manifest = _load_weights_manifest(root) + binding = _binding_from_manifest_entry(root, source_name, manifest.get(source_name)) + if binding is not None: + return binding + return None diff --git a/python/cactus/transpile/weight_compat.py b/python/cactus/transpile/weight_compat.py new file mode 100644 index 000000000..0b7fac063 --- /dev/null +++ b/python/cactus/transpile/weight_compat.py @@ -0,0 +1,532 @@ +from __future__ import annotations + +from dataclasses import replace +import shutil +import struct +from pathlib import Path + +import numpy as np + +from cactus.convert.cactus_adapters.tensor_io import CACTUS_ALIGNMENT +from cactus.convert.cactus_adapters.tensor_io import CACTUS_MAGIC +from cactus.convert.cactus_adapters.tensor_io import FLAG_INTERLEAVED +from cactus.convert.cactus_adapters.tensor_io import align_offset +from cactus.convert.model_adapters.naming import gemma4_scale_factor +from cactus.transpile.runtime_compat import Graph +from cactus.transpile.weight_binding import WeightBinding +from cactus.convert.quantization.cq import FLAG_ORTHOGONAL_ROTATION +from cactus.convert.quantization.cq import GROUP_SIZE as CQ_GROUP_SIZE +from cactus.convert.quantization.cq import PRECISION_CQ +from cactus.convert.quantization.cq import make_codebook +from cactus.convert.quantization.cq import make_hadamard_components +from cactus.convert.quantization.cq import make_hadamard_matrix +from cactus.convert.quantization.cq import make_orthogonal_rotation +from cactus.convert.quantization.cq import pack_indices_lsb +from cactus.convert.quantization.cq import quantize_hadamard +from cactus.convert.quantization.cq import quantize_orthogonal +from cactus.convert.quantization.cq import write_cq_tensor + + +_HEADER_SIZE = 84 +_FLAG_EXTENDED_SHAPE = 1 << 4 +_INT8 = int(Graph.INT8) +_INT4 = int(Graph.INT4) + +_TOKEN_EMBEDDING_FILENAMES = { + "token_embeddings.weights", + "decoder_token_embeddings.weights", +} +_PER_LAYER_EMBEDDING_FILENAMES = { + "embed_tokens_per_layer.weights", +} + + +class _OpenedTensor: + def __init__( + self, + *, + path: Path, + precision: int, + shape: tuple[int, ...], + data: np.memmap, + scales: np.memmap | None, + group_size: int, + num_groups: int, + is_interleaved: bool, + original_n: int, + alignment: int, + ) -> None: + self.path = path + self.precision = precision + self.shape = shape + self.data = data + self.scales = scales + self.group_size = group_size + self.num_groups = num_groups + self.is_interleaved = is_interleaved + self.original_n = original_n + self.alignment = alignment + + +def ensure_binding_compatible(binding: WeightBinding, source_tensor: object | None = None) -> WeightBinding: + binding = ensure_embedding_binding_compatible(binding) + return _ensure_legacy_int4_weight_binding_compatible(binding, source_tensor=source_tensor) + + +def ensure_embedding_binding_compatible(binding: WeightBinding) -> WeightBinding: + if binding.kind != "embedding": + return binding + + source_path = Path(binding.path).expanduser().resolve() + opened = _open_cactus_tensor_file(source_path) + if opened.precision != _INT8: + return binding + if len(opened.shape) != 2 or opened.scales is None or opened.group_size <= 0: + return binding + + config = _embedding_cache_config(source_path.name, source_name=binding.source_name) + if config is None: + return binding + + compat_path = source_path.with_name(source_path.stem + f".cq{config['bits']}.weights") + _materialize_cq_embedding_cache( + opened, + compat_path, + bits=int(config["bits"]), + rotation=str(config["rotation"]), + ) + _cleanup_legacy_fp16_cache(source_path) + return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) + + +def _embedding_cache_config(filename: str, *, source_name: str) -> dict[str, object] | None: + normalized_source = str(source_name or "") + if filename in _PER_LAYER_EMBEDDING_FILENAMES or normalized_source.endswith("embed_tokens_per_layer.weight"): + return {"bits": 2, "rotation": "hadamard"} + if filename in _TOKEN_EMBEDDING_FILENAMES: + return {"bits": 4, "rotation": "orthogonal"} + return {"bits": 4, "rotation": "orthogonal"} + + +def _ensure_legacy_int4_weight_binding_compatible( + binding: WeightBinding, + *, + source_tensor: object | None, +) -> WeightBinding: + source_path = Path(binding.path).expanduser().resolve() + opened = _open_cactus_tensor_file(source_path) + if not _is_legacy_packed_int4_tensor(opened): + return binding + + compat_path = source_path.with_name(source_path.stem + ".cq4.weights") + if compat_path.exists(): + if _can_materialize_compat_weight(source_tensor): + _materialize_source_cq_weight( + source_tensor, + compat_path, + bits=4, + rotation="hadamard", + scale_factor=_compat_weight_scale_factor(source_path), + source_mtime_ns=source_path.stat().st_mtime_ns, + ) + return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) + + if not _can_materialize_compat_weight(source_tensor): + raise RuntimeError( + "legacy packed INT4 weight is not directly executable in the v2 runtime and is " + f"missing a CQ companion file: {source_path.name}. " + "Rerun `cactus transpile ...` so the transpiler can materialize the CQ weights, " + "or generate them through `cq_convert` first." + ) + + _materialize_source_cq_weight( + source_tensor, + compat_path, + bits=4, + rotation="hadamard", + scale_factor=_compat_weight_scale_factor(source_path), + source_mtime_ns=source_path.stat().st_mtime_ns, + ) + return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) + + +def _is_legacy_packed_int4_tensor(opened: _OpenedTensor) -> bool: + if opened.precision != _INT4: + return False + if len(opened.shape) != 2 or opened.scales is None or opened.group_size <= 0: + return False + if not opened.is_interleaved: + return False + return True + + +def _can_materialize_compat_weight(source_tensor: object | None) -> bool: + if source_tensor is None: + return False + if isinstance(source_tensor, (str, bytes, bytearray, Path)): + return False + if np.isscalar(source_tensor): + return False + return hasattr(source_tensor, "shape") + + +def _compat_weight_scale_factor(source_path: Path) -> float: + parent_name = source_path.parent.name.lower() + if "gemma-4" in parent_name or "gemma4" in parent_name: + return gemma4_scale_factor(source_path.name) + return 1.0 + +def _cleanup_legacy_fp16_cache(source_path: Path) -> None: + legacy = source_path.with_name(source_path.stem + ".fp16.weights") + if not legacy.exists(): + return + try: + legacy.unlink() + except OSError: + pass + + +def _materialize_source_cq_weight( + tensor: object, + out_path: Path, + *, + bits: int, + rotation: str, + scale_factor: float = 1.0, + source_mtime_ns: int | None = None, +) -> None: + if source_mtime_ns is not None and out_path.exists() and out_path.stat().st_mtime_ns >= source_mtime_ns: + return + + if rotation == "orthogonal": + cq = quantize_orthogonal(tensor, bits=bits) + else: + cq = quantize_hadamard(tensor, bits=bits, use_gptq=False) + if float(scale_factor) != 1.0: + cq = replace(cq, norms=(cq.norms.astype(np.float32) * float(scale_factor)).astype(np.float16)) + write_cq_tensor(out_path, cq) + + +def _open_cactus_tensor_file(path: str | Path) -> _OpenedTensor: + tensor_path = Path(path).expanduser().resolve() + with tensor_path.open("rb") as handle: + header = handle.read(_HEADER_SIZE) + if len(header) < _HEADER_SIZE: + raise RuntimeError(f"tensor file is too small for a Cactus header: {tensor_path}") + if header[:4] != CACTUS_MAGIC: + raise RuntimeError(f"tensor file is missing the CACT header: {tensor_path}") + + flags = struct.unpack_from(" 0) + + dtype_map = { + int(Graph.INT8): np.int8, + int(Graph.FP16): np.float16, + int(Graph.FP32): np.float32, + int(Graph.INT4): np.uint8, + 3: np.uint8, + 4: np.uint8, + 5: np.uint8, + 6: np.uint8, + } + dtype = dtype_map.get(precision) + if dtype is None: + raise RuntimeError(f"unsupported tensor precision {precision} in {tensor_path}") + + aligned_header = align_offset(header_size, alignment) + scales_offset = aligned_header if scales_bytes > 0 else 0 + data_offset = align_offset(scales_offset + scales_bytes, alignment) if scales_bytes > 0 else aligned_header + + data_element_count = byte_size // np.dtype(dtype).itemsize + data = np.memmap(tensor_path, mode="r", dtype=dtype, offset=data_offset, shape=(data_element_count,)) + scales = None + if scales_bytes > 0: + scales = np.memmap( + tensor_path, + mode="r", + dtype=np.float16, + offset=scales_offset, + shape=(scales_bytes // np.dtype(np.float16).itemsize,), + ) + return _OpenedTensor( + path=tensor_path, + precision=precision, + shape=shape, + data=data, + scales=scales, + group_size=group_size, + num_groups=num_groups, + is_interleaved=bool(flags & FLAG_INTERLEAVED), + original_n=original_n, + alignment=alignment, + ) + + +def _materialize_cq_embedding_cache( + opened: _OpenedTensor, + out_path: Path, + *, + bits: int, + rotation: str, +) -> None: + src_mtime_ns = opened.path.stat().st_mtime_ns + if out_path.exists() and out_path.stat().st_mtime_ns >= src_mtime_ns: + return + + if len(opened.shape) != 2: + raise RuntimeError(f"expected rank-2 embedding tensor, got shape={opened.shape}") + + rows = int(opened.original_n or opened.shape[0]) + cols = int(opened.shape[1]) + if rotation == "hadamard" and cols % CQ_GROUP_SIZE != 0: + raise RuntimeError( + f"Hadamard CQ embedding conversion requires hidden dim divisible by {CQ_GROUP_SIZE}, " + f"got {cols} for {opened.path.name}" + ) + + input_scale = _estimate_embedding_input_scale(opened, rows=rows, cols=cols) + precision = int(PRECISION_CQ[int(bits)]) + group_size = cols if rotation == "orthogonal" else CQ_GROUP_SIZE + num_groups = cols // group_size + codebook = make_codebook(group_size, bits).astype(np.float16) + input_scale = np.asarray(input_scale, dtype=np.float16).reshape(cols) + recip = np.minimum(1.0 / np.maximum(input_scale.astype(np.float32), 1e-8), 65504.0).astype(np.float16) + + chunk_rows = _recommended_chunk_rows(cols=cols, rotation=rotation) + norms_tmp = out_path.with_suffix(out_path.suffix + ".norms.tmp") + data_tmp = out_path.with_suffix(out_path.suffix + ".data.tmp") + final_tmp = out_path.with_suffix(out_path.suffix + ".tmp") + + flags = 0 + trailer_parts: list[bytes] = [] + if rotation == "orthogonal": + flags |= FLAG_ORTHOGONAL_ROTATION + trailer_parts.append(make_orthogonal_rotation(group_size, seed=1234).astype(np.float16).tobytes()) + else: + left, right, perm = make_hadamard_components(group_size, seed=1234) + trailer_parts.extend([ + left.tobytes(), + right.tobytes(), + perm.astype(" current: + handle.write(b"\x00" * (data_offset - current)) + with data_tmp.open("rb") as packed_handle: + shutil.copyfileobj(packed_handle, handle, length=8 * 1024 * 1024) + final_tmp.replace(out_path) + finally: + for temp_path in (norms_tmp, data_tmp): + try: + temp_path.unlink() + except FileNotFoundError: + pass + + +def _recommended_chunk_rows(*, cols: int, rotation: str) -> int: + if rotation == "orthogonal": + if cols >= 8192: + return 8 + if cols >= 4096: + return 16 + return 64 + if cols >= 8192: + return 32 + if cols >= 4096: + return 48 + return 96 + + +def _estimate_embedding_input_scale( + opened: _OpenedTensor, + *, + rows: int, + cols: int, + max_sample_rows: int = 2048, + sample_chunk_rows: int = 64, +) -> np.ndarray: + if rows <= 0 or cols <= 0: + return np.ones((cols,), dtype=np.float16) + + stride = max(1, rows // max(1, max_sample_rows)) + sampled = 0 + accum = np.zeros((cols,), dtype=np.float64) + for start in range(0, rows, stride * sample_chunk_rows): + stop = min(start + sample_chunk_rows, rows) + chunk = _dequantize_int8_rows(opened, start, stop).astype(np.float32, copy=False) + accum += np.abs(chunk).sum(axis=0, dtype=np.float64) + sampled += int(stop - start) + if sampled >= max_sample_rows: + break + + if sampled <= 0: + return np.ones((cols,), dtype=np.float16) + + mean_abs = np.maximum(accum / float(sampled), 1e-6) + raw = np.power(mean_abs, -0.5, dtype=np.float64) + raw /= np.exp(np.mean(np.log(np.clip(raw, 1e-6, None)))) + return np.clip(raw, 1.0 / 8.0, 8.0).astype(np.float16) + + +def _quantize_hadamard_chunk( + chunk: np.ndarray, + *, + bits: int, + input_scale: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + work = chunk.astype(np.float32, copy=False) * input_scale.astype(np.float32, copy=False)[None, :] + rows, cols = work.shape + group_size = CQ_GROUP_SIZE + groups = cols // group_size + rotation = make_hadamard_matrix(group_size, seed=1234).astype(np.float32) + codebook = make_codebook(group_size, bits).astype(np.float32) + + indices = np.empty((rows, cols), dtype=np.uint8) + norms = np.empty((rows, groups), dtype=np.float16) + for group_index in range(groups): + lo = group_index * group_size + hi = lo + group_size + group = work[:, lo:hi] + row_norms = np.linalg.norm(group, axis=1).clip(min=1e-8).astype(np.float32, copy=False) + rotated = (group / row_norms[:, None]) @ rotation + idx = np.abs(rotated[..., None] - codebook[None, None, :]).argmin(axis=-1).astype(np.uint8) + indices[:, lo:hi] = idx + norms[:, group_index] = row_norms.astype(np.float16) + return pack_indices_lsb(indices, group_size, bits), norms + + +def _quantize_orthogonal_chunk( + chunk: np.ndarray, + *, + bits: int, + input_scale: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + work = chunk.astype(np.float32, copy=False) * input_scale.astype(np.float32, copy=False)[None, :] + rows, cols = work.shape + rotation = make_orthogonal_rotation(cols, seed=1234).astype(np.float32) + codebook = make_codebook(cols, bits).astype(np.float32) + norms = np.linalg.norm(work, axis=1).clip(min=1e-8).astype(np.float32, copy=False) + rotated = (work / norms[:, None]) @ rotation + indices = np.abs(rotated[..., None] - codebook[None, None, :]).argmin(axis=-1).astype(np.uint8) + return pack_indices_lsb(indices, cols, bits), norms.astype(np.float16).reshape(rows, 1) + + +def _write_cq_header( + handle, + *, + rows: int, + cols: int, + precision: int, + packed_bytes: int, + scales_bytes: int, + group_size: int, + num_groups: int, + flags: int, +) -> None: + handle.write(CACTUS_MAGIC) + handle.write(struct.pack(" np.ndarray: + if opened.scales is None or opened.group_size <= 0: + raise RuntimeError(f"INT8 embedding file is missing grouped scales: {opened.path}") + + rows = stop - start + cols = int(opened.shape[1]) + num_groups = int(opened.num_groups) + row_group = int(opened.group_size) + padded_rows = int(opened.shape[0]) + + if opened.is_interleaved: + block = 4 + padded_cols = ((cols + block - 1) // block) * block + q_blocks = opened.data.reshape(padded_rows // block, padded_cols // block, block, block) + s_blocks = opened.scales.reshape(padded_rows // block, num_groups, block) + block_start = start // block + block_end = (stop + block - 1) // block + q = np.asarray(q_blocks[block_start:block_end], dtype=np.int8).transpose(0, 2, 1, 3).reshape(-1, padded_cols) + s = np.asarray(s_blocks[block_start:block_end], dtype=np.float16).astype(np.float32).transpose(0, 2, 1).reshape(-1, num_groups) + row_offset = start - block_start * block + q = q[row_offset : row_offset + rows, :cols] + s = s[row_offset : row_offset + rows] + else: + q = np.asarray(opened.data, dtype=np.int8).reshape(padded_rows, cols)[start:stop] + s = np.asarray(opened.scales, dtype=np.float16).astype(np.float32).reshape(padded_rows, num_groups)[start:stop] + + out = np.empty((rows, cols), dtype=np.float16) + for group_index in range(num_groups): + lo = group_index * row_group + hi = min(lo + row_group, cols) + if lo >= hi: + break + scales = s[:, group_index : group_index + 1] + out[:, lo:hi] = (q[:, lo:hi].astype(np.float32) * scales).astype(np.float16) + return out diff --git a/python/pyproject.toml b/python/pyproject.toml index 06d4b8c72..5f719a6cf 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -5,9 +5,10 @@ description = "Cactus Python tools" requires-python = ">=3.10" dependencies = [ "torch", - "transformers", + "transformers>=5.5.4,<6", "numpy", - "huggingface-hub", + "huggingface-hub>=1.5.0,<2.0", + "backports.lzma>=0.0.14", ] [project.optional-dependencies] diff --git a/python/requirements.txt b/python/requirements.txt index f675d0773..06a615e47 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,10 +1,11 @@ torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 -transformers==5.0.0 +transformers==5.5.4 pillow==11.3.0 num2words==0.5.12 -huggingface_hub==1.4.1 +huggingface_hub>=1.5.0,<2.0 +backports.lzma==0.0.14 einops==0.8.2 timm==1.0.25 sentencepiece==0.2.1 diff --git a/python/tests/test_capture.py b/python/tests/test_capture.py index 87ea52ec3..528402de0 100644 --- a/python/tests/test_capture.py +++ b/python/tests/test_capture.py @@ -10,9 +10,9 @@ from transformers import AutoModelForCausalLM from transformers import AutoTokenizer -from cactus.transpile.capture.capture_pytorch import capture_model_with_fallback -from cactus.transpile.capture.capture_pytorch import dump_graph -from cactus.transpile.capture.graph_ir import verify_ir +from cactus.transpile.capture_pytorch import capture_model_with_fallback +from cactus.transpile.capture_pytorch import dump_graph +from cactus.transpile.graph_ir import verify_ir MODELS = ( diff --git a/python/tests/test_cli_run_transpiled.py b/python/tests/test_cli_run_transpiled.py new file mode 100644 index 000000000..39d8c2ab4 --- /dev/null +++ b/python/tests/test_cli_run_transpiled.py @@ -0,0 +1,371 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import torch +from scipy.io import wavfile + +from cactus import cli +from cactus.cli import run as run_cli +from cactus.cli import transpile as transpile_cli +from cactus.transpile import audio_preprocess +from cactus.transpile import component_bundle_runtime +from cactus.transpile import multimodal_runtime +from cactus.transpile import hf_model + + +def test_cactus_run_detects_transpiled_bundle_and_uses_main_style_audio(monkeypatch, tmp_path: Path, capsys) -> None: + bundle_dir = tmp_path / "bundle" + components_dir = bundle_dir / "components" + components_dir.mkdir(parents=True) + (components_dir / "manifest.json").write_text( + '{"model_id":"example/model","family":"generic","task":"causal_lm_logits","components":[]}', + encoding="utf-8", + ) + audio_file = tmp_path / "input.wav" + audio_file.write_bytes(b"RIFF") + + calls = [] + + def _fake_run_transpiled(args): + calls.append(args) + print("hello from transpiled") + return 0 + + monkeypatch.setattr(transpile_cli, "cmd_run_transpiled", _fake_run_transpiled) + + parser = cli.create_parser() + args = parser.parse_args( + [ + "run", + str(bundle_dir), + "--audio", + str(audio_file), + "--prompt", + "Hello", + ] + ) + + rc = run_cli.cmd_run(args) + + assert rc == 0 + assert len(calls) == 1 + forwarded = calls[0] + assert forwarded.bundle_dir == str(bundle_dir) + assert forwarded.audio == str(audio_file.resolve()) + assert forwarded.audio_file == str(audio_file.resolve()) + assert forwarded.prompt == "Hello" + assert forwarded._transpiled_from_run is True + captured = capsys.readouterr().out + assert "Starting Cactus Chat with model:" in captured + assert "hello from transpiled" in captured + + +def test_run_transpiled_human_result_prints_response(capsys) -> None: + transpile_cli._print_transpiled_run_result( + { + "response": " generated text ", + "transcript": "not used", + } + ) + + assert capsys.readouterr().out == "generated text\n" + + +def test_run_transpiled_once_deduplicates_image_aliases(tmp_path: Path) -> None: + image_path = tmp_path / "input.png" + image_path.write_bytes(b"not really an image") + calls: list[dict[str, object]] = [] + + def fake_runner(bundle_dir, **kwargs): + calls.append({"bundle_dir": bundle_dir, **kwargs}) + return {"response": "ok"} + + args = type( + "Args", + (), + { + "image": str(image_path), + "image_file": [str(image_path)], + "image_files": [str(image_path)], + "audio": None, + "audio_file": None, + "prompt": "describe", + "input_ids": None, + "weights_dir": None, + "system": None, + "thinking": False, + "max_new_tokens": 1, + "stop_sequence": [], + }, + )() + + result = transpile_cli._run_transpiled_once(args, fake_runner, bundle_dir="bundle") + + assert result == {"response": "ok"} + assert calls[0]["image_files"] == (str(image_path.resolve()),) + + +def test_multimodal_decoder_inputs_right_align_to_static_tail() -> None: + class FakeComponent: + _input_names = ("inputs_embeds", "per_layer_inputs", "position_ids") + + store = { + "inputs_embeds": np.arange(1 * 6 * 2, dtype=np.float16).reshape(1, 6, 2), + "per_layer_inputs": np.arange(1 * 6 * 1 * 2, dtype=np.float16).reshape(1, 6, 1, 2), + "position_ids": np.arange(6, dtype=np.int64).reshape(1, 6), + } + original = {key: value.copy() for key, value in store.items()} + + component_bundle_runtime._right_align_decoder_inputs_to_static_tail( + store, + component=FakeComponent(), # type: ignore[arg-type] + prompt_token_count=4, + ) + + assert np.all(store["inputs_embeds"][:, :2, :] == 0) + np.testing.assert_array_equal(store["inputs_embeds"][:, 2:, :], original["inputs_embeds"][:, :4, :]) + assert np.all(store["per_layer_inputs"][:, :2, :, :] == 0) + np.testing.assert_array_equal(store["per_layer_inputs"][:, 2:, :, :], original["per_layer_inputs"][:, :4, :, :]) + assert np.all(store["position_ids"][:, :2] == 0) + np.testing.assert_array_equal(store["position_ids"][:, 2:], original["position_ids"][:, :4]) + + +def test_static_input_padding_left_trims_overlong_token_inputs() -> None: + store = { + "input_ids": np.arange(6, dtype=np.int64).reshape(1, 6), + "attention_mask": np.ones((1, 6), dtype=np.int64), + } + + component_bundle_runtime._pad_prepared_store_to_static_input_shapes( + store, + inputs_meta={"input_shapes": {"input_ids": [1, 4], "attention_mask": [1, 4]}}, + tokenizer=None, + ) + + np.testing.assert_array_equal(store["input_ids"], np.asarray([[2, 3, 4, 5]], dtype=np.int64)) + np.testing.assert_array_equal(store["attention_mask"], np.ones((1, 4), dtype=np.int64)) + + +def test_static_input_padding_trims_overlong_audio_features() -> None: + store = { + "input_features": np.ones((1, 6, 2), dtype=np.float16), + "input_features_mask": np.ones((1, 6), dtype=bool), + } + + component_bundle_runtime._pad_prepared_store_to_static_input_shapes( + store, + inputs_meta={"input_shapes": {"input_features": [1, 4, 2], "input_features_mask": [1, 4]}}, + tokenizer=None, + ) + + assert store["input_features"].shape == (1, 4, 2) + assert store["input_features_mask"].shape == (1, 4) + + +def test_gemma4_multimodal_headroom_uses_context_floor() -> None: + prepared = hf_model.PreparedInputs( + names=("input_ids", "attention_mask", "token_type_ids"), + tensors=( + torch.tensor([[1, 2, 3]], dtype=torch.long), + torch.ones((1, 3), dtype=torch.long), + torch.zeros((1, 3), dtype=torch.long), + ), + metadata={"input_shapes": {"input_ids": [1, 3]}}, + ) + + padded = hf_model._add_multimodal_generation_headroom( + prepared, + tokenizer=None, + max_new_tokens=1, + min_context_tokens=8, + ) + + assert padded.tensors[0].shape == (1, 8) + assert padded.metadata["target_token_count"] == 8 + + +def test_audio_waveform_loader_caps_duration(monkeypatch, tmp_path: Path) -> None: + sample_rate = 16000 + audio_path = tmp_path / "long.wav" + wavfile.write(audio_path, sample_rate, np.ones(sample_rate * 2, dtype=np.float32)) + + monkeypatch.setenv("CACTUS_TRANSPILER_MAX_AUDIO_SECONDS", "0.5") + waveform = audio_preprocess.load_audio_waveform( + audio_path, + target_sample_rate=sample_rate, + ) + + assert waveform.shape == (sample_rate // 2,) + + +def test_materialized_transpile_constants_are_cactus_tensor_files(tmp_path: Path) -> None: + tensor_path = tmp_path / "constant.weights" + expected = np.arange(6, dtype=np.float16).reshape(2, 3) + + hf_model._write_cactus_constant_tensor( + output_path=tensor_path, + value=expected, + precision=int(hf_model.Graph.FP16), + ) + + assert {path.name for path in tmp_path.iterdir()} == {"constant.weights"} + loaded = component_bundle_runtime._open_cactus_tensor_file(tensor_path) + assert loaded.precision == int(hf_model.Graph.FP16) + assert tuple(loaded.shape) == (2, 3) + np.testing.assert_array_equal(loaded.data.reshape(loaded.shape), expected) + + +def test_stateful_decode_graphs_are_reloaded_when_bundle_cache_hits(monkeypatch, tmp_path: Path) -> None: + manifest = { + "components": [ + { + "component": "vision_encoder", + "logical_inputs": ["pixel_values"], + "logical_outputs": ["image_features"], + }, + { + "component": "decoder_prefill_chunk", + "logical_inputs": ["inputs_embeds"], + "logical_outputs": ["logits"], + }, + { + "component": "decoder_step", + "logical_inputs": ["inputs_embeds"], + "logical_outputs": ["logits"], + }, + ], + } + calls: list[str] = [] + + class FakeComponent: + def __init__(self, component: str): + self.component = component + + def fake_manifest(_bundle_dir_or_manifest): + return tmp_path, manifest + + def fake_load_saved_component_graph(*, component_entry, **_kwargs): + component = str(component_entry["component"]) + calls.append(component) + return FakeComponent(component) + + component_bundle_runtime._COMPONENT_GRAPH_CACHE.clear() + monkeypatch.setattr(component_bundle_runtime, "load_component_bundle_manifest", fake_manifest) + monkeypatch.setattr(component_bundle_runtime, "load_saved_component_graph", fake_load_saved_component_graph) + + loaded, _ = component_bundle_runtime.load_saved_component_graphs(tmp_path) + assert set(loaded) == {"vision_encoder", "decoder_prefill_chunk", "decoder_step"} + assert calls == ["vision_encoder", "decoder_prefill_chunk", "decoder_step"] + + calls.clear() + loaded, _ = component_bundle_runtime.load_saved_component_graphs(tmp_path) + assert set(loaded) == {"vision_encoder", "decoder_prefill_chunk", "decoder_step"} + assert calls == ["decoder_prefill_chunk", "decoder_step"] + + +def test_skipped_component_outputs_are_seeded_as_zeros() -> None: + class FakeTensor: + dtype = component_bundle_runtime.Graph.FP16 + shape = (1, 2, 3) + + class FakeComponent: + component = "audio_encoder" + outputs = [FakeTensor()] + _output_names = ("audio_features",) + + store: dict[str, np.ndarray] = {} + component_bundle_runtime._seed_skipped_component_outputs( + store, + component_graphs={"audio_encoder": FakeComponent()}, # type: ignore[dict-item] + component_names=("audio_encoder",), + ) + + assert set(store) == {"audio_features"} + assert store["audio_features"].shape == (1, 2, 3) + assert store["audio_features"].dtype == np.float16 + assert np.all(store["audio_features"] == 0) + + +def test_gemma4_multimodal_bundle_uses_text_only_cached_path_without_media(monkeypatch) -> None: + calls: list[dict[str, object]] = [] + + def fake_text_only(**kwargs): + calls.append(kwargs) + return {"response": "ok", "decode_mode": "cached_step_text"} + + monkeypatch.setattr(component_bundle_runtime, "_run_gemma4_text_only_cached_bundle", fake_text_only) + + result = component_bundle_runtime._run_multimodal_causal_lm_bundle( + component_graphs={ + "lm_encoder_step": object(), # type: ignore[dict-item] + "decoder_step": object(), # type: ignore[dict-item] + }, + manifest={"family": "gemma4", "task": "multimodal_causal_lm_logits", "inputs": {}}, + prompt="Hello", + image_files=(), + audio_file=None, + torch_dtype=component_bundle_runtime.torch.float16, + system_prompt=None, + enable_thinking=False, + max_new_tokens=1, + stop_sequences=(), + ) + + assert result == {"response": "ok", "decode_mode": "cached_step_text"} + assert calls and calls[0]["prompt"] == "Hello" + + +def test_gemma4_prompt_uses_chat_turn_format() -> None: + class FakeTokenizer: + def __call__(self, text, **_kwargs): + return {"input_ids": [ord(char) for char in text]} + + ids = component_bundle_runtime._tokenize_bundle_prompt_for_manifest( + {"family": "gemma4"}, + FakeTokenizer(), + "Hello", + ) + decoded = "".join(chr(value) for value in ids) + + assert decoded == "<|turn>user\nHello\n<|turn>model\n" + + +def test_gemma4_stop_token_ids_include_turn_end() -> None: + class FakeTokenizer: + eos_token_id = None + + def convert_tokens_to_ids(self, token): + return {"": 106, "": 1}.get(token) + + assert component_bundle_runtime._bundle_stop_token_ids( + manifest={"family": "gemma4"}, + tokenizer=FakeTokenizer(), + ) == {1, 106} + + +def test_media_turns_do_not_replay_text_history() -> None: + history = [ + ("user", "Tell me about mountains"), + ("assistant", "Mountains are tall landforms." * 20), + ] + + assert transpile_cli._history_for_transpiled_turn(history, has_media=False) is history + assert transpile_cli._history_for_transpiled_turn(history, has_media=True) == [] + + +def test_runtime_image_inputs_resize_to_static_square(tmp_path: Path) -> None: + try: + from PIL import Image + except Exception: + return + + image_path = tmp_path / "tall.png" + Image.new("RGB", (20, 40), color=(255, 0, 0)).save(image_path) + + images = multimodal_runtime._load_image_inputs((str(image_path),)) + lfm_images = component_bundle_runtime._load_image_inputs_for_runtime((str(image_path),)) + + assert images[0].size == (256, 256) + assert lfm_images[0].size == (256, 256) diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py new file mode 100644 index 000000000..676c2eeb2 --- /dev/null +++ b/python/tests/test_cli_transpile_defaults.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +from argparse import Namespace +from pathlib import Path + +from cactus import cli +from cactus.cli import convert as convert_cli + + +def _write_gemma4_multimodal_config(output_dir: Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "hf_config.json").write_text( + ( + '{"model_type":"gemma4",' + '"architectures":["Gemma4ForConditionalGeneration"],' + '"vision_config":{"model_type":"gemma4_vision"},' + '"audio_config":{"model_type":"gemma4_audio"}}' + ), + encoding="utf-8", + ) + + +def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[str]: + assets_dir = convert_cli.PROJECT_ROOT / "cactus-engine" / "tests" / "assets" + return [ + "--weights-dir", + str(model_dir), + "--artifact-dir", + str(artifact_dir), + "--task", + "multimodal_causal_lm_logits", + "--max-new-tokens", + "32", + "--component-pipeline", + "on", + "--prompt", + convert_cli._DEFAULT_MULTIMODAL_PROMPT, + "--components", + "vision_encoder,audio_encoder,lm_encoder,decoder", + "--image-file", + str(assets_dir / "test_monkey.png"), + "--audio-file", + str(assets_dir / "test.wav"), + "--trust-remote-code", + ] + + +def test_cmd_convert_transpiles_into_same_weights_folder(monkeypatch, tmp_path: Path) -> None: + parser = cli.create_parser() + args = parser.parse_args(["convert", "gemma4"]) + + model_dir = tmp_path / "weights" / "gemma-4-e2b-it" + calls: list[tuple[str, object]] = [] + + def _fake_cq_main(command): + calls.append(("cq", list(command))) + _write_gemma4_multimodal_config(model_dir) + return 0 + + def _fake_cmd_transpile(transpile_args): + calls.append(("transpile", transpile_args)) + assert transpile_args.model_id == "google/gemma-4-E2B-it" + assert transpile_args.execute_after_transpile is False + assert transpile_args.allow_unconverted_weights is False + return 0 + + monkeypatch.setattr(convert_cli, "get_weights_dir", lambda model_id: model_dir) + monkeypatch.setattr(convert_cli, "cmd_transpile", _fake_cmd_transpile) + + import cactus.convert.cli as cq_cli + + monkeypatch.setattr(cq_cli, "main", _fake_cq_main) + + rc = convert_cli.cmd_convert(args) + + assert rc == 0 + assert calls[0][0] == "cq" + assert calls[0][1] == [ + "convert", + "--model", + "google/gemma-4-E2B-it", + "--out", + str(model_dir), + "--bits", + "4", + "--force", + ] + assert calls[1][0] == "transpile" + assert calls[1][1].extra_args == _gemma4_multimodal_extra_args(model_dir, model_dir) + assert not (model_dir / "transpile_entrypoints.json").exists() + + +def test_cmd_convert_honors_explicit_output_dir(monkeypatch, tmp_path: Path) -> None: + parser = cli.create_parser() + output_dir = tmp_path / "custom" + args = parser.parse_args(["convert", "google/gemma-4-E2B-it", str(output_dir)]) + + cq_calls: list[list[str]] = [] + + def _fake_cq_main(command): + cq_calls.append(list(command)) + _write_gemma4_multimodal_config(output_dir) + return 0 + + transpile_calls: list[Namespace] = [] + + def _fake_cmd_transpile(transpile_args): + transpile_calls.append(transpile_args) + return 0 + + monkeypatch.setattr(convert_cli, "cmd_transpile", _fake_cmd_transpile) + + import cactus.convert.cli as cq_cli + + monkeypatch.setattr(cq_cli, "main", _fake_cq_main) + + rc = convert_cli.cmd_convert(args) + + assert rc == 0 + assert cq_calls[0] == [ + "convert", + "--model", + "google/gemma-4-E2B-it", + "--out", + str(output_dir), + "--bits", + "4", + "--force", + ] + assert len(transpile_calls) == 1 + assert transpile_calls[0].extra_args == _gemma4_multimodal_extra_args(output_dir, output_dir) + + +def test_cmd_convert_supplies_default_audio_for_parakeet(monkeypatch, tmp_path: Path) -> None: + parser = cli.create_parser() + output_dir = tmp_path / "parakeet" + args = parser.parse_args(["convert", "nvidia/parakeet-tdt-0.6b-v3", str(output_dir)]) + + def _fake_cq_main(command): + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "hf_config.json").write_text( + '{"model_type":"parakeet_tdt","architectures":["ParakeetForTDT"]}', + encoding="utf-8", + ) + return 0 + + transpile_calls: list[Namespace] = [] + + def _fake_cmd_transpile(transpile_args): + transpile_calls.append(transpile_args) + return 0 + + monkeypatch.setattr(convert_cli, "cmd_transpile", _fake_cmd_transpile) + + import cactus.convert.cli as cq_cli + + monkeypatch.setattr(cq_cli, "main", _fake_cq_main) + + rc = convert_cli.cmd_convert(args) + + assert rc == 0 + extra_args = transpile_calls[0].extra_args + assert extra_args[extra_args.index("--task") + 1] == "tdt_transcription" + assert "--audio-file" in extra_args + assert extra_args[extra_args.index("--audio-file") + 1].endswith("cactus-engine/tests/assets/test.wav") + + +def test_cmd_convert_supplies_default_audio_for_whisper(monkeypatch, tmp_path: Path) -> None: + parser = cli.create_parser() + output_dir = tmp_path / "whisper" + args = parser.parse_args(["convert", "whisper", str(output_dir)]) + + def _fake_cq_main(command): + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "hf_config.json").write_text( + '{"model_type":"whisper","architectures":["WhisperForConditionalGeneration"]}', + encoding="utf-8", + ) + return 0 + + transpile_calls: list[Namespace] = [] + + def _fake_cmd_transpile(transpile_args): + transpile_calls.append(transpile_args) + return 0 + + monkeypatch.setattr(convert_cli, "cmd_transpile", _fake_cmd_transpile) + + import cactus.convert.cli as cq_cli + + monkeypatch.setattr(cq_cli, "main", _fake_cq_main) + + rc = convert_cli.cmd_convert(args) + + assert rc == 0 + extra_args = transpile_calls[0].extra_args + assert extra_args[extra_args.index("--task") + 1] == "seq2seq_transcription" + assert "--audio-file" in extra_args + + +def test_cmd_convert_infers_text_tasks_for_qwen_and_lfm(monkeypatch, tmp_path: Path) -> None: + parser = cli.create_parser() + + import cactus.convert.cli as cq_cli + + for alias, model_type, arch in ( + ("qwen", "qwen3", "Qwen3ForCausalLM"), + ("lfm", "lfm2_vl", "Lfm2VlForConditionalGeneration"), + ): + output_dir = tmp_path / alias + args = parser.parse_args(["convert", alias, str(output_dir)]) + transpile_calls: list[Namespace] = [] + + def _fake_cq_main(command, *, output_dir=output_dir, model_type=model_type, arch=arch): + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "hf_config.json").write_text( + f'{{"model_type":"{model_type}","architectures":["{arch}"]}}', + encoding="utf-8", + ) + return 0 + + def _fake_cmd_transpile(transpile_args): + transpile_calls.append(transpile_args) + return 0 + + monkeypatch.setattr(convert_cli, "cmd_transpile", _fake_cmd_transpile) + monkeypatch.setattr(cq_cli, "main", _fake_cq_main) + + rc = convert_cli.cmd_convert(args) + + assert rc == 0 + extra_args = transpile_calls[0].extra_args + assert extra_args[extra_args.index("--task") + 1] == "causal_lm_logits" + assert "--audio-file" not in extra_args + + +def test_cmd_convert_infers_multimodal_components_from_vision_config(monkeypatch, tmp_path: Path) -> None: + parser = cli.create_parser() + output_dir = tmp_path / "lfm2-vl" + args = parser.parse_args(["convert", "LiquidAI/LFM2-VL-450M", str(output_dir)]) + + def _fake_cq_main(command): + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "hf_config.json").write_text( + ( + '{"model_type":"lfm2_vl",' + '"architectures":["Lfm2VlForConditionalGeneration"],' + '"vision_config":{"model_type":"siglip2_vision_model"}}' + ), + encoding="utf-8", + ) + return 0 + + transpile_calls: list[Namespace] = [] + + def _fake_cmd_transpile(transpile_args): + transpile_calls.append(transpile_args) + return 0 + + monkeypatch.setattr(convert_cli, "cmd_transpile", _fake_cmd_transpile) + + import cactus.convert.cli as cq_cli + + monkeypatch.setattr(cq_cli, "main", _fake_cq_main) + + rc = convert_cli.cmd_convert(args) + + assert rc == 0 + assert len(transpile_calls) == 1 + extra_args = transpile_calls[0].extra_args + assert extra_args[extra_args.index("--task") + 1] == "multimodal_causal_lm_logits" + assert extra_args[extra_args.index("--component-pipeline") + 1] == "on" + assert extra_args[extra_args.index("--components") + 1] == "vision_encoder,lm_encoder,decoder" + assert "--image-file" in extra_args + assert "--audio-file" not in extra_args + + +def test_cli_no_longer_registers_transpile_command() -> None: + parser = cli.create_parser() + try: + parser.parse_args(["transpile", "gemma4"]) + except SystemExit as exc: + assert exc.code != 0 + else: + raise AssertionError("transpile should no longer be a public CLI command") diff --git a/python/tests/test_component_partition.py b/python/tests/test_component_partition.py new file mode 100644 index 000000000..a87ca03bd --- /dev/null +++ b/python/tests/test_component_partition.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from cactus.transpile.component_partition import COMPONENT_LM_ENCODER +from cactus.transpile.component_partition import COMPONENT_VISION_ENCODER +from cactus.transpile.component_partition import classify_node_component +from cactus.transpile.component_partition import classify_value_component +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue + + +def test_component_partition_ignores_torch_layer_names() -> None: + value = IRValue( + id="weight", + meta={ + "source_name": "model.vision_tower.encoder.layers.0.self_attn.q_proj.weight", + "torch_name": "model.layers.0.mlp.gate_proj.weight", + }, + ) + + assert classify_value_component( + "weight", + value, + family="", + task="multimodal_causal_lm_logits", + ) is None + + +def test_component_partition_uses_semantic_input_names() -> None: + assert ( + classify_value_component( + "pixel_values", + IRValue(id="pixel_values"), + family="", + task="multimodal_causal_lm_logits", + ) + == COMPONENT_VISION_ENCODER + ) + + +def test_component_partition_uses_ops_for_multimodal_merge() -> None: + node = IRNode( + id="embedding_0", + op="embedding", + inputs=["input_ids", "weight"], + outputs=["inputs_embeds"], + ) + + assert ( + classify_node_component( + node, + family="", + task="multimodal_causal_lm_logits", + ) + == COMPONENT_LM_ENCODER + ) diff --git a/python/tests/test_graph.py b/python/tests/test_graph.py index 5739d91dd..c8b05f88e 100644 --- a/python/tests/test_graph.py +++ b/python/tests/test_graph.py @@ -453,6 +453,46 @@ def test_softmax_2d(self): np.testing.assert_allclose(out, expected, atol=5e-2) +class TestGraphCumsum(unittest.TestCase): + + def test_cumsum_last_axis(self): + g = Graph() + a = g.input((2, 4)) + y = a.cumsum(axis=-1) + + data = np.array([[1, 2, 3, 4], [0.5, 1.5, 2.5, 3.5]], dtype=np.float16) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = np.cumsum(data.astype(np.float32), axis=-1).astype(np.float16) + np.testing.assert_allclose(out, expected, atol=5e-2) + + def test_save_load_roundtrip_cumsum(self): + g = Graph() + a = g.input((3, 2)) + y = a.cumsum(axis=0) + + data = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float16) + g.set_input(a, data) + g.execute() + expected = y.numpy() + + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "cumsum_graph.cg" + g.save(path) + + loaded = Graph.load(path) + loaded_a = Tensor(loaded, a.id, a.shape, a.dtype) + loaded_y = Tensor(loaded, y.id, y.shape, y.dtype) + + loaded.set_input(loaded_a, data) + loaded.execute() + + out = loaded_y.numpy() + np.testing.assert_allclose(out, expected, atol=1e-2) + + class TestGraphSaveLoad(unittest.TestCase): def _rebind_tensor(self, graph, tensor): diff --git a/python/tests/test_optimize_gemma4_attention.py b/python/tests/test_optimize_gemma4_attention.py new file mode 100644 index 000000000..843e3f65c --- /dev/null +++ b/python/tests/test_optimize_gemma4_attention.py @@ -0,0 +1,224 @@ +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.optimize_graph import normalize_gemma4_decoder_attention_semantics + + +def test_normalize_gemma4_full_attention_uses_sequence_window_compat() -> None: + graph = IRGraph( + values={ + "query": IRValue(id="query", shape=(1, 8, 800, 512), dtype="fp16"), + "key": IRValue(id="key", shape=(1, 1, 800, 512), dtype="fp16"), + "value": IRValue(id="value", shape=(1, 1, 800, 512), dtype="fp16"), + "full_attention_mask": IRValue( + id="full_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "proj": IRValue(id="proj", shape=(1536, 4096), dtype="fp16"), + "out": IRValue(id="out", shape=(1, 800, 1536), dtype="fp16", producer="attn"), + }, + nodes={ + "attn": IRNode( + id="attn", + op="attention_block", + inputs=["query", "key", "value", "full_attention_mask", "proj"], + outputs=["out"], + attrs={ + "has_mask": True, + "is_causal": False, + "window_size": 0, + "attention_output_shape": (1, 800, 4096), + }, + meta={"attention_layer_type": "full_attention"}, + ) + }, + order=["attn"], + inputs=["query", "key", "value", "full_attention_mask"], + outputs=["out"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "input_names": ("query", "key", "value", "full_attention_mask"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn"].inputs == ["query", "key", "value", "proj"] + assert graph.nodes["attn"].attrs["has_mask"] is False + assert graph.nodes["attn"].attrs["is_causal"] is True + assert graph.nodes["attn"].attrs["window_size"] == 800 + assert graph.nodes["attn"].meta["gemma4_full_attention_window_compat"] is True + + +def test_normalize_gemma4_sliding_attention_elides_runtime_mask() -> None: + graph = IRGraph( + values={ + "query": IRValue(id="query", shape=(1, 8, 800, 256), dtype="fp16"), + "key": IRValue(id="key", shape=(1, 8, 800, 256), dtype="fp16"), + "value": IRValue(id="value", shape=(1, 8, 800, 256), dtype="fp16"), + "sliding_attention_mask": IRValue( + id="sliding_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "proj": IRValue(id="proj", shape=(1536, 2048), dtype="fp16"), + "out": IRValue(id="out", shape=(1, 800, 1536), dtype="fp16", producer="attn"), + }, + nodes={ + "attn": IRNode( + id="attn", + op="attention_block", + inputs=["query", "key", "value", "sliding_attention_mask", "proj"], + outputs=["out"], + attrs={ + "has_mask": True, + "is_causal": False, + "window_size": 0, + "attention_output_shape": (1, 800, 2048), + }, + meta={"attention_layer_type": "sliding_attention"}, + ) + }, + order=["attn"], + inputs=["query", "key", "value", "sliding_attention_mask"], + outputs=["out"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "sliding_window": 512, + "input_names": ("query", "key", "value", "sliding_attention_mask"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn"].inputs == ["query", "key", "value", "proj"] + assert graph.nodes["attn"].attrs["has_mask"] is False + assert graph.nodes["attn"].attrs["is_causal"] is True + assert graph.nodes["attn"].attrs["window_size"] == 512 + + +def test_normalize_gemma4_attention_recovers_layer_hints_from_graph_meta() -> None: + graph = IRGraph( + values={ + "query_0": IRValue(id="query_0", shape=(1, 8, 800, 256), dtype="fp16"), + "key_0": IRValue(id="key_0", shape=(1, 8, 800, 256), dtype="fp16"), + "value_0": IRValue(id="value_0", shape=(1, 8, 800, 256), dtype="fp16"), + "sliding_attention_mask": IRValue( + id="sliding_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "out_0": IRValue(id="out_0", shape=(1, 8, 800, 256), dtype="fp16", producer="attn_0"), + "query_1": IRValue(id="query_1", shape=(1, 8, 800, 256), dtype="fp16"), + "key_1": IRValue(id="key_1", shape=(1, 8, 800, 256), dtype="fp16"), + "value_1": IRValue(id="value_1", shape=(1, 8, 800, 256), dtype="fp16"), + "full_attention_mask": IRValue( + id="full_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "out_1": IRValue(id="out_1", shape=(1, 8, 800, 256), dtype="fp16", producer="attn_1"), + }, + nodes={ + "attn_0": IRNode( + id="attn_0", + op="attention", + inputs=["query_0", "key_0", "value_0", "sliding_attention_mask"], + outputs=["out_0"], + attrs={"is_causal": False, "window_size": 0}, + ), + "attn_1": IRNode( + id="attn_1", + op="attention", + inputs=["query_1", "key_1", "value_1", "full_attention_mask"], + outputs=["out_1"], + attrs={"is_causal": False, "window_size": 0}, + ), + }, + order=["attn_0", "attn_1"], + inputs=[ + "query_0", + "key_0", + "value_0", + "sliding_attention_mask", + "query_1", + "key_1", + "value_1", + "full_attention_mask", + ], + outputs=["out_1"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "sliding_window": 512, + "layer_types": ("sliding_attention", "full_attention"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn_0"].meta["attention_layer_type"] == "sliding_attention" + assert graph.nodes["attn_0"].meta["attention_layer_index"] == 0 + assert graph.nodes["attn_0"].inputs == ["query_0", "key_0", "value_0"] + assert graph.nodes["attn_0"].attrs["is_causal"] is True + assert graph.nodes["attn_0"].attrs["window_size"] == 512 + assert graph.nodes["attn_1"].meta["attention_layer_type"] == "full_attention" + assert graph.nodes["attn_1"].meta["attention_layer_index"] == 1 + assert graph.nodes["attn_1"].inputs == ["query_1", "key_1", "value_1"] + assert graph.nodes["attn_1"].attrs["is_causal"] is True + assert graph.nodes["attn_1"].attrs["window_size"] == 800 + + +def test_normalize_gemma4_attention_elides_logical_fp16_mask() -> None: + graph = IRGraph( + values={ + "query": IRValue(id="query", shape=(1, 8, 800, 256), dtype="fp16"), + "key": IRValue(id="key", shape=(1, 8, 800, 256), dtype="fp16"), + "value": IRValue(id="value", shape=(1, 8, 800, 256), dtype="fp16"), + "mask_a": IRValue(id="mask_a", shape=(1, 1, 800, 800), dtype="bool"), + "mask_b": IRValue(id="mask_b", shape=(1, 1, 800, 800), dtype="bool"), + "mask_fp16": IRValue(id="mask_fp16", shape=(1, 1, 800, 800), dtype="fp16", producer="mask_and"), + "out": IRValue(id="out", shape=(1, 8, 800, 256), dtype="fp16", producer="attn"), + }, + nodes={ + "mask_and": IRNode( + id="mask_and", + op="logical_and", + inputs=["mask_a", "mask_b"], + outputs=["mask_fp16"], + ), + "attn": IRNode( + id="attn", + op="attention", + inputs=["query", "key", "value", "mask_fp16"], + outputs=["out"], + attrs={"is_causal": False, "window_size": 0}, + meta={"attention_layer_type": "sliding_attention"}, + ), + }, + order=["mask_and", "attn"], + inputs=["query", "key", "value", "mask_a", "mask_b"], + outputs=["out"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "sliding_window": 512, + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn"].inputs == ["query", "key", "value"] + assert graph.nodes["attn"].attrs["is_causal"] is True + assert graph.nodes["attn"].attrs["window_size"] == 512 diff --git a/python/tests/test_transpile_aten_ops.py b/python/tests/test_transpile_aten_ops.py new file mode 100644 index 000000000..b0f0ad7f4 --- /dev/null +++ b/python/tests/test_transpile_aten_ops.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from collections import Counter + +import torch + +from cactus.transpile.aten_ops import canonical_torch_op +from cactus.transpile.capture_pytorch import capture_model +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.normalize import normalize_target +from cactus.transpile.optimize_graph import fuse_dense_mlp_tq + + +class OddlyNamedLinearBlock(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.pear_tree = torch.nn.Linear(4, 3, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return torch.nn.functional.silu(self.pear_tree(x)) + + +class SameOpsDifferentNames(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.not_a_transformer_layer = torch.nn.ModuleDict( + {"banana": torch.nn.Linear(4, 3, bias=False)} + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return torch.nn.functional.silu(self.not_a_transformer_layer["banana"](x)) + + +def _op_counts(module: torch.nn.Module) -> Counter[str]: + example = torch.randn(2, 4) + captured = capture_model(module, (example,)) + return Counter(captured.ir_graph.nodes[node_id].op for node_id in captured.ir_graph.order) + + +def test_aten_targets_normalize_from_real_op_overloads() -> None: + assert canonical_torch_op(torch.ops.aten.add.Tensor) == "aten.add.Tensor" + assert normalize_target(torch.ops.aten.add.Tensor) == "add" + assert normalize_target(torch.ops.aten.linear.default) == "linear" + assert normalize_target("torch.ops.aten.silu.default") == "silu" + + +def test_import_is_based_on_aten_ops_not_module_names() -> None: + first = _op_counts(OddlyNamedLinearBlock()) + second = _op_counts(SameOpsDifferentNames()) + + assert first == second + assert first["linear"] == 1 + assert first["silu"] == 1 + + +def test_import_records_canonical_aten_op_metadata() -> None: + captured = capture_model(OddlyNamedLinearBlock(), (torch.randn(2, 4),)) + node_meta = [captured.ir_graph.nodes[node_id].meta for node_id in captured.ir_graph.order] + + assert {meta["aten_op"] for meta in node_meta} >= {"aten.linear.default", "aten.silu.default"} + assert all("torch_name" in meta for meta in node_meta) + assert all("module_paths" not in meta for meta in node_meta) + + +def test_dense_mlp_fusion_uses_topology_not_layer_names() -> None: + graph = IRGraph( + values={ + "x": IRValue(id="x", shape=(1, 4), dtype="fp16"), + "pear_weight": IRValue(id="pear_weight", shape=(8, 4), dtype="fp16", meta={"path": "pear.cq4.weights"}), + "banana_weight": IRValue(id="banana_weight", shape=(8, 4), dtype="fp16", meta={"path": "banana.cq4.weights"}), + "plum_weight": IRValue(id="plum_weight", shape=(4, 8), dtype="fp16", meta={"path": "plum.cq4.weights"}), + "gate": IRValue(id="gate", shape=(1, 8), dtype="fp16", producer="gate_linear"), + "activated": IRValue(id="activated", shape=(1, 8), dtype="fp16", producer="gate_gelu"), + "up": IRValue(id="up", shape=(1, 8), dtype="fp16", producer="up_linear"), + "product": IRValue(id="product", shape=(1, 8), dtype="fp16", producer="product"), + "out": IRValue(id="out", shape=(1, 4), dtype="fp16", producer="down_linear"), + }, + nodes={ + "gate_linear": IRNode( + id="gate_linear", + op="linear", + inputs=["x", "pear_weight"], + outputs=["gate"], + attrs={"has_bias": False}, + ), + "gate_gelu": IRNode( + id="gate_gelu", + op="gelu", + inputs=["gate"], + outputs=["activated"], + ), + "up_linear": IRNode( + id="up_linear", + op="linear", + inputs=["x", "banana_weight"], + outputs=["up"], + attrs={"has_bias": False}, + ), + "product": IRNode( + id="product", + op="multiply", + inputs=["activated", "up"], + outputs=["product"], + ), + "down_linear": IRNode( + id="down_linear", + op="linear", + inputs=["product", "plum_weight"], + outputs=["out"], + attrs={"has_bias": False}, + ), + }, + order=["gate_linear", "gate_gelu", "up_linear", "product", "down_linear"], + inputs=["x"], + outputs=["out"], + constants={}, + meta={}, + ) + + assert fuse_dense_mlp_tq(graph) is True + assert graph.nodes["down_linear"].op == "dense_mlp_tq_fused" + assert graph.nodes["down_linear"].inputs == ["x", "pear_weight", "banana_weight", "plum_weight"] diff --git a/python/tests/test_transpile_weight_compat.py b/python/tests/test_transpile_weight_compat.py new file mode 100644 index 000000000..daa43950e --- /dev/null +++ b/python/tests/test_transpile_weight_compat.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +from pathlib import Path +import json + +import numpy as np + +from cactus.convert.cactus_adapters.tensor_io import save_tensor_with_header +from cactus.transpile.weight_binding import WeightBinding +from cactus.transpile.weight_binding import resolve_weight_binding +from cactus.transpile.weight_compat import _open_cactus_tensor_file +from cactus.transpile.weight_compat import ensure_binding_compatible +from cactus.transpile.weight_compat import ensure_embedding_binding_compatible + + +def _write_grouped_int8_embedding(path: Path, rows: int, cols: int) -> None: + rng = np.random.default_rng(1234) + tensor = rng.standard_normal((rows, cols), dtype=np.float32) + save_tensor_with_header(tensor, path, precision="INT8", model_type="generic") + + +def test_token_embedding_binding_upgrades_to_cq4(tmp_path: Path) -> None: + source = tmp_path / "token_embeddings.weights" + _write_grouped_int8_embedding(source, rows=8, cols=128) + + legacy_fp16 = tmp_path / "token_embeddings.fp16.weights" + legacy_fp16.write_bytes(b"legacy") + + binding = WeightBinding( + path=str(source), + kind="embedding", + source_name="model.embed_tokens.weight", + ) + compat = ensure_embedding_binding_compatible(binding) + + assert compat.path.endswith(".cq4.weights") + assert compat.kind == "embedding" + assert not legacy_fp16.exists() + + opened = _open_cactus_tensor_file(compat.path) + assert opened.precision == 6 + assert opened.shape == (8, 128) + assert opened.scales is not None + + +def test_per_layer_embedding_binding_upgrades_to_cq2(tmp_path: Path) -> None: + source = tmp_path / "embed_tokens_per_layer.weights" + _write_grouped_int8_embedding(source, rows=8, cols=256) + + binding = WeightBinding( + path=str(source), + kind="embedding", + source_name="model.language_model.embed_tokens_per_layer.weight", + ) + compat = ensure_embedding_binding_compatible(binding) + + assert compat.path.endswith(".cq2.weights") + + opened = _open_cactus_tensor_file(compat.path) + assert opened.precision == 4 + assert opened.shape == (8, 256) + assert opened.scales is not None + + +def test_gemma4_per_layer_projection_binding_upgrades_legacy_int4(tmp_path: Path) -> None: + rng = np.random.default_rng(1234) + source = tmp_path / "per_layer_model_proj.weights" + tensor = rng.standard_normal((8, 128), dtype=np.float32) + save_tensor_with_header(tensor, source, precision="INT4", model_type="generic") + + binding = WeightBinding( + path=str(source), + kind="weight", + source_name="module.model.model.language_model.per_layer_model_projection.weight", + ) + compat = ensure_binding_compatible(binding, source_tensor=tensor) + + assert compat.path.endswith(".cq4.weights") + opened = _open_cactus_tensor_file(compat.path) + assert opened.precision == 6 + assert opened.shape == (8, 128) + assert opened.scales is not None + + +def test_decoder_binding_prefers_existing_cq4_companion(tmp_path: Path) -> None: + rng = np.random.default_rng(4321) + source = tmp_path / "layer_0_attn_q.weights" + tensor = rng.standard_normal((8, 128), dtype=np.float32) + save_tensor_with_header(tensor, source, precision="INT4", model_type="generic") + + companion = tmp_path / "layer_0_attn_q.cq4.weights" + save_tensor_with_header(tensor, companion, precision="FP16", model_type="generic") + + binding = WeightBinding( + path=str(source), + kind="weight", + source_name="module.backbone.layers.0.self_attn.q_proj.weight", + ) + compat = ensure_binding_compatible(binding, source_tensor=0) + + assert compat.path == str(companion) + + +def test_resolve_weight_binding_does_not_guess_legacy_filenames(tmp_path: Path) -> None: + (tmp_path / "token_embeddings.weights").write_bytes(b"") + (tmp_path / "layer_0_attn_q.weights").write_bytes(b"") + (tmp_path / "embed_tokens_per_layer.weights").write_bytes(b"") + (tmp_path / "per_layer_model_proj.weights").write_bytes(b"") + + assert ( + resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="model.embed_tokens.weight", + ) + is None + ) + assert ( + resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="model.layers.0.self_attn.q_proj.weight", + ) + is None + ) + assert ( + resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="model.language_model.embed_tokens_per_layer.weight", + ) + is None + ) + assert ( + resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="model.language_model.per_layer_model_projection.weight", + ) + is None + ) + + +def test_resolve_weight_binding_prefers_exact_manifest_without_name_guessing(tmp_path: Path) -> None: + (tmp_path / "exact.weights").write_bytes(b"") + (tmp_path / "token_embeddings.weights").write_bytes(b"") + (tmp_path / "weights_manifest.json").write_text( + json.dumps( + { + "v_adapter_named_anything": { + "filename": "exact.weights", + "kind": "weight", + } + } + ) + ) + + exact = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="v_adapter_named_anything", + ) + assert exact is not None + assert exact.path.endswith("exact.weights") + + guessed = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="model.embed_tokens.weight", + ) + assert guessed is None + + +def test_resolve_weight_binding_reads_convert_manifest_aliases(tmp_path: Path) -> None: + (tmp_path / "decoder_attn_q.cq4.weights").write_bytes(b"") + (tmp_path / "weights_manifest.json").write_text( + json.dumps( + { + "weights": [ + { + "source_name": "model.layers.0.self_attn.q_proj.weight", + "hf_name": "model.layers.0.self_attn.q_proj.weight", + "adapter_name": "backbone.layers.0.self_attn.q_proj.weight", + "source_names": ["adapter.backbone.layers.0.self_attn.q_proj.weight"], + "output_name": "decoder_attn_q.cq4.weights", + "component": "language", + } + ] + } + ) + ) + + binding = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="adapter.backbone.layers.0.self_attn.q_proj.weight", + ) + + assert binding is not None + assert binding.path.endswith("decoder_attn_q.cq4.weights") + + +def test_resolve_weight_binding_expands_manifest_wrapper_prefixes(tmp_path: Path) -> None: + (tmp_path / "layer_0_q.cq4.weights").write_bytes(b"") + (tmp_path / "weights_manifest.json").write_text( + json.dumps( + { + "model.layers.0.self_attn.q_proj.weight": { + "filename": "layer_0_q.cq4.weights", + "kind": "weight", + } + } + ) + ) + + binding = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="adapter.model.model.layers.0.self_attn.q_proj.weight", + ) + + assert binding is not None + assert binding.path.endswith("layer_0_q.cq4.weights") + + +def test_resolve_weight_binding_expands_component_local_manifest_aliases(tmp_path: Path) -> None: + (tmp_path / "encoder_layer_0_q.cq4.weights").write_bytes(b"") + (tmp_path / "weights_manifest.json").write_text( + json.dumps( + { + "encoder.layers.0.self_attn.q_proj.weight": { + "filename": "encoder_layer_0_q.cq4.weights", + "kind": "weight", + } + } + ) + ) + + binding = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="module.layers.0.self_attn.q_proj.weight", + ) + + assert binding is not None + assert binding.path.endswith("encoder_layer_0_q.cq4.weights") + + +def test_resolve_weight_binding_expands_language_model_backbone_aliases(tmp_path: Path) -> None: + (tmp_path / "layer_0_scalar.weights").write_bytes(b"") + (tmp_path / "weights_manifest.json").write_text( + json.dumps( + { + "model.language_model.layers.0.layer_scalar": { + "filename": "layer_0_scalar.weights", + "kind": "weight", + } + } + ) + ) + + binding = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="module.backbone.layers.0.layer_scalar", + ) + + assert binding is not None + assert binding.path.endswith("layer_0_scalar.weights") + + +def test_resolve_weight_binding_expands_nested_model_tower_aliases(tmp_path: Path) -> None: + (tmp_path / "vision_q.weights").write_bytes(b"") + (tmp_path / "weights_manifest.json").write_text( + json.dumps( + { + "model.vision_tower.encoder.layers.0.self_attn.q_proj.weight": { + "filename": "vision_q.weights", + "kind": "weight", + } + } + ) + ) + + binding = resolve_weight_binding( + weights_dir=str(tmp_path), + source_name="module.model.model.vision_tower.encoder.layers.0.self_attn.q_proj.weight", + ) + + assert binding is not None + assert binding.path.endswith("vision_q.weights") + + +def test_rank5_materialized_constant_uses_cactus_tensor_file(tmp_path: Path) -> None: + from cactus.transpile.hf_model import _write_cactus_constant_tensor + from cactus.transpile.runtime_compat import Graph + + tensor = np.zeros((1, 2, 3, 4, 5), dtype=np.float16) + path = tmp_path / "rank5.weights" + + _write_cactus_constant_tensor(output_path=path, value=tensor, precision=int(Graph.FP16)) + + opened = _open_cactus_tensor_file(path) + assert opened.shape == tensor.shape + assert opened.precision == int(Graph.FP16) diff --git a/python/tests/test_whisper_seq2seq_bundle.py b/python/tests/test_whisper_seq2seq_bundle.py new file mode 100644 index 000000000..37857ace0 --- /dev/null +++ b/python/tests/test_whisper_seq2seq_bundle.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import numpy as np +import torch + +from cactus.transpile import hf_model +from cactus.transpile import component_bundle_runtime +from cactus.transpile import model_adapters + + +class _FakeTensor: + def __init__(self, array: np.ndarray): + self._array = np.asarray(array) + + def numpy(self) -> np.ndarray: + return self._array + + +class _FakeComponent: + def __init__(self, output_fn): + self._output_fn = output_fn + self._input_names = () + self._output_names = () + self._inputs = () + + def set_inputs(self, inputs): + self._inputs = tuple(inputs) + + def execute(self): + return [_FakeTensor(self._output_fn(self._inputs))] + + +class _FakeTokenizer: + eos_token_id = 99 + pad_token_id = 0 + + def encode(self, text: str, add_special_tokens: bool = False): + mapping = { + "<|endoftext|>": [99], + "<|endoftranscript|>": [99], + "": [99], + "": [0], + "override": [42], + } + return list(mapping.get(text, [])) + + def decode(self, token_ids, skip_special_tokens: bool = False): + pieces = [] + for token_id in token_ids: + if skip_special_tokens and token_id in {0, 99}: + continue + pieces.append( + { + 5: "hello", + 6: " world", + 42: "override", + 99: "<|endoftranscript|>", + }.get(int(token_id), f"<{int(token_id)}>") + ) + return "".join(pieces) + + +def test_infer_task_from_config_prefers_whisper_seq2seq(monkeypatch) -> None: + monkeypatch.setattr( + hf_model, + "_load_config_json", + lambda _: {"model_type": "whisper", "architectures": ["WhisperForConditionalGeneration"]}, + ) + assert hf_model._infer_task_from_config("openai/whisper-small") == "seq2seq_transcription" + + +def test_run_seq2seq_transcription_bundle_decodes_until_eos(monkeypatch) -> None: + def _fake_prepare_features(*, audio_file, manifest, expected_shape, torch_dtype): + _ = audio_file, manifest, expected_shape, torch_dtype + return np.zeros((1, 80, 3000), dtype=np.float16), 123 + + monkeypatch.setattr( + component_bundle_runtime, + "_prepare_generic_audio_encoder_features", + _fake_prepare_features, + ) + monkeypatch.setattr( + component_bundle_runtime, + "_load_bundle_tokenizer", + lambda manifest: _FakeTokenizer(), + ) + + encoder = _FakeComponent(lambda inputs: np.zeros((1, 1500, 4), dtype=np.float16)) + + def _decoder_outputs(inputs): + decoder_input_ids, encoder_hidden_states = inputs + _ = encoder_hidden_states + logits = np.full((1, decoder_input_ids.shape[1], 256), -100.0, dtype=np.float32) + active_tokens = int(np.count_nonzero(decoder_input_ids[0] != 0)) + step_index = max(0, active_tokens - 2) + position = max(0, active_tokens - 1) + if step_index == 0: + logits[0, position, 220] = 200.0 + logits[0, position, 5] = 100.0 + elif step_index == 1: + logits[0, position, 7] = 200.0 + logits[0, position, 6] = 100.0 + else: + logits[0, position, 99] = 100.0 + return logits + + decoder = _FakeComponent(_decoder_outputs) + manifest = { + "model_id": "openai/whisper-small", + "family": "whisper", + "task": "seq2seq_transcription", + "component_order": ["audio_encoder", "decoder"], + "inputs": { + "audio_file": "/tmp/fake.wav", + "input_shapes": {"input_features": [1, 80, 3000]}, + "decoder_input_ids": [50258, 50364], + "pad_token_id": 0, + "eos_token_id": 99, + "target_token_count": 8, + "suppress_tokens": [7], + "begin_suppress_tokens": [220], + }, + "components": [ + { + "component": "audio_encoder", + "logical_inputs": ["input_features"], + "logical_outputs": ["encoder_hidden_states"], + }, + { + "component": "decoder", + "logical_inputs": ["decoder_input_ids", "encoder_hidden_states"], + "logical_outputs": ["logits"], + }, + ], + } + + result = component_bundle_runtime._run_seq2seq_transcription_bundle( + component_graphs={"audio_encoder": encoder, "decoder": decoder}, + manifest=manifest, + audio_file="/tmp/fake.wav", + prompt=None, + torch_dtype=np.float16, # type: ignore[arg-type] + max_new_tokens=4, + stop_sequences=(), + ) + + assert result["generated_token_ids"] == [5, 6, 99] + assert result["transcript"] == "hello world" + assert result["stop_reason"] == "eos_token" + assert result["active_feature_frames"] == 123 + + +def test_resolve_whisper_decoder_prompt_token_ids_prefers_forced_decoder_ids() -> None: + class _Tokenizer: + def encode(self, text: str, add_special_tokens: bool = False): + mapping = { + "<|startoftranscript|>": [50258] if not add_special_tokens else [50258, 99999], + "prompt": [77, 88], + } + return list(mapping.get(text, [])) + + prompt_ids = hf_model._resolve_whisper_decoder_prompt_token_ids( + _Tokenizer(), + prompt=None, + decoder_start_token_id=50258, + forced_decoder_ids=[[1, 50259], [2, 50359], [3, 50363]], + ) + assert prompt_ids == [50258, 50259, 50359, 50363] + + +def test_select_last_non_pad_token_uses_last_real_position() -> None: + hidden = torch.tensor( + [[[1.0, 10.0], [2.0, 20.0], [3.0, 30.0], [4.0, 40.0]]], + dtype=torch.float32, + ) + input_ids = torch.tensor([[11, 22, 0, 0]], dtype=torch.int64) + selected = model_adapters._select_last_non_pad_token( + hidden, + input_ids, + pad_token_id=0, + ) + assert tuple(selected.shape) == (1, 1, 2) + assert torch.equal(selected[0, 0], hidden[0, 1]) diff --git a/python/tests/transpile/__init__.py b/python/tests/transpile/__init__.py new file mode 100644 index 000000000..c0dd68e2d --- /dev/null +++ b/python/tests/transpile/__init__.py @@ -0,0 +1 @@ +"""Transpiler regression tests and developer tools.""" diff --git a/python/tests/transpile/tools/__init__.py b/python/tests/transpile/tools/__init__.py new file mode 100644 index 000000000..b58eb5601 --- /dev/null +++ b/python/tests/transpile/tools/__init__.py @@ -0,0 +1 @@ +"""Developer-only transpiler comparison tools.""" diff --git a/python/tests/transpile/tools/compare_gemma4_multimodal.py b/python/tests/transpile/tools/compare_gemma4_multimodal.py new file mode 100644 index 000000000..14bee44e5 --- /dev/null +++ b/python/tests/transpile/tools/compare_gemma4_multimodal.py @@ -0,0 +1,932 @@ +from __future__ import annotations + +import argparse +import copy +import gc +import json +import os +import sys +import time +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +TOOLS_DIR = Path(__file__).resolve().parent +PYTHON_ROOT = TOOLS_DIR.parents[2] +PROJECT_ROOT = PYTHON_ROOT.parent + +sys.path.insert(0, str(PYTHON_ROOT)) + +from cactus.bindings.cactus import cactus_complete +from cactus.bindings.cactus import cactus_destroy +from cactus.bindings.cactus import cactus_init +from cactus.bindings.cactus import cactus_reset +from cactus.transpile.capture_pytorch import capture_model +from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph +from cactus.transpile.model_adapters import canonicalize_model_interface +from cactus.transpile.optimize_graph import FusionConfig +from cactus.transpile.optimize_graph import optimize_graph +from cactus.transpile.hf_model import TranspileWrapper +from cactus.transpile.hf_model import _graph_to_dict +from cactus.transpile.hf_model import _load_transformers_bundle +from cactus.transpile.hf_model import _lower_preoptimized_ir +from cactus.transpile.hf_model import _parse_dtype +from cactus.transpile.hf_model import _prepare_gemma4_multimodal_inputs +from cactus.transpile.hf_model import _serialize_json_compatible +from cactus.transpile.hf_model import _validate_weights_dir +from cactus.transpile.hf_model import _write_json + + +_DEFAULT_STOP_SEQUENCES = ("", "", "", "<|im_end|>") + + +@dataclass +class NumericSummary: + mean: float + min: float + max: float + median: float + + +@dataclass +class HandwrittenRun: + wall_ms: float + total_time_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + total_tokens: int + response: str + response_json: dict[str, Any] + + +@dataclass +class TranspiledRun: + preprocess_ms: float + execute_ms: float + total_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + total_tokens: int + response: str + stop_reason: str + generated_token_ids: list[int] + input_shapes: dict[str, list[int]] + + +def _numeric_summary(values: list[float]) -> NumericSummary: + arr = np.asarray(values, dtype=np.float64) + return NumericSummary( + mean=float(np.mean(arr)), + min=float(np.min(arr)), + max=float(np.max(arr)), + median=float(np.median(arr)), + ) + + +def _roundtrip_jsonable(value: Any) -> Any: + return json.loads(json.dumps(_serialize_json_compatible(value))) + + +def _resolve_default_weights_dir(model_id: str) -> Path | None: + env_path = os.environ.get("CACTUS_TEST_GEMMA4_MODEL") + if env_path: + candidate = Path(env_path).resolve() + if (candidate / "config.txt").exists(): + return candidate + + candidates = ( + PROJECT_ROOT / "weights" / "gemma4_int4", + PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower(), + PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower().replace("_", "-"), + ) + for candidate in candidates: + if (candidate / "config.txt").exists(): + return candidate + return None + + +def _require_processor_tokenizer(processor_or_tokenizer: object) -> object: + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + if not hasattr(tokenizer_like, "decode") or not hasattr(tokenizer_like, "encode"): + raise RuntimeError( + "Gemma4 compare script requires a tokenizer-like object with encode/decode support." + ) + return tokenizer_like + + +def _build_messages_payload( + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + system_prompt: str = "", +) -> list[dict[str, object]]: + messages: list[dict[str, object]] = [] + normalized_system = system_prompt.strip() + if normalized_system: + messages.append({"role": "system", "content": normalized_system}) + + user_message: dict[str, object] = { + "role": "user", + "content": prompt, + } + if image_files: + user_message["images"] = [str(Path(path).resolve()) for path in image_files] + if audio_file: + user_message["audio"] = [str(Path(audio_file).resolve())] + messages.append(user_message) + return messages + + +def _encode_stop_sequences(tokenizer: object, stop_sequences: tuple[str, ...]) -> list[list[int]]: + encoded: list[list[int]] = [] + encode = getattr(tokenizer, "encode", None) + if not callable(encode): + return encoded + for stop_sequence in stop_sequences: + try: + token_ids = list(encode(stop_sequence, add_special_tokens=False)) + except TypeError: + token_ids = list(encode(stop_sequence)) + if token_ids: + encoded.append([int(token_id) for token_id in token_ids]) + return encoded + + +def _has_suffix(token_ids: list[int], suffix: list[int]) -> bool: + if not suffix or len(token_ids) < len(suffix): + return False + return token_ids[-len(suffix) :] == suffix + + +def _trim_stop_suffix(token_ids: list[int], stop_sequences: list[list[int]]) -> bool: + for stop_sequence in stop_sequences: + if _has_suffix(token_ids, stop_sequence): + del token_ids[-len(stop_sequence) :] + return True + return False + + +def _decode_generated_text(tokenizer: object, token_ids: list[int], *, skip_special_tokens: bool) -> str: + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + raise RuntimeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") + try: + return str(decode(token_ids, skip_special_tokens=skip_special_tokens)) + except TypeError: + return str(decode(token_ids)) + + +def _summarize_handwritten_runs( + runs: list[HandwrittenRun], + *, + weights_dir: Path, +) -> dict[str, Any]: + if not runs: + raise ValueError("expected at least one handwritten run") + return { + "weights_dir": str(weights_dir), + "latest_response": runs[-1].response, + "total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), + "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), + "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], + } + + +def _summarize_transpiled_runs( + runs: list[TranspiledRun], + *, + compile_time_ms: float, + model_source: str, + weight_bindings: int, + prepared_input_shapes: dict[str, list[int]], +) -> dict[str, Any]: + if not runs: + raise ValueError("expected at least one transpiled run") + return { + "compile_time_ms": compile_time_ms, + "model_source": model_source, + "weight_bindings": weight_bindings, + "prepared_input_shapes": prepared_input_shapes, + "latest_response": runs[-1].response, + "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), + "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), + "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), + "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], + } + + +def _prepare_repeat_inputs( + processor: object, + *, + tokenizer: object, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str, + enable_thinking_if_supported: bool, + max_new_tokens: int, +) -> tuple[object, int, float]: + start = time.perf_counter() + prepared = _prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + use_gemma4_chat_template=True, + ) + pad_token_id = getattr(tokenizer, "pad_token_id", None) + if pad_token_id is None: + pad_token_id = 0 + prompt_tokens = 0 + for name, tensor in zip(prepared.names, prepared.tensors): + if name == "input_ids": + prompt_tokens = int(tensor.shape[1]) + break + if prompt_tokens <= 0: + raise RuntimeError("Gemma4 multimodal prepared inputs did not include a non-empty input_ids tensor") + target_tokens = prompt_tokens + int(max_new_tokens) + + input_tensors: list[torch.Tensor] = [] + for name, tensor in zip(prepared.names, prepared.tensors): + if name in {"input_ids", "attention_mask", "token_type_ids"}: + padded_shape = list(tensor.shape) + padded_shape[1] = target_tokens + if name == "input_ids": + padded = torch.full( + padded_shape, + int(pad_token_id), + dtype=tensor.dtype, + device=tensor.device, + ) + else: + padded = torch.zeros( + padded_shape, + dtype=tensor.dtype, + device=tensor.device, + ) + padded[:, : tensor.shape[1]] = tensor + input_tensors.append(padded) + else: + input_tensors.append(tensor) + metadata = dict(prepared.metadata) + metadata["prompt_token_count"] = prompt_tokens + metadata["target_token_count"] = int(target_tokens) + metadata["input_shapes"] = { + name: list(tensor.shape) + for name, tensor in zip(prepared.names, input_tensors) + } + prepared = type(prepared)( + names=prepared.names, + tensors=tuple(input_tensors), + metadata=metadata, + ) + preprocess_ms = (time.perf_counter() - start) * 1000.0 + return prepared, prompt_tokens, preprocess_ms + + +def _run_handwritten_once( + *, + model_handle: Any, + messages_json: str, + options_json: str, +) -> HandwrittenRun: + cactus_reset(model_handle) + streamed_token_ids: list[int] = [] + + def _collect(_text: str, token_id: int) -> None: + streamed_token_ids.append(int(token_id)) + + start = time.perf_counter() + response_json_text = cactus_complete( + model_handle, + messages_json, + options_json, + "[]", + _collect, + pcm_data=None, + ) + wall_ms = (time.perf_counter() - start) * 1000.0 + response_payload = json.loads(response_json_text) + return HandwrittenRun( + wall_ms=wall_ms, + total_time_ms=float(response_payload.get("total_time_ms", wall_ms)), + time_to_first_token_ms=float(response_payload.get("time_to_first_token_ms", 0.0)), + prefill_tps=float(response_payload.get("prefill_tps", 0.0)), + decode_tps=float(response_payload.get("decode_tps", 0.0)), + decode_tokens=len(streamed_token_ids), + total_tokens=int(response_payload.get("total_tokens", len(streamed_token_ids))), + response=str(response_payload.get("response", "")), + response_json=response_payload, + ) + + +def _run_transpiled_generation_once( + *, + tg, + processor: object, + tokenizer: object, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str, + enable_thinking_if_supported: bool, + max_new_tokens: int, + stop_sequences: tuple[str, ...], + expected_input_shapes: dict[str, list[int]] | None = None, + prepared_override: object | None = None, + progress_label: str = "transpiled", + progress_every_tokens: int = 8, +) -> TranspiledRun: + if prepared_override is not None: + prepared = type(prepared_override)( + names=prepared_override.names, + tensors=tuple(tensor.clone() for tensor in prepared_override.tensors), + metadata=dict(prepared_override.metadata), + ) + prompt_tokens = int(prepared.metadata.get("prompt_token_count", 0)) + preprocess_ms = 0.0 + else: + prepared, prompt_tokens, preprocess_ms = _prepare_repeat_inputs( + processor, + tokenizer=tokenizer, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + max_new_tokens=max_new_tokens, + ) + inputs_by_name = { + name: tensor.detach().cpu().numpy() + for name, tensor in zip(prepared.names, prepared.tensors) + } + if expected_input_shapes is not None: + for name, tensor in zip(prepared.names, prepared.tensors): + expected_shape = expected_input_shapes.get(name) + if expected_shape is None: + continue + actual_shape = list(tensor.shape) + if actual_shape != list(expected_shape): + raise RuntimeError( + f"runtime prepared input shape mismatch for {name}: " + f"compiled with {list(expected_shape)}, got {actual_shape}" + ) + if "input_ids" not in inputs_by_name: + raise RuntimeError("transpiled Gemma4 generation requires input_ids") + + target_tokens = int(prepared.metadata["target_token_count"]) + + eos_token_id = getattr(tokenizer, "eos_token_id", None) + + padded_inputs: dict[str, np.ndarray] = dict(inputs_by_name) + generated_ids: list[int] = [] + encoded_stop_sequences = _encode_stop_sequences(tokenizer, stop_sequences) + current_length = prompt_tokens + first_token_ms = 0.0 + execute_start = time.perf_counter() + stop_reason = "max_new_tokens" + progress_interval = max(1, int(progress_every_tokens)) + + for step_index in range(max_new_tokens): + runtime_inputs = [padded_inputs[name] for name in prepared.names] + if step_index == 0: + print("transpiled_step0_set_inputs_begin=true", flush=True) + tg.set_inputs(runtime_inputs) + if step_index == 0: + print("transpiled_step0_set_inputs_done=true", flush=True) + print("transpiled_step0_execute_begin=true", flush=True) + logits = tg.execute()[0].numpy() + if step_index == 0: + print("transpiled_step0_execute_done=true", flush=True) + if logits.ndim == 3: + token_position = current_length - 1 + if logits.shape[1] == 1: + token_position = 0 + next_token_id = int(np.argmax(logits[0, token_position])) + elif logits.ndim == 2: + next_token_id = int(np.argmax(logits[0])) + else: + raise RuntimeError(f"unexpected transpiled logits rank: {logits.ndim}") + generated_ids.append(next_token_id) + + if step_index == 0: + first_token_ms = preprocess_ms + (time.perf_counter() - execute_start) * 1000.0 + + if eos_token_id is not None and next_token_id == int(eos_token_id): + stop_reason = "eos_token" + break + if _trim_stop_suffix(generated_ids, encoded_stop_sequences): + stop_reason = "stop_sequence" + break + + if current_length >= target_tokens: + stop_reason = "context_limit" + break + + padded_inputs["input_ids"][0, current_length] = next_token_id + padded_inputs["attention_mask"][0, current_length] = 1 + padded_inputs["token_type_ids"][0, current_length] = 0 + if step_index == 0: + print("transpiled_step0_state_update_done=true", flush=True) + current_length += 1 + generated_count = len(generated_ids) + if generated_count % progress_interval == 0 or generated_count == max_new_tokens: + elapsed_ms = (time.perf_counter() - execute_start) * 1000.0 + print( + f"{progress_label}_decode_progress={generated_count}/{max_new_tokens} " + f"elapsed_ms={elapsed_ms:.3f}", + flush=True, + ) + + execute_ms = (time.perf_counter() - execute_start) * 1000.0 + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() + if not response: + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False).strip() + decode_time_ms = max(0.0, execute_ms + preprocess_ms - first_token_ms) + decode_tps = ( + ((len(generated_ids) - 1) * 1000.0) / decode_time_ms + if len(generated_ids) > 1 and decode_time_ms > 0.0 + else 0.0 + ) + prefill_tps = (prompt_tokens * 1000.0) / first_token_ms if first_token_ms > 0.0 else 0.0 + print( + f"{progress_label}_decode_done=true tokens={len(generated_ids)} " + f"stop_reason={stop_reason} execute_ms={execute_ms:.3f}", + flush=True, + ) + + return TranspiledRun( + preprocess_ms=preprocess_ms, + execute_ms=execute_ms, + total_ms=preprocess_ms + execute_ms, + time_to_first_token_ms=first_token_ms, + prefill_tps=prefill_tps, + decode_tps=decode_tps, + decode_tokens=len(generated_ids), + total_tokens=prompt_tokens + len(generated_ids), + response=response, + stop_reason=stop_reason, + generated_token_ids=generated_ids, + input_shapes={ + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + ) + + +def _print_summary(summary: dict[str, Any]) -> None: + print(f"model_id={summary['model_id']}") + print(f"prompt={summary['prompt']!r}") + if summary.get("image_files"): + print(f"image_files={summary['image_files']}") + if summary.get("audio_file"): + print(f"audio_file={summary['audio_file']}") + + handwritten = summary.get("handwritten") + if handwritten is not None: + hw_total = handwritten["total_time_ms"]["mean"] + hw_ttft = handwritten["time_to_first_token_ms"]["mean"] + hw_tokens = handwritten["decode_tokens"]["mean"] + hw_tps = hw_tokens * 1000.0 / max(hw_total, 1e-9) + print("handwritten:") + print(f" total_ms mean={hw_total:.3f}") + print(f" time_to_first_token_ms mean={hw_ttft:.3f}") + print(f" decode_tokens mean={hw_tokens:.3f}") + print(f" output_tokens_per_sec={hw_tps:.3f}") + print(f" latest_response={handwritten['latest_response']!r}") + + transpiled = summary.get("transpiled") + if transpiled is not None: + tr_total = transpiled["total_ms"]["mean"] + tr_ttft = transpiled["time_to_first_token_ms"]["mean"] + tr_tokens = transpiled["decode_tokens"]["mean"] + tr_tps = tr_tokens * 1000.0 / max(tr_total, 1e-9) + print("transpiled:") + print(f" compile_time_ms={transpiled['compile_time_ms']:.3f}") + print(f" total_ms mean={tr_total:.3f}") + print(f" time_to_first_token_ms mean={tr_ttft:.3f}") + print(f" decode_tokens mean={tr_tokens:.3f}") + print(f" output_tokens_per_sec={tr_tps:.3f}") + print(f" weight_bindings={transpiled['weight_bindings']}") + print(f" latest_response={transpiled['latest_response']!r}") + + comparison = summary.get("comparison") + if comparison is not None: + print("comparison:") + print( + " end_to_end_speedup_vs_handwritten=" + f"{comparison['end_to_end_speedup_vs_handwritten']:.6f}" + ) + print(f" responses_match={comparison['responses_match']}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Compare handwritten Cactus Gemma4 multimodal generation against the " + "transpiled Gemma4 multimodal graph on the same prompt, image, and audio." + ) + ) + parser.add_argument("--model-id", default="google/gemma-4-E2B-it") + parser.add_argument("--prompt", required=True) + parser.add_argument("--system-prompt", default="") + parser.add_argument("--image-file", action="append", default=[]) + parser.add_argument("--audio-file", default="") + parser.add_argument( + "--weights-dir", + default="", + help="Converted Cactus weights directory. Required for handwritten comparison and reused for transpiled mmap bindings.", + ) + parser.add_argument("--artifact-dir", default="", help="Optional directory for summary and transpile artifacts.") + parser.add_argument("--graph-filename", default="graph.cactus") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeats", type=int, default=1) + parser.add_argument("--max-new-tokens", type=int, default=96) + parser.add_argument( + "--transpiled-progress-every-tokens", + type=int, + default=1, + help=( + "Emit transpiled decode progress every N generated tokens. " + "Use 1 for per-token visibility on long CPU-bound Gemma4 runs." + ), + ) + parser.add_argument("--torch-dtype", default="float16") + parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--skip-handwritten", action="store_true") + parser.add_argument("--skip-transpiled", action="store_true") + parser.add_argument("--enable-thinking", action="store_true") + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--top-p", type=float, default=1.0) + parser.add_argument("--top-k", type=int, default=1) + parser.add_argument("--stop-sequence", action="append", default=[]) + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + args = parser.parse_args() + + if args.skip_handwritten and args.skip_transpiled: + raise RuntimeError("cannot skip both handwritten and transpiled runs") + + image_files = tuple(str(Path(path).resolve()) for path in args.image_file if str(path).strip()) + audio_file = str(Path(args.audio_file).resolve()) if args.audio_file.strip() else None + if audio_file and not Path(audio_file).exists(): + raise FileNotFoundError(f"audio file does not exist: {audio_file}") + for image_file in image_files: + if not Path(image_file).exists(): + raise FileNotFoundError(f"image file does not exist: {image_file}") + + torch_dtype = _parse_dtype(args.torch_dtype) + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + + weights_dir_path = None + if args.weights_dir.strip(): + weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) + else: + weights_dir_path = _resolve_default_weights_dir(args.model_id) + + if not args.skip_handwritten and weights_dir_path is None: + raise RuntimeError( + "handwritten Gemma4 comparison requires a converted weights directory.\n" + "\n" + "Create one with something like:\n" + f" cactus convert {args.model_id} /path/to/gemma4_weights\n" + "\n" + "Then rerun with --weights-dir /path/to/gemma4_weights, or rerun with --skip-handwritten." + ) + + stop_sequences = tuple(args.stop_sequence) if args.stop_sequence else _DEFAULT_STOP_SEQUENCES + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + summary: dict[str, Any] = { + "model_id": args.model_id, + "prompt": args.prompt, + "system_prompt": args.system_prompt, + "image_files": list(image_files), + "audio_file": audio_file, + "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, + "warmup": args.warmup, + "repeats": args.repeats, + "max_new_tokens": args.max_new_tokens, + "transpiled_progress_every_tokens": int(args.transpiled_progress_every_tokens), + "torch_dtype": args.torch_dtype, + "trust_remote_code": bool(args.trust_remote_code), + "local_files_only": bool(args.local_files_only), + "enable_thinking": bool(args.enable_thinking), + "temperature": float(args.temperature), + "top_p": float(args.top_p), + "top_k": int(args.top_k), + "stop_sequences": list(stop_sequences), + } + + messages_payload = _build_messages_payload( + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + system_prompt=args.system_prompt, + ) + messages_json = json.dumps(messages_payload) + options_json = json.dumps( + { + "temperature": float(args.temperature), + "top_p": float(args.top_p), + "top_k": int(args.top_k), + "max_tokens": int(args.max_new_tokens), + "stop_sequences": list(stop_sequences), + "enable_thinking_if_supported": bool(args.enable_thinking), + "auto_handoff": False, + "telemetry_enabled": False, + } + ) + + transpiled_runs: list[TranspiledRun] = [] + if not args.skip_transpiled: + model_source, processor_or_tokenizer, model, _ = _load_transformers_bundle( + model_id=args.model_id, + task="multimodal_causal_lm_logits", + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + processor = processor_or_tokenizer + tokenizer = _require_processor_tokenizer(processor_or_tokenizer) + initial_prepared, _, initial_prepare_ms = _prepare_repeat_inputs( + processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + ) + canonical = canonicalize_model_interface( + model, + task="multimodal_causal_lm_logits", + input_names=initial_prepared.names, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + ) + if hasattr(canonical.module, "last_token_logits_only"): + canonical.module.last_token_logits_only = True + prime_static_features = getattr(canonical.module, "prime_static_multimodal_features", None) + if callable(prime_static_features): + prime_static_features(*initial_prepared.tensors) + wrapper = TranspileWrapper( + canonical.module, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + ).eval() + + compile_start = time.perf_counter() + print("transpile_capture_begin=true", flush=True) + prepare_cpu_float32_capture = getattr(canonical.module, "prepare_cpu_float32_capture", None) + restore_cpu_float32_capture = getattr(canonical.module, "restore_cpu_float32_capture", None) + if callable(prepare_cpu_float32_capture): + prepare_cpu_float32_capture() + try: + captured = capture_model(wrapper, initial_prepared.tensors) + finally: + if callable(restore_cpu_float32_capture): + restore_cpu_float32_capture() + print("transpile_capture_done=true", flush=True) + raw_ir_graph = copy.deepcopy(captured.ir_graph) + print("transpile_canonicalize_begin=true", flush=True) + canonicalize_exported_graph(captured.ir_graph) + print("transpile_canonicalize_done=true", flush=True) + print("transpile_optimize_begin=true", flush=True) + optimize_graph(captured.ir_graph, config=fusion_config) + print("transpile_optimize_done=true", flush=True) + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + print("transpile_lower_begin=true", flush=True) + tg = _lower_preoptimized_ir(captured.ir_graph) + print("transpile_lower_done=true", flush=True) + compile_time_ms = (time.perf_counter() - compile_start) * 1000.0 + + op_counts: dict[str, int] = {} + for node_id in optimized_ir_graph.order: + op = optimized_ir_graph.nodes[node_id].op + op_counts[op] = op_counts.get(op, 0) + 1 + weight_bindings = sum( + 1 + for value in optimized_ir_graph.values.values() + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) + ) + + summary["transpile_compile"] = { + "model_source": model_source, + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": weight_bindings, + "compile_time_ms": compile_time_ms, + "initial_prepare_ms": initial_prepare_ms, + "ops": op_counts, + } + prompt_token_count = int(initial_prepared.metadata.get("prompt_token_count", 0)) + target_token_count = int(initial_prepared.metadata.get("target_token_count", 0)) + total_transpiled_runs = int(args.warmup) + int(args.repeats) + print( + "transpiled_decode_plan=" + f"prompt_tokens:{prompt_token_count} " + f"target_tokens:{target_token_count} " + f"max_new_tokens:{int(args.max_new_tokens)} " + f"runs:{total_transpiled_runs} " + f"progress_every_tokens:{int(args.transpiled_progress_every_tokens)}", + flush=True, + ) + + if weights_dir_path is not None and weight_bindings == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir_path}\n" + "\n" + "The converted weights folder exists, but none of the captured constants matched weights_manifest.json.\n" + "Re-convert Gemma4 with the current converter before benchmarking." + ) + + if artifact_dir is not None: + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": "multimodal_causal_lm_logits", + "inputs": _serialize_json_compatible(initial_prepared.metadata), + "graph": _graph_to_dict(raw_ir_graph), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": "multimodal_causal_lm_logits", + "inputs": _serialize_json_compatible(initial_prepared.metadata), + "graph": _graph_to_dict(optimized_ir_graph), + }, + ) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + if weight_bindings > 0: + skip_reason = ( + "skipped saved_graph because graph serialization does not yet preserve " + "mmap-bound weights/embeddings required by this transpiled Gemma4 graph" + ) + summary["transpile_compile"]["saved_graph_skipped_reason"] = skip_reason + print(f"note={skip_reason}") + else: + graph_path = artifact_dir / args.graph_filename + tg.graph.save(graph_path) + print(f"saved_graph={graph_path}") + + for _ in range(args.warmup): + warmup_index = _ + 1 + print(f"transpiled_warmup_begin={warmup_index}/{args.warmup}", flush=True) + _run_transpiled_generation_once( + tg=tg, + processor=processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + stop_sequences=stop_sequences, + expected_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + prepared_override=initial_prepared, + progress_label=f"transpiled_warmup{warmup_index}", + progress_every_tokens=args.transpiled_progress_every_tokens, + ) + for _ in range(args.repeats): + repeat_index = _ + 1 + print(f"transpiled_repeat_begin={repeat_index}/{args.repeats}", flush=True) + transpiled_runs.append( + _run_transpiled_generation_once( + tg=tg, + processor=processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + stop_sequences=stop_sequences, + expected_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + prepared_override=initial_prepared, + progress_label=f"transpiled_repeat{repeat_index}", + progress_every_tokens=args.transpiled_progress_every_tokens, + ) + ) + + summary["transpiled"] = _summarize_transpiled_runs( + transpiled_runs, + compile_time_ms=compile_time_ms, + model_source=model_source, + weight_bindings=weight_bindings, + prepared_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + ) + + # Keep the HF/export capture stack alive until the transpiled graph has + # completed execution. Gemma4 teardown before first execute has been + # unstable in practice, even though the lowered graph is already built. + del model + del wrapper + del canonical + del captured + gc.collect() + + handwritten_runs: list[HandwrittenRun] = [] + if not args.skip_handwritten and weights_dir_path is not None: + model_handle = cactus_init(str(weights_dir_path), None, False) + try: + for _ in range(args.warmup): + warmup_index = _ + 1 + print(f"handwritten_warmup_begin={warmup_index}/{args.warmup}", flush=True) + _run_handwritten_once( + model_handle=model_handle, + messages_json=messages_json, + options_json=options_json, + ) + for _ in range(args.repeats): + repeat_index = _ + 1 + print(f"handwritten_repeat_begin={repeat_index}/{args.repeats}", flush=True) + handwritten_runs.append( + _run_handwritten_once( + model_handle=model_handle, + messages_json=messages_json, + options_json=options_json, + ) + ) + finally: + cactus_destroy(model_handle) + summary["handwritten"] = _summarize_handwritten_runs( + handwritten_runs, + weights_dir=weights_dir_path, + ) + + if handwritten_runs and transpiled_runs: + handwritten_latest = handwritten_runs[-1].response.strip() + transpiled_latest = transpiled_runs[-1].response.strip() + summary["comparison"] = { + "end_to_end_speedup_vs_handwritten": ( + summary["handwritten"]["total_time_ms"]["mean"] + / max(summary["transpiled"]["total_ms"]["mean"], 1e-9) + ), + "responses_match": handwritten_latest == transpiled_latest, + "handwritten_latest_response": handwritten_latest, + "transpiled_latest_response": transpiled_latest, + } + + if artifact_dir is not None: + summary_path = artifact_dir / "summary.json" + summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") + print(f"saved_summary={summary_path}") + + _print_summary(summary) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/tests/transpile/tools/compare_parakeet_performance.py b/python/tests/transpile/tools/compare_parakeet_performance.py new file mode 100644 index 000000000..eeaf06066 --- /dev/null +++ b/python/tests/transpile/tools/compare_parakeet_performance.py @@ -0,0 +1,792 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from collections import defaultdict +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from scipy.io import wavfile + +TOOLS_DIR = Path(__file__).resolve().parent +PYTHON_ROOT = TOOLS_DIR.parents[2] +PROJECT_ROOT = PYTHON_ROOT.parent + +sys.path.insert(0, str(PYTHON_ROOT)) + +from cactus.bindings.cactus import cactus_destroy +from cactus.bindings.cactus import cactus_init +from cactus.bindings.cactus import cactus_transcribe +from cactus.transpile.capture_pytorch import capture_model +from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph +from cactus.transpile.model_adapters import canonicalize_model_interface +from cactus.transpile.optimize_graph import FusionConfig +from cactus.transpile.optimize_graph import optimize_graph +from cactus.transpile.hf_model import TranspileWrapper +from cactus.transpile.hf_model import _ctc_greedy_decode_token_ids +from cactus.transpile.hf_model import _decode_token_ids +from cactus.transpile.hf_model import _infer_task_from_config +from cactus.transpile.hf_model import _load_optional_json +from cactus.transpile.hf_model import _load_optional_tokenizer +from cactus.transpile.hf_model import _load_transformers_bundle +from cactus.transpile.hf_model import _lower_preoptimized_ir +from cactus.transpile.hf_model import _parse_dtype +from cactus.transpile.hf_model import _prepare_audio_inputs +from cactus.transpile.hf_model import _validate_weights_dir + + +@dataclass +class NumericSummary: + mean: float + min: float + max: float + median: float + + +@dataclass +class ProfileEntry: + op: str + time_ms: float + shape: str + raw_line: str + + +@dataclass +class ProfileSection: + entries: list[ProfileEntry] + total_ms: float + + +@dataclass +class HandwrittenRun: + wall_ms: float + total_time_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + prefill_tokens: int + total_tokens: int + confidence: float + transcript: str + response_json: dict[str, Any] + + +@dataclass +class TranspiledRun: + preprocess_ms: float + execute_ms: float + decode_ms: float + total_ms: float + graph_only_ms: float + output_frames: int + output_vocab: int + decode_tokens: int + transcript: str | None + input_shapes: dict[str, list[int]] + + +def _numeric_summary(values: list[float]) -> NumericSummary: + arr = np.asarray(values, dtype=np.float64) + return NumericSummary( + mean=float(np.mean(arr)), + min=float(np.min(arr)), + max=float(np.max(arr)), + median=float(np.median(arr)), + ) + + +def _resolve_default_weights_dir(model_id: str) -> Path | None: + candidate = PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower() + if (candidate / "config.txt").exists(): + return candidate + return None + + +def _audio_duration_seconds(audio_file: str) -> float: + sample_rate, samples = wavfile.read(audio_file) + if sample_rate <= 0: + raise ValueError(f"invalid sample rate in {audio_file}: {sample_rate}") + return float(len(samples)) / float(sample_rate) + + +def _profile_env(profile_file: Path | None): + class _ProfileContext: + def __enter__(self_inner): + self_inner.old_profile = os.environ.get("CACTUS_PROFILE") + self_inner.old_profile_file = os.environ.get("CACTUS_PROFILE_FILE") + if profile_file is None: + os.environ.pop("CACTUS_PROFILE", None) + os.environ.pop("CACTUS_PROFILE_FILE", None) + return self_inner + profile_file.parent.mkdir(parents=True, exist_ok=True) + if profile_file.exists(): + profile_file.unlink() + os.environ.pop("CACTUS_PROFILE", None) + os.environ["CACTUS_PROFILE_FILE"] = str(profile_file) + return self_inner + + def __exit__(self_inner, exc_type, exc, tb): + if self_inner.old_profile is None: + os.environ.pop("CACTUS_PROFILE", None) + else: + os.environ["CACTUS_PROFILE"] = self_inner.old_profile + + if self_inner.old_profile_file is None: + os.environ.pop("CACTUS_PROFILE_FILE", None) + else: + os.environ["CACTUS_PROFILE_FILE"] = self_inner.old_profile_file + return False + + return _ProfileContext() + + +def _parse_profile_file(path: Path) -> list[ProfileSection]: + if not path.exists(): + return [] + + lines = path.read_text().splitlines() + sections: list[ProfileSection] = [] + idx = 0 + while idx < len(lines): + if lines[idx].strip() != "=== Graph Execution Profile ===": + idx += 1 + continue + + entries: list[ProfileEntry] = [] + total_ms = 0.0 + idx += 1 + while idx < len(lines): + line = lines[idx].rstrip() + stripped = line.strip() + if stripped.startswith("Total execution time:"): + try: + total_ms = float(stripped.split(":", 1)[1].strip().split()[0]) + except Exception: + total_ms = 0.0 + elif stripped == "================================": + break + elif ( + stripped + and not stripped.startswith("Operation") + and not set(stripped).issubset({"-"}) + and not stripped.startswith("Total execution time:") + ): + parts = stripped.split() + if len(parts) >= 3: + op = parts[0] + try: + time_ms = float(parts[1]) + except ValueError: + time_ms = 0.0 + shape = parts[2] if parts[2].startswith("[") else "" + entries.append(ProfileEntry(op=op, time_ms=time_ms, shape=shape, raw_line=stripped)) + idx += 1 + + sections.append(ProfileSection(entries=entries, total_ms=total_ms)) + idx += 1 + + return sections + + +def _summarize_profile_sections(sections: list[ProfileSection]) -> dict[str, Any]: + op_total_ms: dict[str, float] = defaultdict(float) + op_count: dict[str, int] = defaultdict(int) + all_entries: list[ProfileEntry] = [] + total_ms_values: list[float] = [] + + for section in sections: + total_ms_values.append(section.total_ms) + all_entries.extend(section.entries) + for entry in section.entries: + op_total_ms[entry.op] += entry.time_ms + op_count[entry.op] += 1 + + by_op = [] + for op, total_ms in sorted(op_total_ms.items(), key=lambda item: item[1], reverse=True): + count = op_count[op] + by_op.append( + { + "op": op, + "count": count, + "total_ms": round(total_ms, 6), + "mean_ms": round(total_ms / max(count, 1), 6), + } + ) + + slowest_nodes = [ + { + "op": entry.op, + "time_ms": round(entry.time_ms, 6), + "shape": entry.shape, + "raw_line": entry.raw_line, + } + for entry in sorted(all_entries, key=lambda item: item.time_ms, reverse=True)[:50] + ] + + return { + "runs_profiled": len(sections), + "graph_total_ms": asdict(_numeric_summary(total_ms_values)) if total_ms_values else None, + "entries_per_run": [len(section.entries) for section in sections], + "by_op": by_op, + "slowest_nodes": slowest_nodes, + } + + +def _run_handwritten_once( + *, + model_handle: Any, + audio_file: str, +) -> HandwrittenRun: + options_json = json.dumps( + { + "use_vad": False, + "max_tokens": 4096, + } + ) + start = time.perf_counter() + response = cactus_transcribe( + model_handle, + audio_file, + "", + options_json, + None, + None, + ) + end = time.perf_counter() + + payload = json.loads(response) + return HandwrittenRun( + wall_ms=(end - start) * 1000.0, + total_time_ms=float(payload.get("total_time_ms", 0.0)), + time_to_first_token_ms=float(payload.get("time_to_first_token_ms", 0.0)), + prefill_tps=float(payload.get("prefill_tps", 0.0)), + decode_tps=float(payload.get("decode_tps", 0.0)), + decode_tokens=int(payload.get("decode_tokens", 0)), + prefill_tokens=int(payload.get("prefill_tokens", 0)), + total_tokens=int(payload.get("total_tokens", 0)), + confidence=float(payload.get("confidence", 0.0)), + transcript=str(payload.get("response", "")), + response_json=payload, + ) + + +def _run_transpiled_once( + *, + tg: Any, + processor: object | None, + input_names: tuple[str, ...], + model_config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + torch_dtype: torch.dtype, + audio_file: str, + tokenizer: object | None, + blank_token_id: int | None, +) -> TranspiledRun: + start_pre = time.perf_counter() + prepared = _prepare_audio_inputs( + processor, + input_names=input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task="ctc_logits", + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + end_pre = time.perf_counter() + + input_arrays = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] + + start_exec = time.perf_counter() + tg.set_inputs(input_arrays) + outputs = tg.execute() + end_exec = time.perf_counter() + + start_decode = time.perf_counter() + logits = outputs[0].numpy().astype(np.float32) + token_ids = _ctc_greedy_decode_token_ids(logits, blank_token_id=blank_token_id) + transcript = _decode_token_ids(tokenizer, token_ids) if tokenizer is not None else None + end_decode = time.perf_counter() + + return TranspiledRun( + preprocess_ms=(end_pre - start_pre) * 1000.0, + execute_ms=(end_exec - start_exec) * 1000.0, + decode_ms=(end_decode - start_decode) * 1000.0, + total_ms=(end_decode - start_pre) * 1000.0, + graph_only_ms=(end_exec - start_exec) * 1000.0, + output_frames=int(logits.shape[1]), + output_vocab=int(logits.shape[2]), + decode_tokens=len(token_ids), + transcript=transcript, + input_shapes={ + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + ) + + +def _run_transpiled_graph_only_once( + *, + tg: Any, + cached_inputs: list[np.ndarray], +) -> float: + start = time.perf_counter() + tg.set_inputs(cached_inputs) + tg.execute() + end = time.perf_counter() + return (end - start) * 1000.0 + + +def _warmup(fn, count: int) -> None: + for _ in range(max(count, 0)): + fn() + + +def _roundtrip_jsonable(obj: Any) -> Any: + if isinstance(obj, dict): + return {str(key): _roundtrip_jsonable(value) for key, value in obj.items()} + if isinstance(obj, list): + return [_roundtrip_jsonable(value) for value in obj] + if isinstance(obj, tuple): + return [_roundtrip_jsonable(value) for value in obj] + if isinstance(obj, Path): + return str(obj) + if isinstance(obj, NumericSummary): + return asdict(obj) + if hasattr(obj, "__dataclass_fields__"): + return asdict(obj) + return obj + + +def _build_transpiled_graph( + *, + model_id: str, + audio_file: str, + weights_dir: str | None, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, + fusion_config: FusionConfig, +): + task = _infer_task_from_config(model_id) + if task != "ctc_logits": + raise RuntimeError(f"expected a CTC model, got task={task}") + + model_source, processor, model, model_config = _load_transformers_bundle( + model_id=model_id, + task=task, + torch_dtype=torch_dtype, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) + + preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") + if not preprocessor_config: + preprocessor_config = _load_optional_json(model_id, "preprocessor_config.json") + + tokenizer = _load_optional_tokenizer( + model_id=model_id, + model_source=model_source, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) + + canonical = canonicalize_model_interface(model, task=task) + prepared = _prepare_audio_inputs( + processor, + input_names=canonical.input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + canonical = canonicalize_model_interface(model, task=task, input_names=prepared.names) + + wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + + capture_started = time.perf_counter() + captured = capture_model(wrapper, prepared.tensors) + canonicalize_exported_graph(captured.ir_graph) + optimize_graph(captured.ir_graph, config=fusion_config) + tg = _lower_preoptimized_ir(captured.ir_graph) + capture_finished = time.perf_counter() + + cached_inputs = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] + blank_token_id = getattr(getattr(model, "config", None), "pad_token_id", None) + + return { + "tg": tg, + "model": model, + "processor": processor, + "model_config": model_config, + "preprocessor_config": preprocessor_config, + "tokenizer": tokenizer, + "blank_token_id": int(blank_token_id) if blank_token_id is not None else None, + "compile_time_ms": (capture_finished - capture_started) * 1000.0, + "cached_inputs": cached_inputs, + "input_names": tuple(prepared.names), + "prepared_input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + "model_source": model_source, + "weight_bindings": sum( + 1 + for value in captured.ir_graph.values.values() + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) + ), + } + + +def _benchmark_handwritten( + *, + weights_dir: Path, + audio_file: str, + warmup: int, + repeats: int, + profile_repeats: int, + artifact_dir: Path | None, +) -> dict[str, Any]: + model_handle = cactus_init(str(weights_dir), None, False) + try: + _warmup(lambda: _run_handwritten_once(model_handle=model_handle, audio_file=audio_file), warmup) + + runs: list[HandwrittenRun] = [] + for _ in range(max(repeats, 1)): + runs.append(_run_handwritten_once(model_handle=model_handle, audio_file=audio_file)) + + profile_path = ( + artifact_dir / "handwritten_profile.txt" + if artifact_dir is not None and profile_repeats > 0 + else None + ) + with _profile_env(profile_path): + for _ in range(max(profile_repeats, 0)): + _run_handwritten_once(model_handle=model_handle, audio_file=audio_file) + + profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] + profile_summary = _summarize_profile_sections(profile_sections) + + return { + "weights_dir": str(weights_dir), + "runs": [_roundtrip_jsonable(run) for run in runs], + "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), + "internal_total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "confidence": asdict(_numeric_summary([run.confidence for run in runs])), + "latest_transcript": runs[-1].transcript if runs else "", + "profile_file": str(profile_path) if profile_path is not None else None, + "profile_summary": profile_summary, + } + finally: + cactus_destroy(model_handle) + + +def _benchmark_transpiled( + *, + model_id: str, + audio_file: str, + weights_dir: str | None, + warmup: int, + repeats: int, + profile_repeats: int, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, + artifact_dir: Path | None, + fusion_config: FusionConfig, +) -> dict[str, Any]: + built = _build_transpiled_graph( + model_id=model_id, + audio_file=audio_file, + weights_dir=weights_dir, + torch_dtype=torch_dtype, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + fusion_config=fusion_config, + ) + + tg = built["tg"] + model = built["model"] + processor = built["processor"] + input_names = built["input_names"] + model_config = built["model_config"] + preprocessor_config = built["preprocessor_config"] + tokenizer = built["tokenizer"] + blank_token_id = built["blank_token_id"] + cached_inputs = built["cached_inputs"] + + _warmup( + lambda: _run_transpiled_once( + tg=tg, + processor=processor, + input_names=input_names, + model_config=model_config, + preprocessor_config=preprocessor_config, + model=model, + torch_dtype=torch_dtype, + audio_file=audio_file, + tokenizer=tokenizer, + blank_token_id=blank_token_id, + ), + warmup, + ) + _warmup(lambda: _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs), warmup) + + runs: list[TranspiledRun] = [] + graph_only_ms: list[float] = [] + for _ in range(max(repeats, 1)): + runs.append( + _run_transpiled_once( + tg=tg, + processor=processor, + input_names=input_names, + model_config=model_config, + preprocessor_config=preprocessor_config, + model=model, + torch_dtype=torch_dtype, + audio_file=audio_file, + tokenizer=tokenizer, + blank_token_id=blank_token_id, + ) + ) + graph_only_ms.append(_run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs)) + + profile_path = ( + artifact_dir / "transpiled_profile.txt" + if artifact_dir is not None and profile_repeats > 0 + else None + ) + with _profile_env(profile_path): + for _ in range(max(profile_repeats, 0)): + _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs) + + profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] + profile_summary = _summarize_profile_sections(profile_sections) + + return { + "model_source": built["model_source"], + "compile_time_ms": built["compile_time_ms"], + "weight_bindings": built["weight_bindings"], + "prepared_input_shapes": built["prepared_input_shapes"], + "runs": [_roundtrip_jsonable(run) for run in runs], + "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), + "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), + "decode_ms": asdict(_numeric_summary([run.decode_ms for run in runs])), + "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), + "graph_only_ms": asdict(_numeric_summary(graph_only_ms)), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "output_frames": asdict(_numeric_summary([float(run.output_frames) for run in runs])), + "latest_transcript": runs[-1].transcript if runs else "", + "profile_file": str(profile_path) if profile_path is not None else None, + "profile_summary": profile_summary, + } + + +def _print_summary( + *, + audio_duration_sec: float, + handwritten: dict[str, Any] | None, + transpiled: dict[str, Any] | None, +) -> None: + print() + print(f"audio_duration_sec={audio_duration_sec:.3f}") + + if handwritten is not None: + hw_total = handwritten["internal_total_time_ms"]["mean"] + hw_graph = None + profile_total = handwritten.get("profile_summary", {}).get("graph_total_ms") + if isinstance(profile_total, dict): + hw_graph = profile_total.get("mean") + hw_tok = handwritten["decode_tokens"]["mean"] + hw_rtf = hw_total / max(audio_duration_sec * 1000.0, 1e-9) + hw_xrt = (audio_duration_sec * 1000.0) / max(hw_total, 1e-9) + hw_tok_s = hw_tok * 1000.0 / max(hw_total, 1e-9) + print("handwritten:") + print(f" total_ms mean={hw_total:.3f}") + print(f" graph_profile_ms mean={hw_graph:.3f}" if hw_graph is not None else " graph_profile_ms unavailable") + print(f" decode_tokens mean={hw_tok:.3f}") + print(f" output_tokens_per_sec={hw_tok_s:.3f}") + print(f" real_time_factor={hw_rtf:.3f}") + print(f" audio_x_realtime={hw_xrt:.3f}") + + if transpiled is not None: + tr_total = transpiled["total_ms"]["mean"] + tr_graph = transpiled["graph_only_ms"]["mean"] + tr_tok = transpiled["decode_tokens"]["mean"] + tr_rtf = tr_total / max(audio_duration_sec * 1000.0, 1e-9) + tr_xrt = (audio_duration_sec * 1000.0) / max(tr_total, 1e-9) + tr_tok_s = tr_tok * 1000.0 / max(tr_total, 1e-9) + tr_graph_tok_s = tr_tok * 1000.0 / max(tr_graph, 1e-9) + print("transpiled:") + print(f" total_ms mean={tr_total:.3f}") + print(f" graph_only_ms mean={tr_graph:.3f}") + print(f" decode_tokens mean={tr_tok:.3f}") + print(f" output_tokens_per_sec={tr_tok_s:.3f}") + print(f" graph_only_tokens_per_sec={tr_graph_tok_s:.3f}") + print(f" real_time_factor={tr_rtf:.3f}") + print(f" audio_x_realtime={tr_xrt:.3f}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Compare handwritten Cactus Parakeet performance against the transpiled " + "Parakeet graph on the same WAV input, including per-op graph profiles." + ) + ) + parser.add_argument("--model-id", default="nvidia/parakeet-ctc-1.1b") + parser.add_argument("--audio-file", required=True) + parser.add_argument( + "--weights-dir", + default="", + help="Converted Cactus weights directory. Reused for handwritten runtime and transpiled mmap bindings.", + ) + parser.add_argument("--artifact-dir", default="", help="Optional directory for JSON summaries and raw profiles.") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--profile-repeats", type=int, default=1) + parser.add_argument("--torch-dtype", default="float16") + parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--skip-handwritten", action="store_true") + parser.add_argument("--skip-transpiled", action="store_true") + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + args = parser.parse_args() + + audio_file = str(Path(args.audio_file).resolve()) + if not Path(audio_file).exists(): + raise FileNotFoundError(f"audio file does not exist: {audio_file}") + + torch_dtype = _parse_dtype(args.torch_dtype) + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + + weights_dir_path = None + if args.weights_dir.strip(): + weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) + else: + weights_dir_path = _resolve_default_weights_dir(args.model_id) + + if not args.skip_handwritten and weights_dir_path is None: + raise RuntimeError( + "handwritten comparison requires a converted weights directory.\n" + "\n" + f"Create one with:\n" + f" cactus convert {args.model_id} {PROJECT_ROOT / 'weights' / args.model_id.split('/')[-1].lower()}\n" + "\n" + "Or rerun with --skip-handwritten." + ) + + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + summary: dict[str, Any] = { + "model_id": args.model_id, + "audio_file": audio_file, + "audio_duration_sec": _audio_duration_seconds(audio_file), + "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, + "warmup": args.warmup, + "repeats": args.repeats, + "profile_repeats": args.profile_repeats, + "torch_dtype": args.torch_dtype, + "trust_remote_code": bool(args.trust_remote_code), + "local_files_only": bool(args.local_files_only), + } + + handwritten_summary = None + if not args.skip_handwritten and weights_dir_path is not None: + handwritten_summary = _benchmark_handwritten( + weights_dir=weights_dir_path, + audio_file=audio_file, + warmup=args.warmup, + repeats=args.repeats, + profile_repeats=args.profile_repeats, + artifact_dir=artifact_dir, + ) + summary["handwritten"] = handwritten_summary + + transpiled_summary = None + if not args.skip_transpiled: + transpiled_summary = _benchmark_transpiled( + model_id=args.model_id, + audio_file=audio_file, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + warmup=args.warmup, + repeats=args.repeats, + profile_repeats=args.profile_repeats, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + artifact_dir=artifact_dir, + fusion_config=fusion_config, + ) + summary["transpiled"] = transpiled_summary + + if handwritten_summary is not None and transpiled_summary is not None: + hw_total = handwritten_summary["internal_total_time_ms"]["mean"] + tr_total = transpiled_summary["total_ms"]["mean"] + hw_graph = handwritten_summary.get("profile_summary", {}).get("graph_total_ms") + tr_graph = transpiled_summary["graph_only_ms"]["mean"] + summary["comparison"] = { + "end_to_end_speedup_vs_handwritten": hw_total / max(tr_total, 1e-9), + "graph_only_speedup_vs_handwritten": ( + float(hw_graph["mean"]) / max(tr_graph, 1e-9) + if isinstance(hw_graph, dict) and "mean" in hw_graph + else None + ), + } + + if artifact_dir is not None: + summary_path = artifact_dir / "summary.json" + summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") + print(f"saved_summary={summary_path}") + if handwritten_summary is not None and handwritten_summary.get("profile_file"): + print(f"saved_handwritten_profile={handwritten_summary['profile_file']}") + if transpiled_summary is not None and transpiled_summary.get("profile_file"): + print(f"saved_transpiled_profile={transpiled_summary['profile_file']}") + + _print_summary( + audio_duration_sec=float(summary["audio_duration_sec"]), + handwritten=handwritten_summary, + transpiled=transpiled_summary, + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/tests/transpile/tools/inspect_gemma4_cleanup.py b/python/tests/transpile/tools/inspect_gemma4_cleanup.py new file mode 100644 index 000000000..835311f2e --- /dev/null +++ b/python/tests/transpile/tools/inspect_gemma4_cleanup.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +import torch +from transformers import AutoModelForCausalLM + +PYTHON_ROOT = Path(__file__).resolve().parents[3] +if str(PYTHON_ROOT) not in sys.path: + sys.path.insert(0, str(PYTHON_ROOT)) + +from cactus.transpile.capture_pytorch import capture_model +from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.model_adapters import canonicalize_model_interface + + +class Gemma4FullModelWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = canonicalize_model_interface(model, task="causal_lm_logits").module + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model(input_ids) + + +class Gemma4FirstBlockCheckpointWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module, checkpoint_name: str): + super().__init__() + adapter = canonicalize_model_interface(model, task="causal_lm_logits") + if adapter.family != "gemma4": + raise ValueError(f"expected gemma4 adapter, got family={adapter.family}") + if not hasattr(adapter.module, "debug_first_block"): + raise ValueError("Gemma4 adapter does not expose debug_first_block()") + self.adapter = adapter.module + self.checkpoint_name = checkpoint_name + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + checkpoints = self.adapter.debug_first_block(input_ids) + if self.checkpoint_name not in checkpoints: + available = ", ".join(sorted(checkpoints.keys())) + raise KeyError(f"unknown checkpoint {self.checkpoint_name!r}; available: {available}") + return checkpoints[self.checkpoint_name] + + +def format_ir_graph(graph: IRGraph, *, max_nodes: int | None = None) -> str: + lines: list[str] = [] + lines.append("IR Summary") + lines.append(f" inputs={graph.inputs}") + lines.append(f" outputs={graph.outputs}") + lines.append(f" nodes={len(graph.order)}") + if graph.meta: + lines.append(f" meta={graph.meta}") + lines.append("") + lines.append("IR Nodes") + + node_ids = graph.order if max_nodes is None else graph.order[:max_nodes] + for index, node_id in enumerate(node_ids): + node = graph.nodes[node_id] + lines.append(f"[{index}] {node.id} op={node.op}") + lines.append(f" inputs={node.inputs}") + lines.append(f" outputs={node.outputs}") + if node.attrs: + lines.append(f" attrs={node.attrs}") + if node.kind != "generic": + lines.append(f" kind={node.kind}") + if node.meta: + lines.append(f" meta={node.meta}") + for output_id in node.outputs: + value = graph.values.get(output_id) + if value is None: + continue + lines.append( + f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" + ) + + if max_nodes is not None and len(graph.order) > max_nodes: + lines.append(f"... ({len(graph.order) - max_nodes} more nodes omitted)") + + if graph.constants: + lines.append("") + lines.append("IR Constants") + constant_ids = sorted(graph.constants.keys()) + preview_ids = constant_ids if max_nodes is None else constant_ids[: max(8, min(len(constant_ids), max_nodes))] + for value_id in preview_ids: + value = graph.constants[value_id] + value_meta = graph.values.get(value_id) + shape = None if value_meta is None else value_meta.shape + dtype = None if value_meta is None else value_meta.dtype + lines.append(f"- {value_id}: type={type(value).__name__} shape={shape} dtype={dtype}") + if len(preview_ids) < len(constant_ids): + lines.append(f"... ({len(constant_ids) - len(preview_ids)} more constants omitted)") + + return "\n".join(lines) + + +def format_unsupported_ops(graph: IRGraph) -> str: + counts = graph.meta.get("canonical_unsupported_op_counts", {}) + if not isinstance(counts, dict): + counts = {} + + examples: dict[str, list[str]] = {} + for node_id in graph.order: + node = graph.nodes[node_id] + print(f"node {node_id} op={node.op}") + if node.op not in counts: + continue + bucket = examples.setdefault(node.op, []) + if len(bucket) < 5: + bucket.append(node.id) + + total = sum(int(count) for count in counts.values()) + lines = ["Unsupported Ops", f" unique={len(counts)}", f" total_nodes={total}"] + if not counts: + lines.append(" none") + return "\n".join(lines) + + for op_name, count in counts.items(): + sample_nodes = ", ".join(examples.get(op_name, ())) + lines.append(f"- {op_name}: {count}") + if sample_nodes: + lines.append(f" examples={sample_nodes}") + return "\n".join(lines) + + +def resolve_local_model_path(model_id: str) -> str: + model_path = Path(model_id) + if model_path.exists(): + return str(model_path) + + cache_root = Path.home() / ".cache" / "huggingface" / "hub" / ("models--" + model_id.replace("/", "--")) + snapshots_dir = cache_root / "snapshots" + if not snapshots_dir.exists(): + raise FileNotFoundError( + f"could not find local snapshot for {model_id!r} under {snapshots_dir}" + ) + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + raise FileNotFoundError(f"no snapshots found for {model_id!r} under {snapshots_dir}") + return str(snapshots[-1]) + + +def load_model(model_id: str) -> torch.nn.Module: + token = os.environ.get("HF_TOKEN") + common_kwargs: dict[str, object] = { + "local_files_only": True, + } + if token: + common_kwargs["token"] = token + + local_model_path = resolve_local_model_path(model_id) + model = AutoModelForCausalLM.from_pretrained( + local_model_path, + torch_dtype=torch.float16, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model + + +def build_module( + model: torch.nn.Module, + *, + mode: str, + checkpoint: str, +) -> torch.nn.Module: + if mode == "full": + return Gemma4FullModelWrapper(model).eval() + if mode == "first_block": + return Gemma4FirstBlockCheckpointWrapper(model, checkpoint).eval() + raise ValueError(f"unknown mode: {mode}") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Capture Gemma4, run canonicalize/cleanup.py, and report unsupported ops.", + ) + parser.add_argument( + "--model-id", + default=os.environ.get("CACTUS_GEMMA_HF_MODEL_ID", "google/gemma-4-E2B"), + help="Hugging Face model id. Uses local cache only.", + ) + parser.add_argument( + "--mode", + choices=("first_block", "full"), + default="first_block", + help="Capture a first-block checkpoint or the full Gemma4 adapter graph.", + ) + parser.add_argument( + "--checkpoint", + default="after_ffn_residual", + help="First-block checkpoint name when --mode=first_block.", + ) + parser.add_argument( + "--input-ids", + default="2,818,5279,529,7001,563", + help="Comma-separated integer token ids to capture.", + ) + parser.add_argument( + "--max-nodes", + type=int, + default=120, + help="Maximum IR nodes to print when --print-ir is enabled.", + ) + parser.add_argument( + "--print-ir", + action="store_true", + help="Also print the IR before and after canonical cleanup.", + ) + args = parser.parse_args() + + model = load_model(args.model_id) + module = build_module(model, mode=args.mode, checkpoint=args.checkpoint) + token_ids = [int(token.strip()) for token in args.input_ids.split(",") if token.strip()] + input_ids = torch.tensor([token_ids], dtype=torch.long) + + print(f"Model: {args.model_id}") + print(f"Mode: {args.mode}") + print(f"Checkpoint: {args.checkpoint if args.mode == 'first_block' else '(full model)'}") + print(f"Input IDs shape: {tuple(input_ids.shape)}") + print("") + + captured = capture_model(module, (input_ids,)) + + if args.print_ir: + print("=== IR Before Canonical Cleanup ===") + print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) + print("") + + canonicalize_exported_graph(captured.ir_graph) + + print("=== Unsupported Ops After Canonical Cleanup ===") + print(format_unsupported_ops(captured.ir_graph)) + + if args.print_ir: + print("") + print("=== IR After Canonical Cleanup ===") + print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) + + +if __name__ == "__main__": + main() diff --git a/python/tests/transpile/tools/inspect_transpile_cleanup.py b/python/tests/transpile/tools/inspect_transpile_cleanup.py new file mode 100644 index 000000000..b5b2f3d40 --- /dev/null +++ b/python/tests/transpile/tools/inspect_transpile_cleanup.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import torch +import torch.nn as nn + +PYTHON_ROOT = Path(__file__).resolve().parents[3] +if str(PYTHON_ROOT) not in sys.path: + sys.path.insert(0, str(PYTHON_ROOT)) + +from cactus.transpile.capture_pytorch import capture_model +from cactus.transpile.capture_pytorch import dump_graph +from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph +from cactus.transpile.graph_ir import IRGraph + + +class Toy(nn.Module): + def forward(self, x, y): + z = x + y + z = z * 0.5 + z = torch.nn.functional.gelu(z) + z = torch.softmax(z, dim=-1) + return z + + +class AttentionBlockToy(nn.Module): + def __init__(self, hidden_size: int = 8): + super().__init__() + self.out_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=torch.float16) + + def forward(self, q, k, v, gate): + attn = torch.nn.functional.scaled_dot_product_attention(q, k, v, is_causal=True, scale=1.0) + attn = attn.transpose(1, 2).reshape(q.shape[0], q.shape[2], -1) + attn = attn * torch.sigmoid(gate) + attn = attn.to(self.out_proj.weight.dtype) + return self.out_proj(attn) + + +def build_case(name: str) -> tuple[nn.Module, tuple[torch.Tensor, ...]]: + if name == "toy": + model = Toy().eval() + args = ( + torch.randn(2, 4, dtype=torch.float16), + torch.randn(2, 4, dtype=torch.float16), + ) + return model, args + + if name == "attention_block": + model = AttentionBlockToy().eval() + args = ( + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 3, 8, dtype=torch.float16), + ) + return model, args + + raise ValueError(f"unknown case: {name}") + + +def format_ir_graph(graph: IRGraph) -> str: + lines: list[str] = [] + lines.append("IR Summary") + lines.append(f" inputs={graph.inputs}") + lines.append(f" outputs={graph.outputs}") + lines.append(f" nodes={len(graph.order)}") + if graph.meta: + lines.append(f" meta={graph.meta}") + lines.append("") + lines.append("IR Nodes") + + for index, node_id in enumerate(graph.order): + node = graph.nodes[node_id] + lines.append(f"[{index}] {node.id} op={node.op}") + lines.append(f" inputs={node.inputs}") + lines.append(f" outputs={node.outputs}") + if node.attrs: + lines.append(f" attrs={node.attrs}") + if node.kind != "generic": + lines.append(f" kind={node.kind}") + if node.meta: + lines.append(f" meta={node.meta}") + for output_id in node.outputs: + value = graph.values.get(output_id) + if value is None: + continue + lines.append( + f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" + ) + + if graph.constants: + lines.append("") + lines.append("IR Constants") + for value_id, value in graph.constants.items(): + value_meta = graph.values.get(value_id) + shape = None if value_meta is None else value_meta.shape + dtype = None if value_meta is None else value_meta.dtype + typename = type(value).__name__ + lines.append(f"- {value_id}: type={typename} shape={shape} dtype={dtype}") + + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Capture a small PyTorch model, dump the exported graph, and print IR before/after cleanup.", + ) + parser.add_argument( + "--case", + choices=("toy", "attention_block"), + default="toy", + help="Which built-in model to inspect.", + ) + parser.add_argument( + "--no-meta", + action="store_true", + help="Hide exported graph metadata to keep the dump shorter.", + ) + args = parser.parse_args() + + model, example_args = build_case(args.case) + captured = capture_model(model, example_args) + + print(f"Case: {args.case}") + print(f"Model: {type(model).__name__}") + print("") + print("=== Exported Graph ===") + print(dump_graph(captured, include_meta=not args.no_meta)) + print("") + print("=== IR Before Cleanup ===") + print(format_ir_graph(captured.ir_graph)) + print("") + + canonicalize_exported_graph(captured.ir_graph) + + print("=== IR After Cleanup ===") + print(format_ir_graph(captured.ir_graph)) + + +if __name__ == "__main__": + main() diff --git a/python/tests/transpile/tools/torch_capture_smoke.py b/python/tests/transpile/tools/torch_capture_smoke.py new file mode 100644 index 000000000..f3be0f51e --- /dev/null +++ b/python/tests/transpile/tools/torch_capture_smoke.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import torch +import torch.nn as nn + +PYTHON_ROOT = Path(__file__).resolve().parents[3] +if str(PYTHON_ROOT) not in sys.path: + sys.path.insert(0, str(PYTHON_ROOT)) + +from cactus.transpile.capture_pytorch import ( + capture_model_with_fallback, + dump_graph, + get_dtype, + get_shape, +) +from cactus.transpile.lower import transpile_captured + + +class TinyBlock(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(8, 16) + self.fc2 = nn.Linear(16, 4) + + def forward(self, x): + h = self.fc1(x) + h = torch.relu(h) + y = self.fc2(h) + return y + + +def main(): + model = TinyBlock().eval() + x = torch.randn(2, 8) + + captured = capture_model_with_fallback(model, args=(x,)) + + print("strict:", captured.strict) + print("graph_module type:", type(captured.graph_module).__name__) + print("state_dict keys:", list(captured.state_dict.keys())) + print() + + print("=== FX Graph ===") + print(captured.graph) + print() + + print("=== Node Walk ===") + for i, node in enumerate(captured.graph.nodes): + print(f"[{i}] op={node.op} name={node.name} target={node.target}") + print(f" args={node.args}") + print(f" kwargs={node.kwargs}") + print(f" shape={get_shape(node)} dtype={get_dtype(node)}") + print() + + print("=== Full Dump ===") + print(dump_graph(captured)) + print() + + print("=== Lower To Cactus ===") + try: + transpiled = transpile_captured(captured) + print("lowering succeeded") + print("lowered graph type:", type(transpiled.graph).__name__) + print("runtime inputs:", transpiled.runtime_inputs) + print("bound constants:", transpiled.bound_constants) + print("outputs:", transpiled.outputs) + print() + print("binding runtime input 0 from example tensor") + transpiled.set_input(0, x) + except NotImplementedError as exc: + print("lowering stopped on unimplemented case:") + print(f" {exc}") + + +if __name__ == "__main__": + main() diff --git a/python/tests/transpile/tools/transpile_hf_causal_lm.py b/python/tests/transpile/tools/transpile_hf_causal_lm.py new file mode 100644 index 000000000..a1982228e --- /dev/null +++ b/python/tests/transpile/tools/transpile_hf_causal_lm.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +PYTHON_ROOT = Path(__file__).resolve().parents[3] +if str(PYTHON_ROOT) not in sys.path: + sys.path.insert(0, str(PYTHON_ROOT)) + +from cactus.transpile.hf_model import main as transpile_main + + +def main() -> int: + if "--task" not in sys.argv[1:]: + sys.argv[1:1] = ["--task", "causal_lm_logits"] + return transpile_main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup b/setup index 4539a168c..0cd57c34b 100755 --- a/setup +++ b/setup @@ -82,6 +82,15 @@ else echo -e "${YELLOW}Warning: requirements.txt not found${NC}" fi +echo "" +echo -e "${BLUE}Step 3b: Upgrading Transformers...${NC}" +if python3 -m pip install --upgrade "transformers==5.5.4" -q; then + echo -e "${GREEN}✓ Transformers upgraded to 5.5.4${NC}" +else + echo -e "${RED}Failed to upgrade Transformers${NC}" + return 1 +fi + if [ -f "$PARENT_REQUIREMENTS" ] && [ "$PARENT_REQUIREMENTS" != "$REQUIREMENTS_FILE" ]; then echo "" echo -e "${BLUE}Detected parent repo requirements at: $PARENT_REQUIREMENTS${NC}"