From 77f808ed0a05f9cb49a86d66a52969903a9337c9 Mon Sep 17 00:00:00 2001 From: justinl66 Date: Wed, 20 May 2026 17:12:51 -0700 Subject: [PATCH 1/8] audio fix --- cactus-engine/src/engine.h | 7 ++ cactus-engine/src/model.cpp | 191 ++++++++++++++++++++++++++++++++---- 2 files changed, 180 insertions(+), 18 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 6cce3a484..bb19781c7 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -688,6 +688,11 @@ class Model { bool run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, const std::vector& audio_features); + bool build_lm_encoder_outputs_dynamic_gemma4( + const std::vector& tokens, + std::map>& store_bytes, + std::map& store_prec, + std::map>& store_shape); std::string bundle_dir_; std::map components_; @@ -698,6 +703,8 @@ class Model { Component* lm_encoder_media_step_ = nullptr; Component* decoder_prefill_chunk_ = nullptr; Component* lm_encoder_ = nullptr; + Component* lm_encoder_text_chunk_ = nullptr; + Component* lm_encoder_media_chunk_ = nullptr; std::string family_; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 4efc957f2..016f98fc5 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -163,11 +163,15 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, lm_encoder_media_step_ = components_.count("lm_encoder_media_step") ? &components_.at("lm_encoder_media_step") : nullptr; decoder_prefill_chunk_ = components_.count("decoder_prefill_chunk") ? &components_.at("decoder_prefill_chunk") : nullptr; lm_encoder_ = components_.count("lm_encoder") ? &components_.at("lm_encoder") : nullptr; + lm_encoder_text_chunk_ = components_.count("lm_encoder_text_chunk") ? &components_.at("lm_encoder_text_chunk") : nullptr; + lm_encoder_media_chunk_ = components_.count("lm_encoder_media_chunk") ? &components_.at("lm_encoder_media_chunk") : nullptr; if (vision_encoder_ && !bind_runtime_buffers(*vision_encoder_)) return false; if (audio_encoder_ && !bind_runtime_buffers(*audio_encoder_)) return false; if (lm_encoder_media_step_ && !bind_runtime_buffers(*lm_encoder_media_step_)) return false; if (decoder_prefill_chunk_ && !bind_runtime_buffers(*decoder_prefill_chunk_)) return false; if (lm_encoder_ && !bind_runtime_buffers(*lm_encoder_)) return false; + if (lm_encoder_text_chunk_ && !bind_runtime_buffers(*lm_encoder_text_chunk_)) return false; + if (lm_encoder_media_chunk_ && !bind_runtime_buffers(*lm_encoder_media_chunk_)) return false; if (vision_encoder_ && tokenizer_ && !vision_encoder_->output_node_ids.empty()) { size_t out_node = static_cast(vision_encoder_->output_node_ids[0]); @@ -471,7 +475,23 @@ void Model::run_audio_encoder(const std::vector& audio_features) { int mask_idx = input_index(*audio_encoder_, "input_features_mask"); if (mask_idx >= 0) { auto& mb = audio_encoder_->input_buffers[mask_idx]; - std::fill(mb.begin(), mb.end(), static_cast(1)); + size_t mask_node = static_cast(audio_encoder_->runtime_input_node_ids[mask_idx]); + const auto& mask_desc = audio_encoder_->graph->get_output_buffer(mask_node); + size_t elem = PrecisionTraits::size_of(mask_desc.precision); + size_t total = elem ? mb.size() / elem : mb.size(); + std::memset(mb.data(), 0, mb.size()); + for (size_t i = 0; i < total; ++i) { + switch (mask_desc.precision) { + case Precision::FP32: reinterpret_cast(mb.data())[i] = 1.0f; break; + case Precision::FP16: reinterpret_cast<__fp16*>(mb.data())[i] = static_cast<__fp16>(1.0f); break; + case Precision::INT8: reinterpret_cast(mb.data())[i] = 1; break; + default: + if (elem == 8) reinterpret_cast(mb.data())[i] = 1; + else if (elem == 4) reinterpret_cast(mb.data())[i] = 1; + else mb[i] = 1; + break; + } + } } audio_encoder_->graph->execute(); for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { @@ -626,6 +646,133 @@ void Model::copy_cache_state(const Component& src, Component& dst) { } } +bool Model::build_lm_encoder_outputs_dynamic_gemma4( + const std::vector& tokens, + std::map>& store_bytes, + std::map& store_prec, + std::map>& store_shape) { + if (!encoder_ || !lm_encoder_media_step_) return false; + if (tokens.empty()) return false; + + const uint32_t image_tok = config_.image_token_id; + const uint32_t audio_tok = config_.audio_token_id; + + auto af_it = media_features_.find("audio_features"); + const bool have_af = (af_it != media_features_.end() && !af_it->second.empty()); + size_t af_count = 0; + size_t af_row_bytes = 0; + Precision af_prec = Precision::FP16; + if (have_af) { + const auto& sh = media_feature_shapes_["audio_features"]; + af_count = sh.size() >= 2 ? sh[sh.size() - 2] : 0; + af_prec = media_feature_precisions_["audio_features"]; + af_row_bytes = af_count > 0 ? af_it->second.size() / af_count : af_it->second.size(); + } + + auto im_it = media_features_.find("image_features"); + const bool have_imf = (im_it != media_features_.end() && !im_it->second.empty()); + size_t imf_count = 0; + size_t imf_row_bytes = 0; + Precision imf_prec = Precision::FP16; + if (have_imf) { + const auto& sh = media_feature_shapes_["image_features"]; + imf_count = sh.size() >= 2 ? sh[sh.size() - 2] : 0; + imf_prec = media_feature_precisions_["image_features"]; + imf_row_bytes = imf_count > 0 ? im_it->second.size() / imf_count : im_it->second.size(); + } + + struct OutputInfo { + std::string name; + int text_idx = -1; + int media_idx = -1; + size_t per_token_bytes = 0; + Precision precision = Precision::FP16; + std::vector shape_template; + }; + std::vector outs; + for (size_t i = 0; i < encoder_->logical_outputs.size() && i < encoder_->output_node_ids.size(); ++i) { + OutputInfo oi; + oi.name = encoder_->logical_outputs[i]; + oi.text_idx = static_cast(i); + size_t node = static_cast(encoder_->output_node_ids[i]); + const auto& desc = encoder_->graph->get_output_buffer(node); + oi.per_token_bytes = desc.byte_size; + oi.precision = desc.precision; + oi.shape_template = desc.shape; + oi.media_idx = output_index(*lm_encoder_media_step_, oi.name); + if (oi.media_idx < 0) { + CACTUS_LOG_WARN("model", "dynamic walker: lm_encoder_media_step missing output '" << oi.name << "'"); + return false; + } + outs.push_back(std::move(oi)); + } + + const size_t N = tokens.size(); + for (auto& oi : outs) { + store_bytes[oi.name].assign(N * oi.per_token_bytes, 0); + store_prec[oi.name] = oi.precision; + std::vector sh = oi.shape_template; + if (sh.size() >= 2 && sh[sh.size() - 2] == 1) { + sh[sh.size() - 2] = N; + } else if (sh.size() == 1) { + sh[0] = N; + } + store_shape[oi.name] = sh; + } + + size_t audio_idx = 0; + size_t image_idx = 0; + for (size_t pos = 0; pos < N; ++pos) { + const uint32_t tk = tokens[pos]; + Component* enc; + const uint8_t* media_row = nullptr; + size_t media_row_bytes = 0; + Precision media_prec = Precision::FP16; + if (have_af && audio_tok != 0 && tk == audio_tok && audio_idx < af_count) { + enc = lm_encoder_media_step_; + media_row = af_it->second.data() + audio_idx * af_row_bytes; + media_row_bytes = af_row_bytes; + media_prec = af_prec; + ++audio_idx; + } else if (have_imf && image_tok != 0 && tk == image_tok && image_idx < imf_count) { + enc = lm_encoder_media_step_; + media_row = im_it->second.data() + image_idx * imf_row_bytes; + media_row_bytes = imf_row_bytes; + media_prec = imf_prec; + ++image_idx; + } else { + enc = encoder_; + } + + if (enc == lm_encoder_media_step_) { + int e_idx = input_index(*enc, "inputs_embeds"); + if (e_idx >= 0) { + auto& buf = enc->input_buffers[e_idx]; + size_t node_id = static_cast(enc->runtime_input_node_ids[e_idx]); + const auto& desc = enc->graph->get_output_buffer(node_id); + write_typed_buffer(buf, desc.precision, media_row, media_row_bytes, media_prec); + } + write_int_input(*enc, "input_ids", 0); + write_int_input(*enc, "position_ids", static_cast(pos)); + } else { + write_int_input(*enc, "input_ids", static_cast(tk)); + write_int_input(*enc, "position_ids", static_cast(pos)); + } + enc->graph->execute(); + + for (auto& oi : outs) { + int out_i = (enc == encoder_) ? oi.text_idx : oi.media_idx; + if (out_i < 0) continue; + size_t node = static_cast(enc->output_node_ids[out_i]); + const auto& desc = enc->graph->get_output_buffer(node); + if (desc.byte_size != oi.per_token_bytes) continue; + const void* ptr = enc->graph->get_output(node); + std::memcpy(store_bytes[oi.name].data() + pos * oi.per_token_bytes, ptr, oi.per_token_bytes); + } + } + return true; +} + bool Model::run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, const std::vector& audio_features) { @@ -720,7 +867,18 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, run_audio_encoder(audio_features); } - { + std::map> store_bytes; + std::map store_prec; + std::map> store_shape; + + const bool needs_dynamic_walk = (family_ == "gemma4") && have_audio + && encoder_ != nullptr && lm_encoder_media_step_ != nullptr; + + if (needs_dynamic_walk) { + if (!build_lm_encoder_outputs_dynamic_gemma4(tokens, store_bytes, store_prec, store_shape)) { + return false; + } + } else { int ids_idx = input_index(*lm_encoder_, "input_ids"); if (ids_idx >= 0) { auto& ids_buf = lm_encoder_->input_buffers[ids_idx]; @@ -748,23 +906,20 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, write_typed_buffer(dst_buf, desc.precision, kv.second.data(), kv.second.size(), src_prec); } - } - lm_encoder_->graph->execute(); + lm_encoder_->graph->execute(); - std::map> store_bytes; - std::map store_prec; - std::map> store_shape; - for (size_t i = 0; i < lm_encoder_->output_node_ids.size() - && i < lm_encoder_->logical_outputs.size(); ++i) { - const std::string& name = lm_encoder_->logical_outputs[i]; - size_t node_id = static_cast(lm_encoder_->output_node_ids[i]); - const auto& desc = lm_encoder_->graph->get_output_buffer(node_id); - void* ptr = lm_encoder_->graph->get_output(node_id); - auto& slot = store_bytes[name]; - slot.assign(desc.byte_size, 0); - std::memcpy(slot.data(), ptr, desc.byte_size); - store_prec[name] = desc.precision; - store_shape[name] = desc.shape; + for (size_t i = 0; i < lm_encoder_->output_node_ids.size() + && i < lm_encoder_->logical_outputs.size(); ++i) { + const std::string& name = lm_encoder_->logical_outputs[i]; + size_t node_id = static_cast(lm_encoder_->output_node_ids[i]); + const auto& desc = lm_encoder_->graph->get_output_buffer(node_id); + void* ptr = lm_encoder_->graph->get_output(node_id); + auto& slot = store_bytes[name]; + slot.assign(desc.byte_size, 0); + std::memcpy(slot.data(), ptr, desc.byte_size); + store_prec[name] = desc.precision; + store_shape[name] = desc.shape; + } } auto embeds_shape_it = store_shape.find("inputs_embeds"); From 96dd82395b65f9424de158c961f1ebc73cb5b8b1 Mon Sep 17 00:00:00 2001 From: justinl66 Date: Thu, 21 May 2026 08:56:45 -0700 Subject: [PATCH 2/8] Merge v2-chunked-prefill into NPU branch --- .gitignore | 8 + android/build.sh | 9 +- cactus-engine/CMakeLists.txt | 1 + cactus-engine/src/complete.cpp | 110 +- cactus-engine/src/engine.h | 63 +- cactus-engine/src/engine_image.cpp | 215 +++- cactus-engine/src/model.cpp | 1033 ++++++++++++--- cactus-engine/src/transcribe.cpp | 2 +- cactus-engine/src/utils.h | 21 +- cactus-engine/tests/asr.cpp | 2 +- cactus-engine/tests/test_stt.cpp | 4 +- cactus-graph/cactus_graph.h | 5 + cactus-graph/src/core.cpp | 32 + cactus-graph/src/execute.cpp | 71 +- cactus-graph/src/ops_cache.cpp | 133 +- cactus-graph/src/ops_math.cpp | 194 ++- cactus-graph/src/ops_tensor.cpp | 7 + cactus-graph/tests/test_cache.cpp | 27 + cactus-graph/tests/test_io.cpp | 8 +- cactus-graph/tests/test_utils.h | 2 +- cactus-kernels/src/threading.h | 9 +- cactus/build.sh | 11 +- local_integration_tests/CMakeLists.txt | 62 + local_integration_tests/README.md | 43 + local_integration_tests/conftest.py | 156 +++ local_integration_tests/native_runner.cpp | 1110 +++++++++++++++++ local_integration_tests/test_modalities.py | 210 ++++ .../gemma4_multimodal/BACKGROUND.md | 53 + .../gemma4_multimodal/EXPERIMENTS.md | 80 ++ .../gemma4_multimodal/FIXES.md | 52 + .../gemma4_multimodal/HYPOTHESES.md | 71 ++ .../gemma4_multimodal/PLAN.md | 29 + .../gemma4_multimodal/README.md | 23 + .../gemma4_multimodal/SUMMARY.md | 9 + .../gemma4_multimodal_v2/00_INDEX.md | 48 + .../gemma4_multimodal_v2/01_BACKGROUND.md | 48 + .../gemma4_multimodal_v2/02_QUESTIONS.md | 41 + .../gemma4_multimodal_v2/03_HYPOTHESES.md | 66 + .../gemma4_multimodal_v2/04_EXPERIMENTS.md | 356 ++++++ .../gemma4_multimodal_v2/05_DECISIONS.md | 118 ++ .../gemma4_multimodal_v2/06_FIX_LOG.md | 156 +++ .../gemma4_multimodal_v2/07_ACCEPTANCE.md | 50 + .../gemma4_multimodal_v2/08_HANDOFF.md | 44 + .../09_STATUS_FOR_NEXT_AGENT.md | 74 ++ .../10_CONTINUATION_PLAN.md | 58 + .../gemma4_multimodal_v2/scripts/run_probe.py | 785 ++++++++++++ python/cactus/cli/__init__.py | 6 + python/cactus/cli/compile.py | 8 + python/cactus/cli/convert.py | 6 + .../convert/cactus_adapters/tensor_io.py | 3 +- .../cactus/convert/model_adapters/adapters.py | 2 +- .../cactus/convert/model_adapters/naming.py | 1 - .../cactus/convert/model_adapters/policy.py | 12 +- python/cactus/convert/tests/test_policy.py | 8 +- python/cactus/transpile/hf_model.py | 80 +- python/cactus/transpile/lower.py | 93 +- python/cactus/transpile/model_adapters.py | 361 +++++- python/cactus/transpile/model_profiles.py | 19 +- python/cactus/transpile/runtime_compat.py | 22 +- python/tests/test_cli_transpile_defaults.py | 25 +- 60 files changed, 5893 insertions(+), 462 deletions(-) create mode 100644 local_integration_tests/CMakeLists.txt create mode 100644 local_integration_tests/README.md create mode 100644 local_integration_tests/conftest.py create mode 100644 local_integration_tests/native_runner.cpp create mode 100644 local_integration_tests/test_modalities.py create mode 100644 local_investigations/gemma4_multimodal/BACKGROUND.md create mode 100644 local_investigations/gemma4_multimodal/EXPERIMENTS.md create mode 100644 local_investigations/gemma4_multimodal/FIXES.md create mode 100644 local_investigations/gemma4_multimodal/HYPOTHESES.md create mode 100644 local_investigations/gemma4_multimodal/PLAN.md create mode 100644 local_investigations/gemma4_multimodal/README.md create mode 100644 local_investigations/gemma4_multimodal/SUMMARY.md create mode 100644 local_investigations/gemma4_multimodal_v2/00_INDEX.md create mode 100644 local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md create mode 100644 local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md create mode 100644 local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md create mode 100644 local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md create mode 100644 local_investigations/gemma4_multimodal_v2/05_DECISIONS.md create mode 100644 local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md create mode 100644 local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md create mode 100644 local_investigations/gemma4_multimodal_v2/08_HANDOFF.md create mode 100644 local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md create mode 100644 local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md create mode 100644 local_investigations/gemma4_multimodal_v2/scripts/run_probe.py diff --git a/.gitignore b/.gitignore index 99271876e..7795b9ebe 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,8 @@ bindings/flutter/*.xcframework .DS_Store __pycache__ +__pycache__/ +*.pyc *.egg-info venv @@ -45,3 +47,9 @@ vision_activations.json gemma-4-eap-extras/ .githooks/ ruy + +# Local integration/investigation outputs +local_integration_tests/.tmp/ +local_integration_tests/__pycache__/ +local_investigations/**/__pycache__/ +local_investigations/**/artifacts/ diff --git a/android/build.sh b/android/build.sh index 9d6f2d1e2..370a3fd83 100755 --- a/android/build.sh +++ b/android/build.sh @@ -12,9 +12,16 @@ CACTUS_CURL_ROOT=${CACTUS_CURL_ROOT:-"$PROJECT_ROOT/cactus-engine/libs/curl"} if [ -z "$ANDROID_NDK_HOME" ]; then if [ -n "$ANDROID_HOME" ]; then ANDROID_NDK_HOME=$(ls -d "$ANDROID_HOME/ndk/"* 2>/dev/null | sort -V | tail -1) - elif [ -d "$HOME/Library/Android/sdk" ]; then + fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "$HOME/Library/Android/sdk" ]; then ANDROID_NDK_HOME=$(ls -d "$HOME/Library/Android/sdk/ndk/"* 2>/dev/null | sort -V | tail -1) fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/Caskroom/android-ndk" ]; then + ANDROID_NDK_HOME=$(ls -d /opt/homebrew/Caskroom/android-ndk/*/AndroidNDK*.app/Contents/NDK 2>/dev/null | sort -V | tail -1) + fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/share/android-ndk" ]; then + ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" + fi fi if [ -z "$ANDROID_NDK_HOME" ] || [ ! -d "$ANDROID_NDK_HOME" ]; then diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index b683acf66..c3cbd7f17 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -74,6 +74,7 @@ set(ENGINE_SOURCES src/sp.cpp src/constraints.cpp src/model.cpp + src/model_npu.cpp src/engine_image.cpp src/index.cpp src/index_ffi.cpp diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index d355eb586..555a40276 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -325,7 +325,7 @@ struct PreparedPrompt { size_t context_token_count = 0; std::vector> images; - std::vector audio_features; + std::vector> audio_features; size_t audio_num_frames = 0; bool has_images() const { @@ -334,7 +334,8 @@ struct PreparedPrompt { } bool has_audio() const { - return !audio_features.empty(); + return std::any_of(audio_features.begin(), audio_features.end(), + [](const auto& mel) { return !mel.empty(); }); } }; @@ -457,38 +458,36 @@ PreparedPrompt prepare_prompt( } if (prompt.model_type == Config::ModelType::GEMMA4) { - std::vector audio_samples; - if (pcm_buffer != nullptr && pcm_buffer_size > 1) { - auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); - audio_samples = resample_to_16k_fp32(waveform_fp32, 16000); - } else if (!prompt.audio_paths.empty()) { - for (auto it = prompt.messages.rbegin(); it != prompt.messages.rend(); ++it) { - if (!it->audio.empty()) { - const std::string& audio_path = it->audio.back(); - AudioFP32 wav = load_wav(audio_path); - audio_samples = resample_to_16k_fp32(wav.samples, wav.sample_rate); - break; - } - } - } std::vector user_indices; for (size_t i = 0; i < prompt.messages.size(); i++) { if (prompt.messages[i].role == "user") user_indices.push_back(i); } auto& counts = handle->user_audio_counts; - for (size_t u = 0; u + 1 < user_indices.size(); u++) { - if (u < counts.size() && counts[u] > 0) { - prompt.messages[user_indices[u]].audio_soft_token_count = counts[u]; + if (counts.size() < user_indices.size()) counts.resize(user_indices.size(), 0); + + if (pcm_buffer != nullptr && pcm_buffer_size > 1 && !user_indices.empty()) { + auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); + auto samples_16k = resample_to_16k_fp32(waveform_fp32, 16000); + if (!samples_16k.empty()) { + auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(samples_16k, handle->model->get_config()); + prompt.audio_features.push_back(std::move(audio_prep.features)); + size_t u = user_indices.size() - 1; + prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; + counts[u] = audio_prep.num_soft_tokens; + } + } else if (!prompt.audio_paths.empty()) { + for (size_t u = 0; u < user_indices.size(); u++) { + const auto& msg = prompt.messages[user_indices[u]]; + if (msg.audio.empty()) continue; + const std::string& audio_path = msg.audio.back(); + AudioFP32 wav = load_wav(audio_path); + auto samples_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); + if (samples_16k.empty()) continue; + auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(samples_16k, handle->model->get_config()); + prompt.audio_features.push_back(std::move(audio_prep.features)); + prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; + counts[u] = audio_prep.num_soft_tokens; } - } - if (!audio_samples.empty() && !user_indices.empty()) { - auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(audio_samples, handle->model->get_config()); - prompt.audio_features = std::move(audio_prep.features); - prompt.audio_num_frames = audio_prep.num_frames; - size_t u = user_indices.size() - 1; - prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; - if (counts.size() <= u) counts.resize(u + 1, 0); - counts[u] = audio_prep.num_soft_tokens; } } @@ -542,6 +541,21 @@ PrefillResult do_prefill( if (tokens_to_process.size() > 1) { std::vector prefill_tokens(tokens_to_process.begin(), tokens_to_process.end() - 1); result.prefilled_count = prefill_tokens.size(); + + auto slice_delta_audio = [&]() -> std::vector> { + if (!result.was_prefix) return prompt.audio_features; + const size_t cached_msg_count = handle->processed_images.size(); + size_t cached_audio_count = 0; + for (size_t i = 0; i < cached_msg_count && i < prompt.messages.size(); ++i) { + const auto& msg = prompt.messages[i]; + if (msg.role == "user" && !msg.audio.empty()) cached_audio_count++; + } + cached_audio_count = std::min(cached_audio_count, prompt.audio_features.size()); + return std::vector>( + prompt.audio_features.begin() + cached_audio_count, + prompt.audio_features.end()); + }; + if (has_images && has_audio) { std::vector delta_image_paths; if (result.was_prefix) { @@ -556,7 +570,7 @@ PrefillResult do_prefill( } else { delta_image_paths = prompt.image_paths; } - handle->model->prefill_with_media(prefill_tokens, delta_image_paths, prompt.audio_features); + handle->model->prefill_with_media(prefill_tokens, delta_image_paths, slice_delta_audio()); } else if (has_images) { std::vector delta_image_paths; if (result.was_prefix) { @@ -573,7 +587,7 @@ PrefillResult do_prefill( } handle->model->prefill_with_images(prefill_tokens, delta_image_paths); } else if (has_audio) { - handle->model->prefill_with_audio(prefill_tokens, prompt.audio_features); + handle->model->prefill_with_audio(prefill_tokens, slice_delta_audio()); } else { handle->model->prefill(prefill_tokens, handle->model->get_prefill_chunk_size()); } @@ -676,14 +690,6 @@ int cactus_complete( bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); - const bool has_gemma4_mixed_media = prompt.model_type == Config::ModelType::GEMMA4 && has_images && has_audio; - auto decode_gemma4_mixed_media = [&](const std::vector& tokens, float* out_entropy) -> uint32_t { - (void)tokens; - (void)out_entropy; - throw std::runtime_error( - "Gemma4 mixed-media native decode is not present in this build. " - "Use cactus run-transpiled for transpiled graph bundles."); - }; auto stop_token_sequences = build_stop_sequences(tokenizer, prompt.options.stop_sequences, prompt.model_type, !prompt.tools.empty()); @@ -693,30 +699,21 @@ int cactus_complete( uint32_t next_token; size_t prompt_tokens; - if ((has_gemma4_mixed_media || has_audio) && !handle->processed_tokens.empty()) { + if (has_audio && !handle->processed_tokens.empty()) { auto& cache = handle->processed_tokens; size_t common = 0; size_t limit = std::min(cache.size(), prompt.tokens.size()); while (common < limit && cache[common] == prompt.tokens[common]) common++; if (common < cache.size()) { CACTUS_LOG_WARN("complete", "KV cache diverges from new prompt at position " << common - << "/" << cache.size() << "; trimming and re-prefilling the divergent suffix"); - size_t kv_len = handle->model->get_cache_size(); - if (kv_len > common) { - handle->model->remove_thinking_tokens({{common, kv_len - common}}); - } - cache.resize(common); + << "/" << cache.size() << "; resetting cache for clean audio re-prefill"); + reset_cache(handle); } } - if (has_gemma4_mixed_media) { - prompt_tokens = prompt.tokens.size(); - next_token = decode_gemma4_mixed_media(prompt.tokens, &first_token_entropy); - } else { - auto prefill_result = do_prefill(handle, prompt, prompt.tokens); - prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); - next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); - } + auto prefill_result = do_prefill(handle, prompt, prompt.tokens); + prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); + next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); handle->processed_tokens = prompt.tokens; handle->processed_images = prompt.images; @@ -784,11 +781,10 @@ int cactus_complete( if (handle->should_stop) break; float token_entropy = 0.0f; - if (has_gemma4_mixed_media) { - next_token = decode_gemma4_mixed_media(handle->processed_tokens, &token_entropy); - } else if (has_audio) { + if (has_audio) { + uint32_t last_token = handle->processed_tokens.empty() ? next_token : handle->processed_tokens.back(); next_token = handle->model->decode_with_audio( - handle->processed_tokens, prompt.audio_features, + {last_token}, prompt.audio_features, prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, "", &token_entropy, prompt.options.min_p, prompt.options.repetition_penalty); diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index bb19781c7..2e7e769d5 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -580,17 +580,19 @@ class Model { size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f); - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + void prefill(const std::vector& tokens, size_t chunk_size = 128, const std::string& profile_file = "", + bool prepare_decode = true); void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, const std::string& profile_file = ""); - void prefill_with_audio(const std::vector& tokens, const std::vector& audio_features, + void prefill_with_audio(const std::vector& tokens, + const std::vector>& audio_features_per_message, const std::string& profile_file = ""); void prefill_with_media(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features, + const std::vector>& audio_features_per_message, const std::string& profile_file = ""); uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, @@ -598,7 +600,8 @@ class Model { size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f); - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, + uint32_t decode_with_audio(const std::vector& tokens, + const std::vector>& audio_features_per_message, float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f, @@ -628,8 +631,8 @@ class Model { size_t get_cache_size() const { return cache_total_seq_len_; } bool load_npu_prefill(const std::string& model_path); - bool has_npu_prefill() const { return false; } - size_t get_prefill_chunk_size() const { return 256; } + bool has_npu_prefill() const { return npu_prefill_ != nullptr; } + size_t get_prefill_chunk_size() const { return 128; } void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} @@ -649,7 +652,7 @@ class Model { std::string path; }; - struct CacheStateEntry { + struct CacheStateBinding { std::string layer_key; int key_node_id = -1; int value_node_id = -1; @@ -663,31 +666,45 @@ class Model { std::vector output_node_ids; std::vector logical_outputs; std::vector bindings; + std::vector cache_states; std::unique_ptr graph; std::vector> input_buffers; - std::vector cache_states; }; - void copy_cache_state(const Component& src, Component& dst); - bool load_manifest(); bool setup_tokenizer(); - bool load_components(); + bool load_components(const std::unordered_set& required_components); + bool load_component_graph(Component& comp); + void unload_component_graph(Component& comp); bool bind_runtime_buffers(Component& comp); void run_step(uint32_t token_id, size_t position, bool read_logits); + void run_encoder_step(uint32_t token_id, size_t position); void run_media_step(size_t position, const uint8_t* feature_row, size_t feature_row_bytes, Precision feature_precision); + void copy_component_outputs_to_inputs(const Component& source, Component& target); void copy_encoder_outputs_to_decoder(const Component& enc); + void copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index); + void copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset); + bool cache_states_compatible(const Component& source, const Component& target) const; + void copy_cache_states(const Component& source, Component& target); + void reset_component_cache_states(Component& comp); + size_t component_chunk_tokens(const Component& comp, const std::string& input_name) const; + size_t component_output_tokens(const Component& comp, const std::string& output_name) const; + size_t run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, + bool prepare_decode); + void run_full_context_text(); void write_int_input(Component& comp, const std::string& name, int64_t value); + void write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value); void write_bytes_input(Component& comp, const std::string& name, const void* data, size_t byte_size); int input_index(const Component& comp, const std::string& name) const; int output_index(const Component& comp, const std::string& name) const; uint32_t argmax_last_logits(); void run_vision_encoder(const std::string& image_path); void run_audio_encoder(const std::vector& audio_features); + void run_audio_encoder_messages(const std::vector>& audio_features_per_message); bool run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features); + const std::vector>& audio_features_per_message); bool build_lm_encoder_outputs_dynamic_gemma4( const std::vector& tokens, std::map>& store_bytes, @@ -698,6 +715,10 @@ class Model { std::map components_; Component* encoder_ = nullptr; Component* decoder_ = nullptr; + Component* decoder_prefill_ = nullptr; + Component* prefill_encoder_ = nullptr; + enum class DecodeRoute { CACHED_STEP, DIRECT_DECODER_STEP, FULL_CONTEXT_TEXT }; + DecodeRoute decode_route_ = DecodeRoute::CACHED_STEP; Component* vision_encoder_ = nullptr; Component* audio_encoder_ = nullptr; Component* lm_encoder_media_step_ = nullptr; @@ -707,6 +728,12 @@ class Model { Component* lm_encoder_media_chunk_ = nullptr; std::string family_; + std::string npu_prefill_mlpackage_; + + std::unique_ptr npu_prefill_; + + bool prefill_via_npu(const std::vector& tokens); + void write_npu_kv_to_cache(const npu::NPUPrefillDirectResult& result, size_t prefill_len); std::map> media_features_; std::map> media_feature_shapes_; @@ -717,6 +744,8 @@ class Model { bool initialized_ = false; size_t cache_total_seq_len_ = 0; size_t cache_max_seq_len_ = 4096; + size_t last_logit_position_ = 0; + std::vector context_tokens_; static constexpr size_t MAX_TOKEN_HISTORY = 128; std::vector token_history_; @@ -864,6 +893,16 @@ struct Lfm2VlImagePreprocessed { Lfm2VlImagePreprocessed preprocess_lfm2_vl_image(const std::string& image_path, const Config& config); +struct Qwen3VlImagePreprocessed { + std::vector pixel_values; + size_t grid_t = 1; + size_t grid_h = 0; + size_t grid_w = 0; + size_t patch_dim = 0; +}; + +Qwen3VlImagePreprocessed preprocess_qwen3_vl_image(const std::string& image_path, const Config& config); + struct SpectrogramConfig { size_t n_fft = 400; diff --git a/cactus-engine/src/engine_image.cpp b/cactus-engine/src/engine_image.cpp index bd9ee452d..f73c2429a 100644 --- a/cactus-engine/src/engine_image.cpp +++ b/cactus-engine/src/engine_image.cpp @@ -19,6 +19,115 @@ namespace cactus { namespace engine { +namespace { + +constexpr int kPillowResizePrecisionBits = 22; + +static unsigned char pillow_clip8(int64_t value) { + int shifted = static_cast(value >> kPillowResizePrecisionBits); + if (shifted < 0) return 0; + if (shifted > 255) return 255; + return static_cast(shifted); +} + +static void precompute_pillow_bilinear_coeffs(int in_size, int out_size, + std::vector& bounds, + std::vector& coeffs, + int& ksize) { + const double scale = static_cast(in_size) / static_cast(out_size); + const double filterscale = std::max(scale, 1.0); + const double support = filterscale; + const double inv_filterscale = 1.0 / filterscale; + ksize = static_cast(std::ceil(support)) * 2 + 1; + bounds.assign(static_cast(out_size) * 2, 0); + coeffs.assign(static_cast(out_size) * ksize, 0); + + for (int out = 0; out < out_size; ++out) { + const double center = (static_cast(out) + 0.5) * scale; + int xmin = static_cast(center - support + 0.5); + if (xmin < 0) xmin = 0; + int xmax = static_cast(center + support + 0.5); + if (xmax > in_size) xmax = in_size; + xmax -= xmin; + + bounds[static_cast(out) * 2] = xmin; + bounds[static_cast(out) * 2 + 1] = xmax; + + std::vector weights(static_cast(ksize), 0.0); + double total = 0.0; + for (int i = 0; i < xmax; ++i) { + double x = (static_cast(i + xmin) - center + 0.5) * inv_filterscale; + if (x < 0.0) x = -x; + double weight = x < 1.0 ? 1.0 - x : 0.0; + weights[static_cast(i)] = weight; + total += weight; + } + for (int i = 0; i < ksize; ++i) { + double weight = total != 0.0 ? weights[static_cast(i)] / total : 0.0; + coeffs[static_cast(out) * ksize + i] = static_cast( + weight < 0.0 + ? -0.5 + weight * static_cast(1 << kPillowResizePrecisionBits) + : 0.5 + weight * static_cast(1 << kPillowResizePrecisionBits)); + } + } +} + +static std::vector resize_rgb_uint8_pillow_bilinear(const unsigned char* src, + int src_w, + int src_h, + int dst_w, + int dst_h) { + std::vector x_bounds; + std::vector y_bounds; + std::vector x_coeffs; + std::vector y_coeffs; + int x_ksize = 0; + int y_ksize = 0; + precompute_pillow_bilinear_coeffs(src_w, dst_w, x_bounds, x_coeffs, x_ksize); + precompute_pillow_bilinear_coeffs(src_h, dst_h, y_bounds, y_coeffs, y_ksize); + + const int y_first = y_bounds[0]; + const int y_last = y_bounds[static_cast(dst_h - 1) * 2] + + y_bounds[static_cast(dst_h - 1) * 2 + 1]; + const int temp_h = y_last - y_first; + std::vector temp(static_cast(dst_w) * temp_h * 3); + + for (int ty = 0; ty < temp_h; ++ty) { + const int sy = y_first + ty; + for (int dx = 0; dx < dst_w; ++dx) { + const int xmin = x_bounds[static_cast(dx) * 2]; + const int count = x_bounds[static_cast(dx) * 2 + 1]; + const int32_t* k = x_coeffs.data() + static_cast(dx) * x_ksize; + for (int c = 0; c < 3; ++c) { + int64_t sum = static_cast(1) << (kPillowResizePrecisionBits - 1); + for (int x = 0; x < count; ++x) { + sum += static_cast(src[(static_cast(sy) * src_w + xmin + x) * 3 + c]) * k[x]; + } + temp[(static_cast(ty) * dst_w + dx) * 3 + c] = pillow_clip8(sum); + } + } + } + + std::vector dst(static_cast(dst_w) * dst_h * 3); + for (int dy = 0; dy < dst_h; ++dy) { + const int ymin = y_bounds[static_cast(dy) * 2] - y_first; + const int count = y_bounds[static_cast(dy) * 2 + 1]; + const int32_t* k = y_coeffs.data() + static_cast(dy) * y_ksize; + for (int dx = 0; dx < dst_w; ++dx) { + for (int c = 0; c < 3; ++c) { + int64_t sum = static_cast(1) << (kPillowResizePrecisionBits - 1); + for (int y = 0; y < count; ++y) { + sum += static_cast(temp[(static_cast(ymin + y) * dst_w + dx) * 3 + c]) * k[y]; + } + dst[(static_cast(dy) * dst_w + dx) * 3 + c] = pillow_clip8(sum); + } + } + } + return dst; +} + +} // namespace + Siglip2Preprocessor::PreprocessedImage::~PreprocessedImage() { pixel_values.clear(); pixel_attention_mask.clear(); @@ -634,16 +743,9 @@ Gemma4ImagePreprocessed preprocess_gemma4_image(const std::string& image_path, c resized[i] = static_cast(raw[i]); } } else { - std::vector src_float(static_cast(width) * height * 3); - for (size_t i = 0; i < src_float.size(); ++i) { - src_float[i] = static_cast(raw[i]); - } - if (!stbir_resize_float_linear( - src_float.data(), width, height, 0, - resized.data(), target_w, target_h, 0, - STBIR_RGB)) { - stbi_image_free(raw); - throw std::runtime_error("Failed to resize image: " + image_path); + std::vector resized_u8 = resize_rgb_uint8_pillow_bilinear(raw, width, height, target_w, target_h); + for (size_t i = 0; i < resized.size(); ++i) { + resized[i] = static_cast(resized_u8[i]); } } stbi_image_free(raw); @@ -700,6 +802,99 @@ Gemma4ImagePreprocessed preprocess_gemma4_image(const std::string& image_path, c return result; } +Qwen3VlImagePreprocessed preprocess_qwen3_vl_image(const std::string& image_path, const Config& config) { + Qwen3VlImagePreprocessed result; + const int patch_size = static_cast(config.vision_patch_size ? config.vision_patch_size : 16); + const int temporal_patch_size = 2; + const int merge_size = 2; + const size_t image_tokens = config.image_seq_len ? static_cast(config.image_seq_len) : 64; + const size_t merged_patches = image_tokens * static_cast(merge_size * merge_size); + const int grid_side = static_cast(std::sqrt(static_cast(merged_patches))); + if (patch_size <= 0 || grid_side <= 0 || static_cast(grid_side * grid_side) != merged_patches) { + throw std::runtime_error("Qwen3-VL native image preprocessing requires a square static image grid"); + } + + int width = 0, height = 0, channels = 0; + unsigned char* raw = stbi_load(image_path.c_str(), &width, &height, &channels, 3); + if (!raw) { + throw std::runtime_error("Failed to load image: " + image_path + " - " + std::string(stbi_failure_reason())); + } + if (width <= 0 || height <= 0) { + stbi_image_free(raw); + throw std::runtime_error("Loaded image has invalid dimensions: " + image_path); + } + + const int target_h = grid_side * patch_size; + const int target_w = grid_side * patch_size; + std::vector resized(static_cast(target_w) * target_h * 3); + if (target_w == width && target_h == height) { + for (size_t i = 0; i < resized.size(); ++i) { + resized[i] = static_cast(raw[i]); + } + } else { + std::vector src_float(static_cast(width) * height * 3); + for (size_t i = 0; i < src_float.size(); ++i) { + src_float[i] = static_cast(raw[i]); + } + if (!stbir_resize_float_linear( + src_float.data(), width, height, 0, + resized.data(), target_w, target_h, 0, + STBIR_RGB)) { + stbi_image_free(raw); + throw std::runtime_error("Failed to resize image: " + image_path); + } + } + stbi_image_free(raw); + + const float rescale_factor = config.rescale_factor > 0.0f ? config.rescale_factor : (1.0f / 255.0f); + const float mean_v = config.image_mean; + const float std_v = config.image_std != 0.0f ? config.image_std : 1.0f; + for (float& v : resized) { + v = (v * rescale_factor - mean_v) / std_v; + } + + const size_t channel = 3; + const size_t grid_t = 1; + const size_t grid_h = static_cast(grid_side); + const size_t grid_w = static_cast(grid_side); + const size_t patch_dim = channel * static_cast(temporal_patch_size) * patch_size * patch_size; + result.pixel_values.assign(grid_t * grid_h * grid_w * patch_dim, 0.0f); + + for (size_t gh_major = 0; gh_major < grid_h / merge_size; ++gh_major) { + for (size_t gw_major = 0; gw_major < grid_w / merge_size; ++gw_major) { + for (size_t mh = 0; mh < static_cast(merge_size); ++mh) { + for (size_t mw = 0; mw < static_cast(merge_size); ++mw) { + const size_t patch_y = gh_major * merge_size + mh; + const size_t patch_x = gw_major * merge_size + mw; + const size_t patch_index = + (((gh_major * (grid_w / merge_size) + gw_major) * merge_size + mh) * merge_size + mw); + float* dst = result.pixel_values.data() + patch_index * patch_dim; + size_t dst_i = 0; + for (size_t c = 0; c < channel; ++c) { + for (size_t t = 0; t < static_cast(temporal_patch_size); ++t) { + (void)t; + for (int y = 0; y < patch_size; ++y) { + const size_t img_y = patch_y * patch_size + static_cast(y); + for (int x = 0; x < patch_size; ++x) { + const size_t img_x = patch_x * patch_size + static_cast(x); + const size_t src_off = (img_y * static_cast(target_w) + img_x) * channel + c; + dst[dst_i++] = resized[src_off]; + } + } + } + } + } + } + } + } + + result.grid_t = grid_t; + result.grid_h = grid_h; + result.grid_w = grid_w; + result.patch_dim = patch_dim; + return result; +} + Lfm2VlImagePreprocessed preprocess_lfm2_vl_image(const std::string& image_path, const Config& config) { Siglip2Preprocessor::Config sp_cfg; sp_cfg.patch_size = config.vision_patch_size ? static_cast(config.vision_patch_size) : 16; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 016f98fc5..e31440a09 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -19,10 +19,70 @@ #include #include #include +#include namespace cactus { namespace engine { +float read_scalar_value(Precision precision, const uint8_t* data, size_t index) { + const uint8_t* ptr = data + PrecisionTraits::byte_offset_of(precision, index); + switch (precision) { + case Precision::FP32: + return *reinterpret_cast(ptr); + case Precision::FP16: + return static_cast(*reinterpret_cast(ptr)); + case Precision::INT8: + return static_cast(*reinterpret_cast(ptr)); + default: + return 0.0f; + } +} + +void write_scalar_value(Precision precision, uint8_t* data, size_t index, float value) { + uint8_t* ptr = data + PrecisionTraits::byte_offset_of(precision, index); + switch (precision) { + case Precision::FP32: + *reinterpret_cast(ptr) = value; + break; + case Precision::FP16: + *reinterpret_cast<__fp16*>(ptr) = static_cast<__fp16>(value); + break; + case Precision::INT8: + *reinterpret_cast(ptr) = static_cast(value); + break; + default: + break; + } +} + +bool copy_component_tensor(CactusGraph& source_graph, + const BufferDesc& src_desc, + size_t src_node, + const BufferDesc& dst_desc, + std::vector& dst_buffer, + size_t dst_element_offset, + size_t element_count, + const std::string& name) { + const auto* src_ptr = static_cast(source_graph.get_output(src_node)); + if (src_desc.precision == dst_desc.precision) { + size_t dst_offset = PrecisionTraits::byte_offset_of(dst_desc.precision, dst_element_offset); + std::memcpy( + dst_buffer.data() + dst_offset, + src_ptr, + PrecisionTraits::packed_size_of(src_desc.precision, element_count)); + return true; + } + if (name != "position_ids" && name != "attention_mask") return false; + for (size_t i = 0; i < element_count; ++i) { + write_scalar_value( + dst_desc.precision, + dst_buffer.data(), + dst_element_offset + i, + read_scalar_value(src_desc.precision, src_ptr, i)); + } + return true; +} + void ConvCache::init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision) { num_layers = layers; hidden_size = hidden_dim; @@ -144,34 +204,99 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_ERROR("model", "Tokenizer init failed for bundle: " << bundle_dir); return false; } - if (!load_components()) return false; - - encoder_ = components_.count("lm_encoder_step") ? &components_.at("lm_encoder_step") : nullptr; - decoder_ = components_.count("decoder_step") ? &components_.at("decoder_step") : nullptr; - const bool is_lm = (encoder_ != nullptr && decoder_ != nullptr); - const bool is_transcription = components_.count("audio_encoder") && - (components_.count("decoder") || components_.count("decoder_joint")); - if (!is_lm && !is_transcription) { - CACTUS_LOG_ERROR("model", "Bundle missing required components: need lm_encoder_step+decoder_step (LM) or audio_encoder+decoder (transcription)"); + std::string encoder_name; + std::string decoder_name; + std::unordered_set required_components; + bool has_chunked_prefill = components_.count("lm_encoder_step") + && components_.count("decoder_media_step") + && components_.count("lm_encoder_text_chunk") + && components_.count("decoder_prefill_chunk"); + if (has_chunked_prefill) { + encoder_name = "lm_encoder_step"; + decoder_name = "decoder_media_step"; + decode_route_ = DecodeRoute::CACHED_STEP; + required_components = { + encoder_name, + decoder_name, + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + }; + } else if (components_.count("decoder_step") + && input_index(components_.at("decoder_step"), "input_ids") >= 0 + && input_index(components_.at("decoder_step"), "position_ids") >= 0) { + decoder_name = "decoder_step"; + decode_route_ = DecodeRoute::DIRECT_DECODER_STEP; + required_components = {decoder_name}; + } else if (components_.count("lm_encoder_step") && components_.count("decoder_step")) { + encoder_name = "lm_encoder_step"; + decoder_name = "decoder_step"; + decode_route_ = DecodeRoute::CACHED_STEP; + required_components = {encoder_name, decoder_name}; + if (components_.count("decoder_prefill_chunk")) { + required_components.insert("decoder_prefill_chunk"); + } + if (components_.count("lm_encoder_text_chunk")) { + required_components.insert("lm_encoder_text_chunk"); + } + } else if (components_.count("text_lm_encoder") && components_.count("decoder")) { + encoder_name = "text_lm_encoder"; + decoder_name = "decoder"; + decode_route_ = DecodeRoute::FULL_CONTEXT_TEXT; + required_components = {encoder_name, decoder_name}; + } else if (components_.count("audio_encoder") && + (components_.count("decoder") || components_.count("decoder_joint"))) { + decoder_name = components_.count("decoder") ? "decoder" : "decoder_joint"; + decode_route_ = DecodeRoute::DIRECT_DECODER_STEP; + required_components = {"audio_encoder", decoder_name}; + } else { + CACTUS_LOG_ERROR("model", "Bundle missing lm_encoder_step+decoder_step or text_lm_encoder+decoder components"); return false; } - if (encoder_ && !bind_runtime_buffers(*encoder_)) return false; - if (decoder_ && !bind_runtime_buffers(*decoder_)) return false; - + for (const auto& optional : { + "vision_encoder", + "audio_encoder", + "lm_encoder_media_step", + "decoder_prefill_chunk", + "lm_encoder", + }) { + if (components_.count(optional)) { + required_components.insert(optional); + } + } + if (!load_components(required_components)) return false; + if (!encoder_name.empty()) encoder_ = &components_.at(encoder_name); + if (!decoder_name.empty()) decoder_ = &components_.at(decoder_name); + if (components_.count("decoder_prefill_chunk") && components_.at("decoder_prefill_chunk").graph) { + decoder_prefill_ = &components_.at("decoder_prefill_chunk"); + decoder_prefill_chunk_ = decoder_prefill_; + } + if (components_.count("lm_encoder_text_chunk") && components_.at("lm_encoder_text_chunk").graph) { + prefill_encoder_ = &components_.at("lm_encoder_text_chunk"); + } vision_encoder_ = components_.count("vision_encoder") ? &components_.at("vision_encoder") : nullptr; audio_encoder_ = components_.count("audio_encoder") ? &components_.at("audio_encoder") : nullptr; lm_encoder_media_step_ = components_.count("lm_encoder_media_step") ? &components_.at("lm_encoder_media_step") : nullptr; - decoder_prefill_chunk_ = components_.count("decoder_prefill_chunk") ? &components_.at("decoder_prefill_chunk") : nullptr; lm_encoder_ = components_.count("lm_encoder") ? &components_.at("lm_encoder") : nullptr; lm_encoder_text_chunk_ = components_.count("lm_encoder_text_chunk") ? &components_.at("lm_encoder_text_chunk") : nullptr; lm_encoder_media_chunk_ = components_.count("lm_encoder_media_chunk") ? &components_.at("lm_encoder_media_chunk") : nullptr; - if (vision_encoder_ && !bind_runtime_buffers(*vision_encoder_)) return false; - if (audio_encoder_ && !bind_runtime_buffers(*audio_encoder_)) return false; - if (lm_encoder_media_step_ && !bind_runtime_buffers(*lm_encoder_media_step_)) return false; - if (decoder_prefill_chunk_ && !bind_runtime_buffers(*decoder_prefill_chunk_)) return false; - if (lm_encoder_ && !bind_runtime_buffers(*lm_encoder_)) return false; - if (lm_encoder_text_chunk_ && !bind_runtime_buffers(*lm_encoder_text_chunk_)) return false; - if (lm_encoder_media_chunk_ && !bind_runtime_buffers(*lm_encoder_media_chunk_)) return false; + std::vector to_bind = { + encoder_, + prefill_encoder_, + decoder_, + decoder_prefill_, + vision_encoder_, + audio_encoder_, + lm_encoder_media_step_, + lm_encoder_, + lm_encoder_text_chunk_, + lm_encoder_media_chunk_, + }; + std::unordered_set bound; + for (Component* comp : to_bind) { + if (!comp || !comp->graph || bound.count(comp)) continue; + if (!bind_runtime_buffers(*comp)) return false; + bound.insert(comp); + } if (vision_encoder_ && tokenizer_ && !vision_encoder_->output_node_ids.empty()) { size_t out_node = static_cast(vision_encoder_->output_node_ids[0]); @@ -183,6 +308,14 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, } cache_max_seq_len_ = context_size; + + if (!npu_prefill_mlpackage_.empty()) { + std::string full_path = bundle_dir + "/" + npu_prefill_mlpackage_; + if (!load_npu_prefill(full_path)) { + CACTUS_LOG_WARN("model", "NPU prefill load failed for " << full_path << "; falling back to CPU"); + } + } + initialized_ = true; return true; } @@ -200,6 +333,9 @@ bool Model::load_manifest() { if (obj.count("family") && obj.at("family").is()) { family_ = obj.at("family").get(); } + if (obj.count("npu_prefill") && obj.at("npu_prefill").is()) { + npu_prefill_mlpackage_ = obj.at("npu_prefill").get(); + } if (!obj.count("components")) return false; for (const auto& cv : obj.at("components").get()) { const auto& c = cv.get(); @@ -232,15 +368,15 @@ bool Model::load_manifest() { } } if (c.count("cache_state_node_ids")) { - for (const auto& ev : c.at("cache_state_node_ids").get()) { - if (!ev.is()) continue; - const auto& e = ev.get(); - CacheStateEntry cs; - if (e.count("layer_key")) cs.layer_key = e.at("layer_key").get(); - if (e.count("key") && e.at("key").is()) - cs.key_node_id = static_cast(e.at("key").get()); - if (e.count("value") && e.at("value").is()) - cs.value_node_id = static_cast(e.at("value").get()); + for (const auto& sv : c.at("cache_state_node_ids").get()) { + if (!sv.is()) continue; + const auto& s = sv.get(); + CacheStateBinding cs; + if (s.count("layer_key")) cs.layer_key = s.at("layer_key").get(); + if (s.count("key") && s.at("key").is()) + cs.key_node_id = static_cast(s.at("key").get()); + if (s.count("value") && s.at("value").is()) + cs.value_node_id = static_cast(s.at("value").get()); if (cs.key_node_id >= 0 && cs.value_node_id >= 0) { comp.cache_states.push_back(std::move(cs)); } @@ -265,28 +401,51 @@ bool Model::setup_tokenizer() { return tokenizer_->load_vocabulary_with_config(vocab, merges, cfg); } -bool Model::load_components() { +bool Model::load_components(const std::unordered_set& required_components) { for (auto& [name, comp] : components_) { - if (comp.graph_path.empty()) continue; - fs::path full = fs::path(bundle_dir_) / comp.graph_path; + if (!required_components.empty() && !required_components.count(name)) continue; + if (!load_component_graph(comp)) return false; + } + return true; +} + +bool Model::load_component_graph(Component& comp) { + if (comp.graph) return true; + if (comp.graph_path.empty()) return true; + fs::path full = fs::path(bundle_dir_) / comp.graph_path; + try { + comp.graph = std::make_unique(CactusGraph::load(full.string())); + comp.graph->retain_outputs(comp.output_node_ids); + } catch (const std::exception& e) { + CACTUS_LOG_ERROR("model", "load " << comp.graph_path << ": " << e.what()); + return false; + } + for (const auto& b : comp.bindings) { + if (b.node_id < 0) continue; try { - comp.graph = std::make_unique(CactusGraph::load(full.string())); + fs::path weight_path(b.path); + if (weight_path.is_absolute()) { + fs::path local = fs::path(bundle_dir_) / weight_path.filename(); + if (fs::exists(local)) weight_path = local; + } else { + weight_path = fs::path(bundle_dir_) / weight_path; + } + comp.graph->bind_mmap_weights(static_cast(b.node_id), weight_path.string()); } catch (const std::exception& e) { - CACTUS_LOG_ERROR("model", "load " << comp.graph_path << ": " << e.what()); + CACTUS_LOG_ERROR("model", "bind " << b.path << ": " << e.what()); return false; } - for (const auto& b : comp.bindings) { - if (b.node_id < 0) continue; - try { - comp.graph->bind_mmap_weights(static_cast(b.node_id), - (fs::path(bundle_dir_) / b.path).string()); - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("model", "bind " << b.path << ": " << e.what()); - return false; - } - } } - return true; + return bind_runtime_buffers(comp); +} + +void Model::unload_component_graph(Component& comp) { + if (comp.graph) { + comp.graph->release_runtime_buffers(); + comp.graph->release_all_weight_pages(); + } + comp.input_buffers.clear(); + comp.graph.reset(); } bool Model::bind_runtime_buffers(Component& comp) { @@ -308,23 +467,30 @@ int Model::input_index(const Component& comp, const std::string& name) const { } void Model::write_int_input(Component& comp, const std::string& name, int64_t value) { + write_int_input_at(comp, name, 0, value); +} + +void Model::write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value) { int idx = input_index(comp, name); if (idx < 0) return; size_t node_id = static_cast(comp.runtime_input_node_ids[idx]); const auto& desc = comp.graph->get_output_buffer(node_id); auto& buf = comp.input_buffers[idx]; + if (index >= desc.total_size) return; + size_t offset = PrecisionTraits::byte_offset_of(desc.precision, index); + auto* dst = buf.data() + offset; switch (desc.precision) { case Precision::FP32: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; case Precision::FP16: - *reinterpret_cast<__fp16*>(buf.data()) = static_cast<__fp16>(value); + *reinterpret_cast<__fp16*>(dst) = static_cast<__fp16>(value); break; case Precision::INT8: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; default: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; } } @@ -362,10 +528,367 @@ void Model::copy_encoder_outputs_to_decoder(const Component& enc) { } void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { + if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { + write_int_input(*decoder_, "input_ids", static_cast(token_id)); + write_int_input(*decoder_, "position_ids", static_cast(position)); + decoder_->graph->execute(); + return; + } + run_encoder_step(token_id, position); + copy_component_outputs_to_inputs(*encoder_, *decoder_); + decoder_->graph->execute(); +} + +void Model::run_encoder_step(uint32_t token_id, size_t position) { write_int_input(*encoder_, "input_ids", static_cast(token_id)); write_int_input(*encoder_, "position_ids", static_cast(position)); encoder_->graph->execute(); - copy_encoder_outputs_to_decoder(*encoder_); +} + +void Model::copy_component_outputs_to_inputs(const Component& source, Component& target) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + std::fill(target.input_buffers[dst_idx].begin(), target.input_buffers[dst_idx].end(), 0); + size_t elements = std::min(src_desc.total_size, dst_desc.total_size); + if (!copy_component_tensor(*source.graph, src_desc, src_node, dst_desc, target.input_buffers[dst_idx], 0, elements, out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +void Model::copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + size_t chunk_tokens = component_chunk_tokens(target, out_name); + if (chunk_tokens <= token_index || chunk_tokens == 0) { + throw std::runtime_error("chunk prefill token index exceeds input capacity for " + out_name); + } + if (dst_desc.total_size % chunk_tokens != 0) { + throw std::runtime_error("chunk prefill input shape is not token-aligned for " + out_name); + } + size_t elements_per_token = dst_desc.total_size / chunk_tokens; + if (src_desc.total_size != elements_per_token) { + throw std::runtime_error("component output/input token shape mismatch for " + out_name); + } + if (!copy_component_tensor( + *source.graph, + src_desc, + src_node, + dst_desc, + target.input_buffers[dst_idx], + token_index * elements_per_token, + src_desc.total_size, + out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +void Model::copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + size_t src_tokens = component_output_tokens(source, out_name); + size_t dst_tokens = component_chunk_tokens(target, out_name); + if (src_tokens == 0 || dst_tokens == 0 || token_offset + src_tokens > dst_tokens) { + throw std::runtime_error("chunk prefill output range exceeds input capacity for " + out_name); + } + if (src_desc.total_size % src_tokens != 0 || dst_desc.total_size % dst_tokens != 0) { + throw std::runtime_error("chunk prefill output/input shape is not token-aligned for " + out_name); + } + size_t src_elements_per_token = src_desc.total_size / src_tokens; + size_t dst_elements_per_token = dst_desc.total_size / dst_tokens; + if (src_elements_per_token != dst_elements_per_token) { + throw std::runtime_error("component output/input token shape mismatch for " + out_name); + } + if (!copy_component_tensor( + *source.graph, + src_desc, + src_node, + dst_desc, + target.input_buffers[dst_idx], + token_offset * dst_elements_per_token, + src_desc.total_size, + out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +bool Model::cache_states_compatible(const Component& source, const Component& target) const { + if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) return false; + for (size_t i = 0; i < source.cache_states.size(); ++i) { + const auto& src = source.cache_states[i]; + const auto& dst = target.cache_states[i]; + if (src.layer_key != dst.layer_key) return false; + if (src.key_node_id < 0 || src.value_node_id < 0 || dst.key_node_id < 0 || dst.value_node_id < 0) return false; + } + return true; +} + +void Model::copy_cache_states(const Component& source, Component& target) { + if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) { + throw std::runtime_error("prefill and step cache states are not compatible"); + } + for (size_t i = 0; i < source.cache_states.size(); ++i) { + const auto& src = source.cache_states[i]; + const auto& dst = target.cache_states[i]; + if (src.layer_key != dst.layer_key) { + throw std::runtime_error("prefill and step cache layer mismatch: " + src.layer_key + " != " + dst.layer_key); + } + for (auto [src_node, dst_node] : {std::pair{src.key_node_id, dst.key_node_id}, std::pair{src.value_node_id, dst.value_node_id}}) { + const auto& src_desc = source.graph->get_output_buffer(static_cast(src_node)); + const auto& dst_desc = target.graph->get_output_buffer(static_cast(dst_node)); + if (src_desc.precision != dst_desc.precision) { + std::ostringstream oss; + oss << "prefill and step cache precision mismatch at layer " << src.layer_key + << ": " << static_cast(src_desc.precision) + << " vs " << static_cast(dst_desc.precision); + throw std::runtime_error(oss.str()); + } + void* src_ptr = source.graph->get_output(static_cast(src_node)); + void* dst_ptr = target.graph->get_output(static_cast(dst_node)); + if (src_desc.byte_size == dst_desc.byte_size) { + std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); + continue; + } + if (src_desc.precision == Precision::INT8) { + auto* src_meta = static_cast(src_ptr); + auto* dst_meta = static_cast(dst_ptr); + const size_t src_current = static_cast(src_meta[0]); + const size_t src_max = static_cast(src_meta[1]); + const size_t kv_heads = static_cast(src_meta[2]); + const size_t head_dim = static_cast(src_meta[3]); + const size_t sink = static_cast(src_meta[4]); + if (src_max == 0 || kv_heads == 0 || head_dim == 0) { + throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); + } + const size_t groups = (head_dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + const size_t int8_stride = kv_heads * head_dim; + const size_t scale_stride = kv_heads * groups; + const size_t row_bytes = int8_stride + scale_stride * sizeof(float); + const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; + if (dst_max == 0) { + throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); + } + const size_t dst_current = std::min(src_current, dst_max); + dst_meta[0] = dst_current; + dst_meta[1] = dst_max; + dst_meta[2] = kv_heads; + dst_meta[3] = head_dim; + dst_meta[4] = std::min(sink, dst_current); + std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); + + const auto* src_i8 = static_cast(src_ptr) + 64; + const auto* src_scales = reinterpret_cast( + static_cast(src_ptr) + 64 + src_max * int8_stride); + auto* dst_i8 = static_cast(dst_ptr) + 64; + auto* dst_scales = reinterpret_cast( + static_cast(dst_ptr) + 64 + dst_max * int8_stride); + auto copy_rows = [&](size_t dst_row, size_t src_row, size_t rows) { + if (rows == 0) return; + std::memcpy( + dst_i8 + dst_row * int8_stride, + src_i8 + src_row * int8_stride, + rows * int8_stride); + std::memcpy( + dst_scales + dst_row * scale_stride, + src_scales + src_row * scale_stride, + rows * scale_stride * sizeof(float)); + }; + if (src_current <= dst_max) { + copy_rows(0, 0, src_current); + } else { + const size_t copied_sink = std::min(sink, dst_max); + const size_t tail_rows = dst_max - copied_sink; + copy_rows(0, 0, copied_sink); + if (tail_rows > 0) copy_rows(copied_sink, src_current - tail_rows, tail_rows); + } + continue; + } + if (PrecisionTraits::is_cq(src_desc.precision) || src_desc.byte_size < 64 || dst_desc.byte_size < 64) { + std::ostringstream oss; + oss << "prefill and step cache buffer mismatch at layer " << src.layer_key + << ": " << src_desc.byte_size << " bytes vs " << dst_desc.byte_size << " bytes"; + throw std::runtime_error(oss.str()); + } + + auto* src_meta = static_cast(src_ptr); + auto* dst_meta = static_cast(dst_ptr); + const size_t src_current = static_cast(src_meta[0]); + const size_t src_max = static_cast(src_meta[1]); + const size_t kv_heads = static_cast(src_meta[2]); + const size_t head_dim = static_cast(src_meta[3]); + const size_t sink = static_cast(src_meta[4]); + if (src_max == 0 || kv_heads == 0 || head_dim == 0) { + throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); + } + const size_t row_bytes = kv_heads * head_dim * PrecisionTraits::size_of(src_desc.precision); + const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; + if (dst_max == 0) { + throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); + } + const size_t dst_current = std::min(src_current, dst_max); + dst_meta[0] = dst_current; + dst_meta[1] = dst_max; + dst_meta[2] = kv_heads; + dst_meta[3] = head_dim; + dst_meta[4] = std::min(sink, dst_current); + std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); + const auto* src_rows = static_cast(src_ptr) + 64; + auto* dst_rows = static_cast(dst_ptr) + 64; + if (src_current <= dst_max) { + std::memcpy(dst_rows, src_rows, src_current * row_bytes); + } else { + const size_t copied_sink = std::min(sink, dst_max); + const size_t tail_rows = dst_max - copied_sink; + if (copied_sink > 0) { + std::memcpy(dst_rows, src_rows, copied_sink * row_bytes); + } + if (tail_rows > 0) { + std::memcpy( + dst_rows + copied_sink * row_bytes, + src_rows + (src_current - tail_rows) * row_bytes, + tail_rows * row_bytes); + } + } + } + } +} + +void Model::reset_component_cache_states(Component& comp) { + for (const auto& state : comp.cache_states) { + for (int node_id : {state.key_node_id, state.value_node_id}) { + if (node_id < 0) continue; + const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); + if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue; + void* ptr = comp.graph->get_output(static_cast(node_id)); + if (!ptr) continue; + auto* metadata = static_cast(ptr); + metadata[0] = 0; + } + } +} + +size_t Model::component_chunk_tokens(const Component& comp, const std::string& input_name) const { + int idx = input_index(comp, input_name); + if (idx < 0) return 0; + const auto& desc = comp.graph->get_output_buffer(static_cast(comp.runtime_input_node_ids[idx])); + if (desc.shape.size() >= 2 && desc.shape[0] == 1) return desc.shape[1]; + return desc.shape.empty() ? 0 : desc.shape[0]; +} + +size_t Model::component_output_tokens(const Component& comp, const std::string& output_name) const { + for (size_t i = 0; i < comp.logical_outputs.size() && i < comp.output_node_ids.size(); ++i) { + if (comp.logical_outputs[i] != output_name) continue; + const auto& desc = comp.graph->get_output_buffer(static_cast(comp.output_node_ids[i])); + if (desc.shape.size() >= 2 && desc.shape[0] == 1) return desc.shape[1]; + return desc.shape.empty() ? 0 : desc.shape[0]; + } + return 0; +} + +size_t Model::run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, bool prepare_decode) { + if (decode_route_ != DecodeRoute::CACHED_STEP || !encoder_ || !decoder_ || !decoder_prefill_) return 0; + if (start_position != 0) return 0; + if (!load_component_graph(*decoder_prefill_)) return 0; + if (prefill_encoder_ && !load_component_graph(*prefill_encoder_)) return 0; + if (!cache_states_compatible(*decoder_prefill_, *decoder_)) return 0; + size_t component_tokens = component_chunk_tokens(*decoder_prefill_, "inputs_embeds"); + if (component_tokens <= 1) return 0; + size_t effective_chunk = chunk_size > 0 ? std::min(chunk_size, component_tokens) : component_tokens; + if (effective_chunk != component_tokens) effective_chunk = component_tokens; + if (tokens.size() < effective_chunk) return 0; + + size_t encoder_chunk = 0; + if (prefill_encoder_ && input_index(*prefill_encoder_, "input_ids") >= 0 && input_index(*prefill_encoder_, "position_ids") >= 0) { + encoder_chunk = component_chunk_tokens(*prefill_encoder_, "input_ids"); + if (encoder_chunk == 0 || effective_chunk % encoder_chunk != 0) { + encoder_chunk = 0; + } + } + + size_t processed = 0; + while (processed + effective_chunk <= tokens.size()) { + for (size_t i = 0; i < decoder_prefill_->input_buffers.size(); ++i) { + std::fill(decoder_prefill_->input_buffers[i].begin(), decoder_prefill_->input_buffers[i].end(), 0); + } + if (encoder_chunk > 0) { + for (size_t chunk_offset = 0; chunk_offset < effective_chunk; chunk_offset += encoder_chunk) { + for (size_t i = 0; i < prefill_encoder_->input_buffers.size(); ++i) { + std::fill(prefill_encoder_->input_buffers[i].begin(), prefill_encoder_->input_buffers[i].end(), 0); + } + for (size_t i = 0; i < encoder_chunk; ++i) { + write_int_input_at(*prefill_encoder_, "input_ids", i, static_cast(tokens[processed + chunk_offset + i])); + write_int_input_at(*prefill_encoder_, "position_ids", i, static_cast(start_position + processed + chunk_offset + i)); + } + prefill_encoder_->graph->execute(); + copy_component_outputs_to_chunk_inputs_range(*prefill_encoder_, *decoder_prefill_, chunk_offset); + } + } else { + for (size_t i = 0; i < effective_chunk; ++i) { + run_encoder_step(tokens[processed + i], start_position + processed + i); + copy_component_outputs_to_chunk_inputs(*encoder_, *decoder_prefill_, i); + } + } + decoder_prefill_->graph->execute(); + processed += effective_chunk; + } + if (processed > 0 && prepare_decode) { + for (size_t i = 0; i < decoder_->input_buffers.size(); ++i) { + std::fill(decoder_->input_buffers[i].begin(), decoder_->input_buffers[i].end(), 0); + } + copy_cache_states(*decoder_prefill_, *decoder_); + } + return processed; +} + +void Model::run_full_context_text() { + if (!encoder_ || !decoder_ || context_tokens_.empty()) return; + int input_ids_idx = input_index(*encoder_, "input_ids"); + int attention_mask_idx = input_index(*encoder_, "attention_mask"); + if (input_ids_idx < 0 || attention_mask_idx < 0) { + throw std::runtime_error("text_lm_encoder requires input_ids and attention_mask inputs"); + } + size_t input_node = static_cast(encoder_->runtime_input_node_ids[input_ids_idx]); + const auto& input_desc = encoder_->graph->get_output_buffer(input_node); + if (context_tokens_.size() > input_desc.total_size) { + throw std::runtime_error("context exceeds transpiled text_lm_encoder capacity"); + } + std::fill(encoder_->input_buffers[input_ids_idx].begin(), encoder_->input_buffers[input_ids_idx].end(), 0); + std::fill(encoder_->input_buffers[attention_mask_idx].begin(), encoder_->input_buffers[attention_mask_idx].end(), 0); + for (size_t i = 0; i < context_tokens_.size(); ++i) { + write_int_input_at(*encoder_, "input_ids", i, static_cast(context_tokens_[i])); + write_int_input_at(*encoder_, "attention_mask", i, 1); + } + encoder_->graph->execute(); + for (size_t i = 0; i < encoder_->output_node_ids.size() && i < encoder_->logical_outputs.size(); ++i) { + const std::string& out_name = encoder_->logical_outputs[i]; + int dst_idx = input_index(*decoder_, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(encoder_->output_node_ids[i]); + const auto& src_desc = encoder_->graph->get_output_buffer(src_node); + void* src_ptr = encoder_->graph->get_output(src_node); + std::memcpy(decoder_->input_buffers[dst_idx].data(), src_ptr, src_desc.byte_size); + } + last_logit_position_ = context_tokens_.empty() ? 0 : context_tokens_.size() - 1; decoder_->graph->execute(); } @@ -419,6 +942,9 @@ void Model::run_media_step(size_t position, const uint8_t* feature_row, size_t f void Model::run_vision_encoder(const std::string& image_path) { if (!vision_encoder_) return; + if (!load_component_graph(*vision_encoder_)) { + throw std::runtime_error("failed to load vision_encoder"); + } Gemma4ImagePreprocessed prep = preprocess_gemma4_image(image_path, config_); write_bytes_input(*vision_encoder_, "pixel_values", prep.pixel_values.data(), prep.pixel_values.size() * sizeof(float)); @@ -436,74 +962,125 @@ void Model::run_vision_encoder(const std::string& image_path) { media_feature_shapes_[name] = desc.shape; media_feature_precisions_[name] = desc.precision; } + vision_encoder_->graph->release_runtime_buffers(); + vision_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*vision_encoder_); +} + +void Model::run_audio_encoder_messages(const std::vector>& audio_features_per_message) { + if (!audio_encoder_) return; + if (audio_features_per_message.empty()) return; + if (!load_component_graph(*audio_encoder_)) { + throw std::runtime_error("failed to load audio_encoder"); + } + for (const std::string& logical : audio_encoder_->logical_outputs) { + media_features_.erase(logical); + media_feature_shapes_.erase(logical); + media_feature_precisions_.erase(logical); + } + for (const auto& mel : audio_features_per_message) { + if (mel.empty()) continue; + run_audio_encoder(mel); + } + audio_encoder_->graph->release_runtime_buffers(); + audio_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*audio_encoder_); } void Model::run_audio_encoder(const std::vector& audio_features) { if (!audio_encoder_) return; - std::vector candidate_input_names = {"input_features", "audio_features"}; - bool wrote = false; + const std::vector candidate_input_names = {"input_features", "audio_features"}; + int feature_idx = -1; for (const auto& name : candidate_input_names) { int idx = input_index(*audio_encoder_, name); - if (idx < 0) continue; - size_t node_id = static_cast(audio_encoder_->runtime_input_node_ids[idx]); - const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); - auto& buf = audio_encoder_->input_buffers[idx]; - if (desc.precision == Precision::FP32) { - size_t n = std::min(audio_features.size() * sizeof(float), buf.size()); - std::memcpy(buf.data(), audio_features.data(), n); - if (n < buf.size()) std::memset(buf.data() + n, 0, buf.size() - n); - } else if (desc.precision == Precision::FP16) { - size_t n_elems = std::min(audio_features.size(), buf.size() / sizeof(__fp16)); + if (idx >= 0) { feature_idx = idx; break; } + } + if (feature_idx < 0) { + CACTUS_LOG_WARN("model", "audio_encoder has no input named input_features/audio_features; skipping"); + return; + } + const size_t feature_node = static_cast(audio_encoder_->runtime_input_node_ids[feature_idx]); + const auto& feature_desc = audio_encoder_->graph->get_output_buffer(feature_node); + const size_t mel_bins = feature_desc.shape.size() >= 3 + ? feature_desc.shape[2] + : static_cast(config_.audio_input_feat_size); + const size_t max_frames_per_chunk = feature_desc.shape.size() >= 2 ? feature_desc.shape[1] : 0; + if (max_frames_per_chunk == 0 || mel_bins == 0) { + CACTUS_LOG_WARN("model", "audio_encoder feature input has unexpected shape; skipping"); + return; + } + const size_t total_frames = audio_features.size() / mel_bins; + const size_t num_chunks = total_frames == 0 + ? 1 + : (total_frames + max_frames_per_chunk - 1) / max_frames_per_chunk; + + const int mask_idx = input_index(*audio_encoder_, "input_features_mask"); + + for (size_t chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) { + const size_t frame_start = chunk_idx * max_frames_per_chunk; + const size_t frames_in_chunk = frame_start >= total_frames + ? 0 + : std::min(max_frames_per_chunk, total_frames - frame_start); + const size_t chunk_feature_elems = frames_in_chunk * mel_bins; + const float* chunk_src = audio_features.data() + frame_start * mel_bins; + + auto& buf = audio_encoder_->input_buffers[feature_idx]; + if (feature_desc.precision == Precision::FP32) { + const size_t n_bytes = std::min(chunk_feature_elems * sizeof(float), buf.size()); + std::memcpy(buf.data(), chunk_src, n_bytes); + if (n_bytes < buf.size()) std::memset(buf.data() + n_bytes, 0, buf.size() - n_bytes); + } else if (feature_desc.precision == Precision::FP16) { + const size_t cap_elems = buf.size() / sizeof(__fp16); + const size_t n_elems = std::min(chunk_feature_elems, cap_elems); __fp16* dst = reinterpret_cast<__fp16*>(buf.data()); - for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast<__fp16>(audio_features[i]); - if (n_elems * sizeof(__fp16) < buf.size()) { - std::memset(buf.data() + n_elems * sizeof(__fp16), 0, buf.size() - n_elems * sizeof(__fp16)); + for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast<__fp16>(chunk_src[i]); + if (n_elems < cap_elems) { + std::memset(buf.data() + n_elems * sizeof(__fp16), 0, (cap_elems - n_elems) * sizeof(__fp16)); } } else { - size_t n_elems = std::min(audio_features.size(), buf.size()); + const size_t n_elems = std::min(chunk_feature_elems, buf.size()); int8_t* dst = reinterpret_cast(buf.data()); - for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast(audio_features[i]); + for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast(chunk_src[i]); if (n_elems < buf.size()) std::memset(buf.data() + n_elems, 0, buf.size() - n_elems); } - wrote = true; - break; - } - if (!wrote) { - CACTUS_LOG_WARN("model", "audio_encoder has no input named input_features/audio_features; skipping"); - return; - } - int mask_idx = input_index(*audio_encoder_, "input_features_mask"); - if (mask_idx >= 0) { - auto& mb = audio_encoder_->input_buffers[mask_idx]; - size_t mask_node = static_cast(audio_encoder_->runtime_input_node_ids[mask_idx]); - const auto& mask_desc = audio_encoder_->graph->get_output_buffer(mask_node); - size_t elem = PrecisionTraits::size_of(mask_desc.precision); - size_t total = elem ? mb.size() / elem : mb.size(); - std::memset(mb.data(), 0, mb.size()); - for (size_t i = 0; i < total; ++i) { - switch (mask_desc.precision) { - case Precision::FP32: reinterpret_cast(mb.data())[i] = 1.0f; break; - case Precision::FP16: reinterpret_cast<__fp16*>(mb.data())[i] = static_cast<__fp16>(1.0f); break; - case Precision::INT8: reinterpret_cast(mb.data())[i] = 1; break; - default: - if (elem == 8) reinterpret_cast(mb.data())[i] = 1; - else if (elem == 4) reinterpret_cast(mb.data())[i] = 1; - else mb[i] = 1; - break; + + if (mask_idx >= 0) { + auto& mb = audio_encoder_->input_buffers[mask_idx]; + const size_t mask_node = static_cast(audio_encoder_->runtime_input_node_ids[mask_idx]); + const auto& mask_desc = audio_encoder_->graph->get_output_buffer(mask_node); + const size_t elem = PrecisionTraits::size_of(mask_desc.precision); + const size_t cap = elem ? mb.size() / elem : 0; + const size_t n = std::min(cap, frames_in_chunk); + for (size_t i = 0; i < n; ++i) { + switch (mask_desc.precision) { + case Precision::FP32: reinterpret_cast(mb.data())[i] = 1.0f; break; + case Precision::FP16: reinterpret_cast<__fp16*>(mb.data())[i] = static_cast<__fp16>(1.0f); break; + case Precision::INT8: reinterpret_cast(mb.data())[i] = 1; break; + default: reinterpret_cast(mb.data())[i] = 1; break; + } } + if (n < cap) std::memset(mb.data() + n * elem, 0, (cap - n) * elem); + } + + audio_encoder_->graph->execute(); + + for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { + const std::string& name = audio_encoder_->logical_outputs[i]; + const size_t node_id = static_cast(audio_encoder_->output_node_ids[i]); + const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); + void* ptr = audio_encoder_->graph->get_output(node_id); + auto& slot = media_features_[name]; + const size_t prev_bytes = slot.size(); + slot.resize(prev_bytes + desc.byte_size); + std::memcpy(slot.data() + prev_bytes, ptr, desc.byte_size); + auto shape_it = media_feature_shapes_.find(name); + if (shape_it == media_feature_shapes_.end() || shape_it->second.empty()) { + media_feature_shapes_[name] = desc.shape; + } else if (desc.shape.size() >= 2 && shape_it->second.size() == desc.shape.size()) { + shape_it->second[shape_it->second.size() - 2] += desc.shape[desc.shape.size() - 2]; + } + media_feature_precisions_[name] = desc.precision; } - } - audio_encoder_->graph->execute(); - for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { - const std::string& name = audio_encoder_->logical_outputs[i]; - size_t node_id = static_cast(audio_encoder_->output_node_ids[i]); - const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); - void* ptr = audio_encoder_->graph->get_output(node_id); - auto& slot = media_features_[name]; - slot.assign(desc.byte_size, 0); - std::memcpy(slot.data(), ptr, desc.byte_size); - media_feature_shapes_[name] = desc.shape; - media_feature_precisions_[name] = desc.precision; } } @@ -513,7 +1090,8 @@ uint32_t Model::argmax_last_logits() { void* ptr = decoder_->graph->get_output(out_node); size_t vocab = desc.shape.empty() ? 0 : desc.shape.back(); size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; - size_t row_off = (seq > 0 ? (seq - 1) * vocab : 0); + size_t row = decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT ? std::min(last_logit_position_, seq > 0 ? seq - 1 : 0) : (seq > 0 ? seq - 1 : 0); + size_t row_off = row * vocab; uint32_t best = 0; float best_v = -std::numeric_limits::infinity(); if (desc.precision == Precision::FP32) { @@ -532,11 +1110,22 @@ uint32_t Model::argmax_last_logits() { return best; } -void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& /*profile_file*/) { - for (size_t i = 0; i < tokens.size(); ++i) { - run_step(tokens[i], cache_total_seq_len_ + i, /*read_logits=*/false); +void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& /*profile_file*/, bool prepare_decode) { + if (has_npu_prefill() && prefill_via_npu(tokens)) { + return; + } + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + if (!context_tokens_.empty()) run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + return; + } + size_t processed = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), prepare_decode); + cache_total_seq_len_ += processed; + for (size_t i = processed; i < tokens.size(); ++i) { + run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false); + ++cache_total_seq_len_; } - cache_total_seq_len_ += tokens.size(); } void Model::prefill_with_images(const std::vector& tokens, @@ -546,9 +1135,9 @@ void Model::prefill_with_images(const std::vector& tokens, } void Model::prefill_with_audio(const std::vector& tokens, - const std::vector& audio_features, + const std::vector>& audio_features_per_message, const std::string& profile_file) { - prefill_with_media(tokens, {}, audio_features, profile_file); + prefill_with_media(tokens, {}, audio_features_per_message, profile_file); } namespace { @@ -619,33 +1208,91 @@ void write_tokens_buffer(std::vector& buf, Precision prec, } } -} // namespace +void write_int_vector_buffer(std::vector& buf, Precision prec, const std::vector& values) { + const size_t elem = PrecisionTraits::size_of(prec); + const size_t cap = elem ? buf.size() / elem : 0; + const size_t n = std::min(cap, values.size()); + for (size_t i = 0; i < n; ++i) { + int64_t v = values[i]; + switch (prec) { + case Precision::FP32: reinterpret_cast(buf.data())[i] = static_cast(v); break; + case Precision::FP16: reinterpret_cast<__fp16*>(buf.data())[i] = static_cast<__fp16>(v); break; + case Precision::INT8: reinterpret_cast(buf.data())[i] = static_cast(v); break; + default: + if (elem == 8) reinterpret_cast(buf.data())[i] = v; + else if (elem == 4) reinterpret_cast(buf.data())[i] = static_cast(v); + break; + } + } + if (n < cap) { + std::memset(buf.data() + n * elem, 0, (cap - n) * elem); + } +} -void Model::copy_cache_state(const Component& src, Component& dst) { - if (src.cache_states.empty() || dst.cache_states.empty()) return; - if (src.cache_states.size() != dst.cache_states.size()) { - throw std::runtime_error("cache state count mismatch between " + src.name + " and " + dst.name); - } - for (size_t i = 0; i < src.cache_states.size(); ++i) { - const auto& s = src.cache_states[i]; - const auto& d = dst.cache_states[i]; - if (s.layer_key != d.layer_key) { - throw std::runtime_error("cache layer mismatch: " + s.layer_key + " vs " + d.layer_key); - } - for (auto pair : {std::pair{s.key_node_id, d.key_node_id}, - std::pair{s.value_node_id, d.value_node_id}}) { - const auto& sd = src.graph->get_output_buffer(static_cast(pair.first)); - const auto& dd = dst.graph->get_output_buffer(static_cast(pair.second)); - if (sd.byte_size != dd.byte_size) { - throw std::runtime_error("cache byte size mismatch for layer " + s.layer_key); +std::vector qwen3_vl_position_ids(const std::vector& tokens, + size_t capacity, + const std::vector& images, + uint32_t image_token_id) { + std::vector positions(3 * capacity, 0); + size_t token_index = 0; + size_t image_index = 0; + int64_t current_pos = 0; + while (token_index < tokens.size() && token_index < capacity) { + if (image_token_id != 0 && tokens[token_index] == image_token_id) { + if (image_index >= images.size()) { + throw std::runtime_error("Qwen3-VL prompt contains more image token groups than image inputs"); + } + const auto& image = images[image_index++]; + const size_t merge_size = 2; + const size_t grid_t = image.grid_t; + const size_t llm_grid_h = image.grid_h / merge_size; + const size_t llm_grid_w = image.grid_w / merge_size; + const size_t image_seq = grid_t * llm_grid_h * llm_grid_w; + size_t count = 0; + while (token_index + count < tokens.size() + && token_index + count < capacity + && tokens[token_index + count] == image_token_id) { + ++count; + } + if (count != image_seq) { + throw std::runtime_error("Qwen3-VL image token count does not match vision feature grid"); + } + size_t local = 0; + for (size_t t = 0; t < grid_t; ++t) { + (void)t; + for (size_t h = 0; h < llm_grid_h; ++h) { + for (size_t w = 0; w < llm_grid_w; ++w) { + size_t pos = token_index + local++; + positions[pos] = current_pos; + positions[capacity + pos] = current_pos + static_cast(h); + positions[2 * capacity + pos] = current_pos + static_cast(w); + } + } } - std::memcpy(dst.graph->get_output(static_cast(pair.second)), - src.graph->get_output(static_cast(pair.first)), - sd.byte_size); + current_pos += static_cast(std::max(image.grid_h, image.grid_w) / merge_size); + token_index += count; + continue; + } + + size_t text_count = 0; + while (token_index + text_count < tokens.size() + && token_index + text_count < capacity + && (image_token_id == 0 || tokens[token_index + text_count] != image_token_id)) { + size_t pos = token_index + text_count; + int64_t value = current_pos + static_cast(text_count); + positions[pos] = value; + positions[capacity + pos] = value; + positions[2 * capacity + pos] = value; + ++text_count; } + current_pos += static_cast(text_count); + token_index += text_count; } + return positions; } +} // namespace + bool Model::build_lm_encoder_outputs_dynamic_gemma4( const std::vector& tokens, std::map>& store_bytes, @@ -775,11 +1422,23 @@ bool Model::build_lm_encoder_outputs_dynamic_gemma4( bool Model::run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features) { + const std::vector>& audio_features_per_message) { + if (cache_total_seq_len_ > 0) return false; const bool have_images = !image_paths.empty() && vision_encoder_ != nullptr; - const bool have_audio = !audio_features.empty() && audio_encoder_ != nullptr; + bool any_audio = false; + for (const auto& mel : audio_features_per_message) { if (!mel.empty()) { any_audio = true; break; } } + const bool have_audio = any_audio && audio_encoder_ != nullptr; + std::vector qwen_images; if (have_images) { + if (!load_component_graph(*vision_encoder_)) { + throw std::runtime_error("failed to load vision_encoder"); + } + for (const std::string& logical : vision_encoder_->logical_outputs) { + media_features_.erase(logical); + media_feature_shapes_.erase(logical); + media_feature_precisions_.erase(logical); + } for (const auto& path : image_paths) { if (family_ == "lfm2_vl") { Lfm2VlImagePreprocessed prep = preprocess_lfm2_vl_image(path, config_); @@ -812,6 +1471,20 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, } if (n < cap) std::memset(pm_buf.data() + n * elem, 0, (cap - n) * elem); } + } else if (family_ == "qwen3_5" || family_ == "qwen3_vl" || config_.model_type == Config::ModelType::QWEN) { + Qwen3VlImagePreprocessed prep = preprocess_qwen3_vl_image(path, config_); + int pv_idx = input_index(*vision_encoder_, "pixel_values"); + if (pv_idx < 0) { + throw std::runtime_error("Qwen3-VL vision_encoder missing pixel_values input"); + } + auto& pv_buf = vision_encoder_->input_buffers[pv_idx]; + size_t pv_node = static_cast(vision_encoder_->runtime_input_node_ids[pv_idx]); + const auto& pv_desc = vision_encoder_->graph->get_output_buffer(pv_node); + write_typed_buffer(pv_buf, pv_desc.precision, + prep.pixel_values.data(), + prep.pixel_values.size() * sizeof(float), + Precision::FP32); + qwen_images.push_back(std::move(prep)); } else { Gemma4ImagePreprocessed prep = preprocess_gemma4_image(path, config_); int pv_idx = input_index(*vision_encoder_, "pixel_values"); @@ -855,30 +1528,40 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, const auto& desc = vision_encoder_->graph->get_output_buffer(node_id); void* ptr = vision_encoder_->graph->get_output(node_id); auto& slot = media_features_[name]; - slot.assign(desc.byte_size, 0); - std::memcpy(slot.data(), ptr, desc.byte_size); - media_feature_shapes_[name] = desc.shape; + const size_t prev_bytes = slot.size(); + slot.resize(prev_bytes + desc.byte_size); + std::memcpy(slot.data() + prev_bytes, ptr, desc.byte_size); + auto shape_it = media_feature_shapes_.find(name); + if (shape_it == media_feature_shapes_.end() || shape_it->second.empty()) { + media_feature_shapes_[name] = desc.shape; + } else if (desc.shape.size() >= 2 && shape_it->second.size() == desc.shape.size()) { + shape_it->second[shape_it->second.size() - 2] += desc.shape[desc.shape.size() - 2]; + } media_feature_precisions_[name] = desc.precision; } + vision_encoder_->graph->release_runtime_buffers(); + vision_encoder_->graph->release_all_weight_pages(); } + unload_component_graph(*vision_encoder_); } if (have_audio) { - run_audio_encoder(audio_features); + run_audio_encoder_messages(audio_features_per_message); } std::map> store_bytes; std::map store_prec; std::map> store_shape; - - const bool needs_dynamic_walk = (family_ == "gemma4") && have_audio - && encoder_ != nullptr && lm_encoder_media_step_ != nullptr; + const bool needs_dynamic_walk = family_ == "gemma4" && (have_images || have_audio) && lm_encoder_media_step_ != nullptr; if (needs_dynamic_walk) { if (!build_lm_encoder_outputs_dynamic_gemma4(tokens, store_bytes, store_prec, store_shape)) { return false; } } else { + if (!load_component_graph(*lm_encoder_)) { + throw std::runtime_error("failed to load lm_encoder"); + } int ids_idx = input_index(*lm_encoder_, "input_ids"); if (ids_idx >= 0) { auto& ids_buf = lm_encoder_->input_buffers[ids_idx]; @@ -895,6 +1578,18 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, fill_int_buffer(mb, mdesc.precision, 1, tokens.size()); } + int pos_idx = input_index(*lm_encoder_, "position_ids"); + if (pos_idx >= 0) { + auto& pos_buf = lm_encoder_->input_buffers[pos_idx]; + size_t pos_node = static_cast(lm_encoder_->runtime_input_node_ids[pos_idx]); + const auto& pos_desc = lm_encoder_->graph->get_output_buffer(pos_node); + if (pos_desc.shape.size() >= 3 && pos_desc.shape[0] == 3 && !qwen_images.empty()) { + size_t capacity = pos_desc.shape[pos_desc.shape.size() - 1]; + auto positions = qwen3_vl_position_ids(tokens, capacity, qwen_images, config_.image_token_id); + write_int_vector_buffer(pos_buf, pos_desc.precision, positions); + } + } + for (const auto& kv : media_features_) { const std::string& name = kv.first; int idx = input_index(*lm_encoder_, name); @@ -920,6 +1615,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, store_prec[name] = desc.precision; store_shape[name] = desc.shape; } + lm_encoder_->graph->release_runtime_buffers(); + lm_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*lm_encoder_); } auto embeds_shape_it = store_shape.find("inputs_embeds"); @@ -936,6 +1634,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, size_t chunk_seq = 0; { + if (!load_component_graph(*decoder_prefill_chunk_)) { + throw std::runtime_error("failed to load decoder_prefill_chunk"); + } int idx = input_index(*decoder_prefill_chunk_, "inputs_embeds"); if (idx < 0) return false; size_t node_id = static_cast(decoder_prefill_chunk_->runtime_input_node_ids[idx]); @@ -997,7 +1698,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, decoder_prefill_chunk_->graph->execute(); } if (whole_chunks_end > 0 && decoder_ != nullptr) { - copy_cache_state(*decoder_prefill_chunk_, *decoder_); + copy_cache_states(*decoder_prefill_chunk_, *decoder_); + decoder_prefill_chunk_->graph->release_runtime_buffers(); + unload_component_graph(*decoder_prefill_chunk_); } for (size_t pos = whole_chunks_end; pos < valid_seq; ++pos) { for (const auto& kv : store_bytes) { @@ -1020,11 +1723,19 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, void Model::prefill_with_media(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features, + const std::vector>& audio_features_per_message, const std::string& profile_file) { if (tokens.empty()) return; - const bool have_images = !image_paths.empty() && vision_encoder_ != nullptr; - const bool have_audio = !audio_features.empty() && audio_encoder_ != nullptr; + if (!image_paths.empty() && vision_encoder_ == nullptr) { + throw std::runtime_error("Model bundle does not include a vision_encoder for image input"); + } + bool any_audio = false; + for (const auto& mel : audio_features_per_message) { if (!mel.empty()) { any_audio = true; break; } } + if (any_audio && audio_encoder_ == nullptr) { + throw std::runtime_error("Model bundle does not include an audio_encoder for audio input"); + } + const bool have_images = !image_paths.empty(); + const bool have_audio = any_audio; if (!have_images && !have_audio) { prefill(tokens, get_prefill_chunk_size(), profile_file); return; @@ -1034,7 +1745,7 @@ void Model::prefill_with_media(const std::vector& tokens, lm_encoder_ != nullptr && decoder_prefill_chunk_ != nullptr && (vision_encoder_ != nullptr || audio_encoder_ != nullptr); if (can_chunk_prefill) { - if (run_chunk_prefill_path(tokens, image_paths, audio_features)) { + if (run_chunk_prefill_path(tokens, image_paths, audio_features_per_message)) { (void)profile_file; return; } @@ -1051,7 +1762,7 @@ void Model::prefill_with_media(const std::vector& tokens, } } if (have_audio) { - run_audio_encoder(audio_features); + run_audio_encoder_messages(audio_features_per_message); } std::string image_feature_name; @@ -1140,6 +1851,15 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* size_t /*top_k*/, const std::string& /*profile_file*/, float* out_entropy, float /*min_p*/, float /*repetition_penalty*/) { if (tokens.empty()) return 0; + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + if (out_entropy) *out_entropy = 0.0f; + uint32_t result = argmax_last_logits(); + record_sampled_token(result); + return result; + } for (size_t i = 0; i + 1 < tokens.size(); ++i) { run_step(tokens[i], cache_total_seq_len_ + i, /*read_logits=*/false); } @@ -1151,7 +1871,8 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* return result; } -uint32_t Model::decode_with_audio(const std::vector& tokens, const std::vector& /*mel_bins*/, +uint32_t Model::decode_with_audio(const std::vector& tokens, + const std::vector>& /*audio_features_per_message*/, float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, float min_p, float repetition_penalty, float* /*out_token_time_start*/, float* /*out_token_time_end*/) { @@ -1356,20 +2077,13 @@ std::vector Model::get_audio_embeddings(const std::vector& /*mel_b void Model::reset_cache() { cache_total_seq_len_ = 0; + last_logit_position_ = 0; + context_tokens_.clear(); token_history_.clear(); for (auto& kv : components_) { Component& comp = kv.second; if (!comp.graph) continue; - for (const auto& cs : comp.cache_states) { - for (int node_id : {cs.key_node_id, cs.value_node_id}) { - if (node_id < 0) continue; - const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); - if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue; - void* data = comp.graph->get_output(static_cast(node_id)); - if (!data) continue; - *static_cast(data) = 0; - } - } + reset_component_cache_states(comp); } } @@ -1759,9 +2473,6 @@ const std::vector& Model::get_debug_nodes() const { return debug_nodes_; } -bool Model::load_npu_prefill(const std::string& /*model_path*/) { - return false; -} double Model::score_tokens_window_logprob(const std::vector& /*tokens*/, size_t /*start*/, size_t /*end*/, size_t /*context*/, size_t* tokens_scored) { diff --git a/cactus-engine/src/transcribe.cpp b/cactus-engine/src/transcribe.cpp index f64370d5d..b9be2e0ac 100644 --- a/cactus-engine/src/transcribe.cpp +++ b/cactus-engine/src/transcribe.cpp @@ -290,7 +290,7 @@ int cactus_transcribe( float tok_time_start = 0.0f; float tok_time_end = 0.0f; uint32_t next_token = handle->model->decode_with_audio( - tokens, audio_features, + tokens, std::vector>{audio_features}, options.temperature, options.top_p, options.top_k, "", &token_entropy, options.min_p, options.repetition_penalty, diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 42001d8e9..25736f5f6 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -416,16 +416,6 @@ inline AudioPreprocessResult preprocess_audio_for_gemma4( AudioPreprocessResult result; if (audio_samples.empty()) return result; - size_t max_soft = model_config.audio_soft_tokens; - if (max_soft > 0) { - size_t max_frames = (max_soft * 2 - 1) * 2 - 1; - size_t hop_length = 160; - size_t max_samples = max_frames * hop_length; - if (audio_samples.size() > max_samples) { - audio_samples.resize(max_samples); - } - } - size_t pad_amt = 320 - (audio_samples.size() % 320); if (pad_amt < 320) audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); @@ -442,8 +432,15 @@ inline AudioPreprocessResult preprocess_audio_for_gemma4( result.num_frames = mel.size() / mel_bins; result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); - size_t after_stage1 = (result.num_frames + 1) / 2; - result.num_soft_tokens = (after_stage1 + 1) / 2; + size_t max_soft = model_config.audio_soft_tokens; + size_t max_frames_per_chunk = max_soft > 0 ? (max_soft * 2 - 1) * 2 - 1 : 0; + if (max_soft > 0 && result.num_frames > max_frames_per_chunk) { + size_t num_chunks = (result.num_frames + max_frames_per_chunk - 1) / max_frames_per_chunk; + result.num_soft_tokens = num_chunks * max_soft; + } else { + size_t after_stage1 = (result.num_frames + 1) / 2; + result.num_soft_tokens = (after_stage1 + 1) / 2; + } return result; } diff --git a/cactus-engine/tests/asr.cpp b/cactus-engine/tests/asr.cpp index 494a62e1d..7cc10455a 100644 --- a/cactus-engine/tests/asr.cpp +++ b/cactus-engine/tests/asr.cpp @@ -52,7 +52,7 @@ int main(int argc, char** argv) { std::cout << "Model loaded.\n"; std::ostringstream opts; - opts << "{\"language\":\"" << language << "\",\"telemetry_enabled\":false}"; + opts << "{\"language\":\"" << language << "\",\"telemetry_enabled\":false,\"auto_handoff\":false,\"max_tokens\":512}"; constexpr size_t kBufSize = 1 << 16; std::string response(kBufSize, '\0'); diff --git a/cactus-engine/tests/test_stt.cpp b/cactus-engine/tests/test_stt.cpp index d22396a6e..55f9b2ca0 100644 --- a/cactus-engine/tests/test_stt.cpp +++ b/cactus-engine/tests/test_stt.cpp @@ -35,7 +35,7 @@ bool test_transcription() { Timer timer; int rc = cactus_transcribe(model, audio_path.c_str(), nullptr, response, sizeof(response), - R"({"max_tokens": 200, "telemetry_enabled": false})", + R"({"max_tokens": 200, "telemetry_enabled": false, "auto_handoff": false})", nullptr, nullptr, nullptr, 0); double elapsed = timer.elapsed_ms(); @@ -105,7 +105,7 @@ bool test_transcription_pcm() { Timer timer; int rc = cactus_transcribe(model, nullptr, nullptr, response, sizeof(response), - R"({"max_tokens": 200, "telemetry_enabled": false})", + R"({"max_tokens": 200, "telemetry_enabled": false, "auto_handoff": false})", nullptr, nullptr, pcm_data.data(), pcm_data.size()); double elapsed = timer.elapsed_ms(); diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index c5373bcbe..02a6eadda 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -299,6 +299,7 @@ struct BufferDesc { void allocate(); void allocate_from_pool(BufferPool& pool); void release_to_pool(BufferPool& pool); + void release_memory(BufferPool& pool); void set_external(void* ptr); }; @@ -676,6 +677,9 @@ class CactusGraph { void release_weight_pages(size_t node_id); void prefetch_weight_pages(size_t node_id); void release_all_weight_pages(); + void release_runtime_buffers(); + void clear_buffer_pool(); + void retain_outputs(const std::vector& node_ids); size_t persistent(size_t source_node); bool is_populated(size_t persistent_node_id) const; @@ -715,6 +719,7 @@ class CactusGraph { std::unordered_set persistent_node_ids_; std::unordered_set populated_node_ids_; std::unordered_set embedded_input_node_ids_; + std::unordered_set retained_output_node_ids_; }; namespace GraphFile { diff --git a/cactus-graph/src/core.cpp b/cactus-graph/src/core.cpp index 691f57b41..78a0a6523 100644 --- a/cactus-graph/src/core.cpp +++ b/cactus-graph/src/core.cpp @@ -160,9 +160,18 @@ void BufferDesc::release_to_pool(BufferPool& pool) { } } +void BufferDesc::release_memory(BufferPool& pool) { + release_to_pool(pool); + data.reset(); + external_data = nullptr; +} + void BufferDesc::set_external(void* ptr) { external_data = ptr; data.reset(); + if (pooled_data) { + delete[] pooled_data; + } pooled_data = nullptr; } @@ -249,3 +258,26 @@ void CactusGraph::allocate_buffers() { } } } + +void CactusGraph::release_runtime_buffers() { + for (auto& node : nodes_) { + if (node->op_type == OpType::INPUT) continue; + if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) continue; + if (persistent_node_ids_.count(node->id)) continue; + node->output_buffer.release_memory(buffer_pool_); + } + buffer_pool_.clear(); + shrink_thread_local_buffers(); +} + +void CactusGraph::clear_buffer_pool() { + buffer_pool_.clear(); +} + +void CactusGraph::retain_outputs(const std::vector& node_ids) { + for (int node_id : node_ids) { + if (node_id >= 0) { + retained_output_node_ids_.insert(static_cast(node_id)); + } + } +} diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 71fdb3449..a7dc86e1f 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -359,6 +359,57 @@ void CactusGraph::execute(const std::string& profile_file) { } }; + auto can_release_node = [&](size_t node_idx) { + const auto& node = nodes_[node_idx]; + if (node->op_type == OpType::INPUT) return false; + if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) return false; + if (persistent_node_ids_.count(node->id)) return false; + if (retained_output_node_ids_.count(node->id)) return false; + return true; + }; + + auto may_alias_input = [](const GraphNode& node) { + return (node.op_type == OpType::SLICE && node.params.axis == 0) || + (node.op_type == OpType::INDEX && node.params.axis == 0); + }; + + auto preallocates_output = [](const GraphNode& node) { + return !((node.op_type == OpType::SLICE && node.params.axis == 0) || + (node.op_type == OpType::INDEX && node.params.axis == 0)); + }; + + std::vector last_use(n, 0); + std::vector use_count(n, 0); + for (size_t i = 0; i < n; ++i) { + for (size_t input_id : nodes_[i]->input_ids) { + auto it = node_index_map_.find(input_id); + if (it != node_index_map_.end()) { + last_use[it->second] = std::max(last_use[it->second], i); + ++use_count[it->second]; + } + } + } + + std::vector keep_until_graph_cleanup(n, false); + for (size_t i = 0; i < n; ++i) { + const auto& node = *nodes_[i]; + if (!may_alias_input(node) || node.input_ids.empty()) continue; + auto it = node_index_map_.find(node.input_ids[0]); + if (it == node_index_map_.end()) continue; + size_t base_idx = it->second; + if (use_count[i] == 0) { + keep_until_graph_cleanup[base_idx] = true; + } else { + last_use[base_idx] = std::max(last_use[base_idx], last_use[i]); + } + } + + std::vector> release_after(n); + for (size_t i = 0; i < n; ++i) { + if (!can_release_node(i) || use_count[i] == 0 || keep_until_graph_cleanup[i]) continue; + release_after[last_use[i]].push_back(i); + } + if (!need_debug) { for (size_t i = 0; i < n; ++i) { auto& node = nodes_[i]; @@ -366,26 +417,24 @@ void CactusGraph::execute(const std::string& profile_file) { if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) { dispatch_node(*node, nodes_, node_index_map_); populated_node_ids_.insert(node->id); + for (size_t release_idx : release_after[i]) { + nodes_[release_idx]->output_buffer.release_memory(pool); + } continue; } - node->output_buffer.allocate_from_pool(pool); + if (preallocates_output(*node)) { + node->output_buffer.allocate_from_pool(pool); + } dispatch_node(*node, nodes_, node_index_map_); trace_nonfinite(i, *node); if (node->op_type == OpType::PERSISTENT) { populated_node_ids_.insert(node->id); } - } - return; - } - - std::vector last_use(n, 0); - for (size_t i = 0; i < n; ++i) { - for (size_t input_id : nodes_[i]->input_ids) { - auto it = node_index_map_.find(input_id); - if (it != node_index_map_.end()) { - last_use[it->second] = std::max(last_use[it->second], i); + for (size_t release_idx : release_after[i]) { + nodes_[release_idx]->output_buffer.release_memory(pool); } } + return; } auto get_env_str = [](const char* name) -> std::string { diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 04fa56207..ad722f7ca 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -146,24 +146,33 @@ void compute_kv_cache_append_node( size_t new_total = current_len + new_seq_len; bool needs_eviction = new_total > window; if (needs_eviction) { - size_t sink_eff = std::min(sink, window); - if (new_seq_len >= window) { - size_t src_skip = new_seq_len - window; - std::memcpy(fp16_base, source + src_skip * stride, window * stride * sizeof(__fp16)); - } else { - size_t remaining = (new_seq_len + sink_eff < window) - ? window - sink_eff - new_seq_len : 0; - size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; - if (remaining > 0 && shift_src > sink_eff) { - std::memmove( - fp16_base + sink_eff * stride, - fp16_base + shift_src * stride, - remaining * stride * sizeof(__fp16)); + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) { + if (tail_capacity > 0) { + size_t source_offset = new_seq_len - tail_capacity; + std::memcpy( + fp16_base + keep_sink * stride, + source + source_offset * stride, + tail_capacity * stride * sizeof(__fp16)); } - size_t append_offset = window - new_seq_len; - std::memcpy(fp16_base + append_offset * stride, source, new_seq_len * stride * sizeof(__fp16)); + meta->current_seq_len = keep_sink + tail_capacity; + *node.output_buffer.data_as() = static_cast(meta->current_seq_len); + return; } - meta->current_seq_len = window; + + size_t remaining = tail_capacity - new_seq_len; + remaining = std::min(remaining, current_len - keep_sink); + size_t shift_src = current_len - remaining; + if (remaining > 0 && shift_src > keep_sink) { + std::memmove( + fp16_base + keep_sink * stride, + fp16_base + shift_src * stride, + remaining * stride * sizeof(__fp16)); + } + size_t append_offset = keep_sink + remaining; + std::memcpy(fp16_base + append_offset * stride, source, new_seq_len * stride * sizeof(__fp16)); + meta->current_seq_len = append_offset + new_seq_len; } else { std::memcpy(fp16_base + current_len * stride, source, new_seq_len * stride * sizeof(__fp16)); meta->current_seq_len = new_total; @@ -183,37 +192,43 @@ void compute_kv_cache_append_node( bool needs_eviction = new_total > window; if (needs_eviction) { - size_t sink_eff = std::min(sink, window); - if (new_seq_len >= window) { - size_t src_skip = new_seq_len - window; - cactus_quantize_kv_fp16_to_int8( - new_kv.data_as<__fp16>() + src_skip * (kv_heads * hdim), - int8_base, - scale_base, - window, kv_heads, hdim); - } else { - size_t remaining = (new_seq_len + sink_eff < window) - ? window - sink_eff - new_seq_len : 0; - size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; - - if (remaining > 0 && shift_src > sink_eff) { - std::memmove(int8_base + sink_eff * int8_stride, - int8_base + shift_src * int8_stride, - remaining * int8_stride); - std::memmove(scale_base + sink_eff * scale_stride, - scale_base + shift_src * scale_stride, - remaining * scale_stride * sizeof(float)); + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) { + if (tail_capacity > 0) { + size_t source_offset = new_seq_len - tail_capacity; + cactus_quantize_kv_fp16_to_int8( + new_kv.data_as<__fp16>() + source_offset * int8_stride, + int8_base + keep_sink * int8_stride, + scale_base + keep_sink * scale_stride, + tail_capacity, kv_heads, hdim); } + meta->current_seq_len = keep_sink + tail_capacity; + *node.output_buffer.data_as() = static_cast(meta->current_seq_len); + return; + } - size_t append_offset = window - new_seq_len; - cactus_quantize_kv_fp16_to_int8( - new_kv.data_as<__fp16>(), - int8_base + append_offset * int8_stride, - scale_base + append_offset * scale_stride, - new_seq_len, kv_heads, hdim); + size_t remaining = tail_capacity - new_seq_len; + remaining = std::min(remaining, current_len - keep_sink); + size_t shift_src = current_len - remaining; + + if (remaining > 0 && shift_src > keep_sink) { + std::memmove(int8_base + keep_sink * int8_stride, + int8_base + shift_src * int8_stride, + remaining * int8_stride); + std::memmove(scale_base + keep_sink * scale_stride, + scale_base + shift_src * scale_stride, + remaining * scale_stride * sizeof(float)); } - meta->current_seq_len = window; + size_t append_offset = keep_sink + remaining; + cactus_quantize_kv_fp16_to_int8( + new_kv.data_as<__fp16>(), + int8_base + append_offset * int8_stride, + scale_base + append_offset * scale_stride, + new_seq_len, kv_heads, hdim); + + meta->current_seq_len = append_offset + new_seq_len; } else { cactus_quantize_kv_fp16_to_int8( new_kv.data_as<__fp16>(), @@ -260,9 +275,15 @@ void compute_attention_cached_node( size_t new_seq_len = key_new_buf.total_size / (kv_heads * hdim); size_t history_len = (cache_len >= new_seq_len) ? cache_len - new_seq_len : 0; + bool cache_only_attention = false; size_t position_offset = node.params.position_offset; if (position_offset == std::numeric_limits::max()) { position_offset = history_len; + } else if (position_offset == std::numeric_limits::max() - 1) { + position_offset = (cache_len >= seq_len) ? cache_len - seq_len : 0; + history_len = cache_len; + new_seq_len = 0; + cache_only_attention = true; } if (k_cache_buf.precision == Precision::FP16 || v_cache_buf.precision == Precision::FP16) { @@ -284,19 +305,19 @@ void compute_attention_cached_node( return; } - cactus_attention_hybrid_int8_fp16( - query_buf.data_as<__fp16>(), - cached_keys, - cached_values, - k_scales, - v_scales, - key_new_buf.data_as<__fp16>(), - val_new_buf.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, history_len, seq_len, - num_q_heads, kv_heads, hdim, - node.params.scale, - position_offset, + cactus_attention_hybrid_int8_fp16( + query_buf.data_as<__fp16>(), + cached_keys, + cached_values, + k_scales, + v_scales, + key_new_buf.data_as<__fp16>(), + val_new_buf.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, history_len, cache_only_attention ? 0 : seq_len, + num_q_heads, kv_heads, hdim, + node.params.scale, + position_offset, true, node.params.window_size, KV_QUANT_GROUP_SIZE, diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index 909328e5e..e87fc110b 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -104,6 +104,33 @@ void dispatch_binary_op_f16(OpType op, const __fp16* lhs, const __fp16* rhs, __f } } +static float apply_binary_op_f32(OpType op, float lhs, float rhs) { + switch (op) { + case OpType::ADD: + case OpType::ADD_CLIPPED: + return lhs + rhs; + case OpType::SUBTRACT: + return lhs - rhs; + case OpType::MULTIPLY: + return lhs * rhs; + case OpType::DIVIDE: + return lhs / rhs; + case OpType::NOT_EQUAL: + return lhs != rhs ? 1.0f : 0.0f; + default: + return lhs; + } +} + +void dispatch_binary_op_f32(OpType op, const float* lhs, const float* rhs, float* output, size_t count) { + CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + for (size_t i = start_idx; i < end_idx; ++i) { + output[i] = apply_binary_op_f32(op, lhs[i], rhs[i]); + } + }); +} + void dispatch_unary_op_f16(OpType op, const __fp16* input, __fp16* output, size_t count, float param) { ScalarOpType scalar_op; switch (op) { @@ -154,13 +181,42 @@ void compute_binary_op_node(GraphNode& node, const std::vector(lhs.precision)) + ")"); + if (lhs.precision != rhs.precision) { + throw std::runtime_error("Binary operations require matching precision"); + } + if (lhs.precision != Precision::FP16 && lhs.precision != Precision::FP32) { + throw std::runtime_error("Binary operations only support FP16/FP32 precision"); } if (node.params.broadcast_info.needs_broadcasting) { std::vector lhs_strides = compute_strides(lhs.shape, node.params.broadcast_info.output_shape); std::vector rhs_strides = compute_strides(rhs.shape, node.params.broadcast_info.output_shape); + const auto& output_shape = node.params.broadcast_info.output_shape; + size_t total_elements = 1; + for (size_t dim : output_shape) { + total_elements *= dim; + } + if (lhs.precision == Precision::FP32) { + const float* lhs_data = lhs.data_as(); + const float* rhs_data = rhs.data_as(); + float* out_data = node.output_buffer.data_as(); + CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + std::vector coords(output_shape.size()); + size_t tmp = start_idx; + for (int axis = static_cast(output_shape.size()) - 1; axis >= 0; --axis) { + coords[static_cast(axis)] = tmp % output_shape[static_cast(axis)]; + tmp /= output_shape[static_cast(axis)]; + } + for (size_t linear_idx = start_idx; linear_idx < end_idx; ++linear_idx) { + size_t lhs_idx = broadcast_linear_index(coords, lhs_strides); + size_t rhs_idx = broadcast_linear_index(coords, rhs_strides); + out_data[linear_idx] = apply_binary_op_f32(node.op_type, lhs_data[lhs_idx], rhs_data[rhs_idx]); + increment_broadcast_coords(coords, output_shape); + } + }); + return; + } switch (node.op_type) { case OpType::ADD: @@ -196,11 +252,6 @@ void compute_binary_op_node(GraphNode& node, const std::vector(); const __fp16* rhs_data = rhs.data_as<__fp16>(); __fp16* out_data = node.output_buffer.data_as<__fp16>(); - const auto& output_shape = node.params.broadcast_info.output_shape; - size_t total_elements = 1; - for (size_t dim : output_shape) { - total_elements *= dim; - } CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ELEMENT_WISE, [&](size_t start_idx, size_t end_idx) { std::vector coords(output_shape.size()); @@ -222,9 +273,15 @@ void compute_binary_op_node(GraphNode& node, const std::vector(), - rhs.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); + if (lhs.precision == Precision::FP16) { + dispatch_binary_op_f16(node.op_type, lhs.data_as<__fp16>(), + rhs.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + } else { + dispatch_binary_op_f32(node.op_type, lhs.data_as(), + rhs.data_as(), node.output_buffer.data_as(), + node.output_buffer.total_size); + } } } @@ -329,8 +386,8 @@ void compute_reduce_node(GraphNode& node, const std::vector(axis)); + if (input_buffer.precision == Precision::FP32) { + const float* input = input_buffer.data_as(); + float* output = node.output_buffer.data_as(); + for (size_t outer = 0; outer < dims.outer; ++outer) { + for (size_t inner = 0; inner < dims.inner; ++inner) { + float running = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + size_t flat_index = ((outer * dims.axis_size) + axis_index) * dims.inner + inner; + running += input[flat_index]; + output[flat_index] = running; + } + } + } + return; + } + const __fp16* input = input_buffer.data_as<__fp16>(); __fp16* output = node.output_buffer.data_as<__fp16>(); @@ -355,6 +428,103 @@ void compute_reduce_node(GraphNode& node, const std::vector(); + float* output = node.output_buffer.data_as(); + if (axis == -1) { + float result = 0.0f; + switch (node.op_type) { + case OpType::SUM: + case OpType::MEAN: + for (size_t i = 0; i < input_buffer.total_size; ++i) result += input[i]; + if (node.op_type == OpType::MEAN && input_buffer.total_size > 0) { + result /= static_cast(input_buffer.total_size); + } + output[0] = result; + break; + case OpType::VARIANCE: { + float mean = 0.0f; + for (size_t i = 0; i < input_buffer.total_size; ++i) mean += input[i]; + mean /= static_cast(std::max(input_buffer.total_size, 1)); + float variance = 0.0f; + for (size_t i = 0; i < input_buffer.total_size; ++i) { + float diff = input[i] - mean; + variance += diff * diff; + } + output[0] = variance / static_cast(std::max(input_buffer.total_size, 1)); + break; + } + case OpType::MIN: + result = input_buffer.total_size == 0 ? 0.0f : input[0]; + for (size_t i = 1; i < input_buffer.total_size; ++i) result = std::min(result, input[i]); + output[0] = result; + break; + case OpType::MAX: + result = input_buffer.total_size == 0 ? 0.0f : input[0]; + for (size_t i = 1; i < input_buffer.total_size; ++i) result = std::max(result, input[i]); + output[0] = result; + break; + default: + break; + } + return; + } + + auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); + CactusThreading::parallel_for(dims.outer * dims.inner, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + for (size_t out_idx = start_idx; out_idx < end_idx; ++out_idx) { + size_t outer = out_idx / dims.inner; + size_t inner = out_idx % dims.inner; + size_t base = outer * dims.axis_size * dims.inner + inner; + float result = 0.0f; + switch (node.op_type) { + case OpType::SUM: + case OpType::MEAN: + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + result += input[base + axis_index * dims.inner]; + } + if (node.op_type == OpType::MEAN && dims.axis_size > 0) { + result /= static_cast(dims.axis_size); + } + output[out_idx] = result; + break; + case OpType::VARIANCE: { + float mean = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + mean += input[base + axis_index * dims.inner]; + } + mean /= static_cast(std::max(dims.axis_size, 1)); + float variance = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + float diff = input[base + axis_index * dims.inner] - mean; + variance += diff * diff; + } + output[out_idx] = variance / static_cast(std::max(dims.axis_size, 1)); + break; + } + case OpType::MIN: + result = dims.axis_size == 0 ? 0.0f : input[base]; + for (size_t axis_index = 1; axis_index < dims.axis_size; ++axis_index) { + result = std::min(result, input[base + axis_index * dims.inner]); + } + output[out_idx] = result; + break; + case OpType::MAX: + result = dims.axis_size == 0 ? 0.0f : input[base]; + for (size_t axis_index = 1; axis_index < dims.axis_size; ++axis_index) { + result = std::max(result, input[base + axis_index * dims.inner]); + } + output[out_idx] = result; + break; + default: + break; + } + } + }); + return; + } + if (axis == -1) { switch (node.op_type) { case OpType::SUM: { diff --git a/cactus-graph/src/ops_tensor.cpp b/cactus-graph/src/ops_tensor.cpp index a0dae609f..42f967ce5 100644 --- a/cactus-graph/src/ops_tensor.cpp +++ b/cactus-graph/src/ops_tensor.cpp @@ -136,6 +136,13 @@ void compute_embedding_node(GraphNode& node, const std::vector(); + } else if (indices_buffer.precision == Precision::FP16) { + indices_float.resize(num_indices); + const __fp16* half_indices = indices_buffer.data_as<__fp16>(); + for (size_t i = 0; i < num_indices; i++) { + indices_float[i] = static_cast(half_indices[i]); + } + indices_ptr = indices_float.data(); } else { indices_float.resize(num_indices); const int8_t* int_indices = indices_buffer.data_as(); diff --git a/cactus-graph/tests/test_cache.cpp b/cactus-graph/tests/test_cache.cpp index 38947597f..21cbdca7d 100644 --- a/cactus-graph/tests/test_cache.cpp +++ b/cactus-graph/tests/test_cache.cpp @@ -163,6 +163,32 @@ bool test_kv_cache_append_eviction() { return true; } +bool test_kv_cache_append_full_window_eviction() { + CactusGraph g; + + const size_t kv_heads = 1, head_dim = 16; + const size_t window = 8, sink = 2; + + size_t cache_node = g.kv_cache_state(window, kv_heads, head_dim, window, sink); + + for (int pass = 0; pass < 2; pass++) { + if (pass > 0) g.soft_reset(); + const size_t tokens = window; + const size_t elements = tokens * kv_heads * head_dim; + size_t kv_input = g.input({elements}, Precision::FP16); + std::vector<__fp16> data(elements, static_cast<__fp16>(static_cast(pass + 1))); + g.set_input(kv_input, data.data(), Precision::FP16); + g.kv_cache_append(kv_input, cache_node, window, sink); + g.execute(); + + auto* raw = static_cast(g.get_output(cache_node)); + uint64_t seq = *reinterpret_cast(raw); + if (seq != window) return false; + } + + return true; +} + bool test_attention_cached_basic() { const size_t b = 1, s = 1, h = 2, kv = 2, d = 16; const size_t max_seq = 64; @@ -493,6 +519,7 @@ int main() { runner.run_test("KV Cache Append Basic", test_kv_cache_append_basic()); runner.run_test("KV Cache Append Multiple", test_kv_cache_append_multiple()); runner.run_test("KV Cache Append Eviction", test_kv_cache_append_eviction()); + runner.run_test("KV Cache Append Full Window Eviction", test_kv_cache_append_full_window_eviction()); runner.run_test("Attention Cached Basic", test_attention_cached_basic()); runner.run_test("Attention Cached Multistep", test_attention_cached_multistep()); runner.run_test("KV Cache Invalidate", test_kv_cache_invalidate()); diff --git a/cactus-graph/tests/test_io.cpp b/cactus-graph/tests/test_io.cpp index 826bf54dc..298e6c440 100644 --- a/cactus-graph/tests/test_io.cpp +++ b/cactus-graph/tests/test_io.cpp @@ -75,7 +75,7 @@ bool test_graph_save_load() { size_t input_a = graph.input({2, 3}, Precision::FP16); size_t input_b = graph.input({2, 3}, Precision::FP16); size_t sum_id = graph.add(input_a, input_b); - graph.pow(sum_id, 2.0f); + size_t pow_id = graph.pow(sum_id, 2.0f); graph.save(filename); @@ -88,8 +88,8 @@ bool test_graph_save_load() { return false; } - if (sg.graph_inputs.size() != 2 || sg.graph_inputs[0] != 0 || - sg.graph_inputs[1] != 1) { + if (sg.graph_inputs.size() != 2 || sg.graph_inputs[0] != input_a || + sg.graph_inputs[1] != input_b) { std::cout << "[graph_save_load] unexpected graph_inputs:"; for (uint32_t idx : sg.graph_inputs) { std::cout << " " << idx; @@ -99,7 +99,7 @@ bool test_graph_save_load() { return false; } - if (sg.graph_outputs.size() != 1 || sg.graph_outputs[0] != 3) { + if (sg.graph_outputs.size() != 1 || sg.graph_outputs[0] != pow_id) { std::cout << "[graph_save_load] unexpected graph_outputs:"; for (uint32_t idx : sg.graph_outputs) { std::cout << " " << idx; diff --git a/cactus-graph/tests/test_utils.h b/cactus-graph/tests/test_utils.h index 31bd14945..511fd446e 100644 --- a/cactus-graph/tests/test_utils.h +++ b/cactus-graph/tests/test_utils.h @@ -135,7 +135,7 @@ inline void fill_random_fp16(std::vector<__fp16>& data) { } inline size_t runtime_id_from_serialized_index(uint32_t serialized_index) { - return static_cast(serialized_index) + 1; + return static_cast(serialized_index); } inline bool test_basic_operation(const std::string& op_name, diff --git a/cactus-kernels/src/threading.h b/cactus-kernels/src/threading.h index 665fecc69..8d5c7cdcc 100644 --- a/cactus-kernels/src/threading.h +++ b/cactus-kernels/src/threading.h @@ -240,11 +240,18 @@ namespace CactusThreading { inline bool pin_current_thread_to_cores(const std::vector& cores) { if (cores.empty()) return false; + cpu_set_t current_mask; + const bool has_current_mask = sched_getaffinity(0, sizeof(current_mask), ¤t_mask) == 0; cpu_set_t mask; CPU_ZERO(&mask); + bool selected = false; for (int core : cores) { - CPU_SET(core, &mask); + if (!has_current_mask || CPU_ISSET(core, ¤t_mask)) { + CPU_SET(core, &mask); + selected = true; + } } + if (!selected && has_current_mask) return true; return sched_setaffinity(0, sizeof(mask), &mask) == 0; } #endif diff --git a/cactus/build.sh b/cactus/build.sh index b139b23e0..dc41a98d3 100755 --- a/cactus/build.sh +++ b/cactus/build.sh @@ -32,7 +32,16 @@ echo "Building Cactus library..." cd "$(dirname "$0")/../cactus" -rm -rf build +LOCK_DIR="${TMPDIR:-/tmp}/cactus-build.lock" +while ! mkdir "$LOCK_DIR" 2>/dev/null; do + if [ -f "$LOCK_DIR/pid" ] && ! kill -0 "$(cat "$LOCK_DIR/pid" 2>/dev/null)" 2>/dev/null; then + rm -rf "$LOCK_DIR" + continue + fi + sleep 2 +done +echo "$$" > "$LOCK_DIR/pid" +trap 'rm -rf "$LOCK_DIR"' EXIT mkdir -p build cd build diff --git a/local_integration_tests/CMakeLists.txt b/local_integration_tests/CMakeLists.txt new file mode 100644 index 000000000..317318503 --- /dev/null +++ b/local_integration_tests/CMakeLists.txt @@ -0,0 +1,62 @@ +cmake_minimum_required(VERSION 3.16) +project(CactusLocalIntegrationTests LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +if(NOT CACTUS_REPO_ROOT) + get_filename_component(CACTUS_REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.." ABSOLUTE) +endif() + +set(CACTUS_LIB "${CACTUS_REPO_ROOT}/cactus/build/libcactus.a") +if(NOT EXISTS "${CACTUS_LIB}") + message(FATAL_ERROR "Missing ${CACTUS_LIB}; run ./cactus/build.sh first") +endif() + +add_executable(integration_runner native_runner.cpp) +target_link_libraries(integration_runner PRIVATE "${CACTUS_LIB}") +target_include_directories(integration_runner PRIVATE + "${CACTUS_REPO_ROOT}/cactus-engine" + "${CACTUS_REPO_ROOT}/cactus-engine/src" + "${CACTUS_REPO_ROOT}/cactus-graph" + "${CACTUS_REPO_ROOT}/cactus-kernels" + "${CACTUS_REPO_ROOT}/cactus-kernels/src" + "${CACTUS_REPO_ROOT}/cactus-kernels/libs" +) + +find_package(Threads REQUIRED) +target_link_libraries(integration_runner PRIVATE Threads::Threads) + +if(APPLE) + find_library(COREML_FRAMEWORK CoreML REQUIRED) + find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) + find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) + find_library(SECURITY_FRAMEWORK Security REQUIRED) + find_library(SYSTEMCONFIGURATION_FRAMEWORK SystemConfiguration REQUIRED) + find_library(CFNETWORK_FRAMEWORK CFNetwork REQUIRED) + target_link_libraries(integration_runner PRIVATE + "${ACCELERATE_FRAMEWORK}" + "${COREML_FRAMEWORK}" + "${FOUNDATION_FRAMEWORK}" + "${SECURITY_FRAMEWORK}" + "${SYSTEMCONFIGURATION_FRAMEWORK}" + "${CFNETWORK_FRAMEWORK}" + ) + set(CACTUS_CURL_ROOT "${CACTUS_REPO_ROOT}/cactus-engine/libs/curl") + set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") + if(EXISTS "${CACTUS_CURL_LIBRARY_MACOS}") + target_link_libraries(integration_runner PRIVATE "${CACTUS_CURL_LIBRARY_MACOS}") + target_include_directories(integration_runner PRIVATE "${CACTUS_CURL_ROOT}/include") + else() + find_library(CURL_LIBRARY NAMES curl) + if(CURL_LIBRARY) + target_link_libraries(integration_runner PRIVATE "${CURL_LIBRARY}") + endif() + endif() +else() + find_package(CURL) + if(CURL_FOUND) + target_link_libraries(integration_runner PRIVATE ${CURL_LIBRARIES}) + target_include_directories(integration_runner PRIVATE ${CURL_INCLUDE_DIRS}) + endif() +endif() diff --git a/local_integration_tests/README.md b/local_integration_tests/README.md new file mode 100644 index 000000000..c6442606a --- /dev/null +++ b/local_integration_tests/README.md @@ -0,0 +1,43 @@ +# Local integration tests + +These tests are intentionally local and uncommitted until the suite is ready to become part of the repository. + +## Requirements + +- Run from the repository root. +- Use existing converted model artifacts only. +- Set `CACTUS_INTEGRATION_WEIGHTS` to override model discovery. +- Default discovery checks `../cactus/weights`, then `./weights`. + +## Commands + +```bash +PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -m smoke +PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -m full +PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -k parakeet +``` + +The pytest session builds `./cactus/build.sh` once and compiles `local_integration_tests/.tmp/build/integration_runner`. + +## Coverage + +- Gemma4 E2B: text and multiturn reuse with the matrix Cactus artifact, `gemma-4-e2b-it-sharedkv`. +- Gemma4 E2B: image, audio, and image+audio combined with the latest local multimodal artifact, preferring `gemma-4-e2b-it-alt-fixed-rowtuned-mm`. +- Qwen3-VL: text and long-prompt chunked prefill smoke. +- Qwen3-VL: image with the matrix Cactus artifact, `qwen3-vl-2b-instruct-reconvert`. +- LFM2-VL: text, image, long-prompt chunked prefill smoke. +- Parakeet TDT: file and PCM transcription. +- Whisper small: full-mode file and PCM transcription when a component-backed local bundle is available. + +## Current local result + +```bash +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s +# 7 passed, 2 deselected + +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -k parakeet -q -s +# 1 passed, 8 deselected + +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m full -q -s +# 2 skipped, 7 deselected +``` diff --git a/local_integration_tests/conftest.py b/local_integration_tests/conftest.py new file mode 100644 index 000000000..adcfe06a4 --- /dev/null +++ b/local_integration_tests/conftest.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from typing import Any + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +TMP_DIR = REPO_ROOT / "local_integration_tests" / ".tmp" +ASSET_DIR = REPO_ROOT / "cactus-engine" / "tests" / "assets" + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", "smoke: fast local integration coverage") + config.addinivalue_line("markers", "full: slower coverage for every available key model") + + +def _weights_root() -> Path: + explicit = os.environ.get("CACTUS_INTEGRATION_WEIGHTS") + candidates = [explicit] if explicit else [] + candidates.extend([str(REPO_ROOT.parent / "cactus" / "weights"), str(REPO_ROOT / "weights")]) + for candidate in candidates: + if not candidate: + continue + path = Path(candidate).expanduser().resolve() + if path.exists(): + return path + return Path(candidates[0]).expanduser().resolve() + + +@pytest.fixture(scope="session") +def weights_root() -> Path: + root = _weights_root() + if not root.exists(): + pytest.fail(f"weights root does not exist: {root}") + return root + + +@pytest.fixture(scope="session") +def assets() -> dict[str, Path]: + result = { + "audio": ASSET_DIR / "test.wav", + "image_monkey": ASSET_DIR / "test_monkey.png", + "image_thing": ASSET_DIR / "test_thing.png", + } + missing = [str(path) for path in result.values() if not path.exists()] + if missing: + pytest.fail("missing integration fixtures: " + ", ".join(missing)) + return result + + +@pytest.fixture(scope="session") +def runner() -> Path: + env = os.environ.copy() + env["CACTUS_NO_CLOUD_TELE"] = "1" + subprocess.run([str(REPO_ROOT / "cactus" / "build.sh")], cwd=REPO_ROOT, env=env, check=True) + + build_dir = TMP_DIR / "build" + build_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + "cmake", + "-S", + str(REPO_ROOT / "local_integration_tests"), + "-B", + str(build_dir), + f"-DCACTUS_REPO_ROOT={REPO_ROOT}", + "-DCMAKE_BUILD_TYPE=Release", + ], + cwd=REPO_ROOT, + check=True, + ) + subprocess.run( + ["cmake", "--build", str(build_dir), "--target", "integration_runner", "-j", str(os.cpu_count() or 4)], + cwd=REPO_ROOT, + check=True, + ) + binary = build_dir / "integration_runner" + if not binary.exists(): + pytest.fail(f"integration runner was not built: {binary}") + return binary + + +def _is_transpiled_bundle(path: Path) -> bool: + return (path / "components" / "manifest.json").exists() or (path / "manifest.json").exists() + + +def model_path( + weights_root: Path, + *candidates: str, + required: bool = True, + transpiled: bool = True, + native_config: bool = False, +) -> Path | None: + for name in candidates: + path = weights_root / name + if not path.exists(): + continue + if transpiled and not _is_transpiled_bundle(path): + continue + if native_config and not (path / "config.txt").exists(): + continue + if path.exists(): + return path + if required: + suffixes = [] + if transpiled: + suffixes.append("components/manifest.json") + if native_config: + suffixes.append("config.txt") + suffix = " with " + " and ".join(suffixes) if suffixes else "" + pytest.fail("missing required model artifact" + suffix + "; tried: " + ", ".join(candidates)) + pytest.skip("optional model artifact not available; tried: " + ", ".join(candidates)) + return None + + +def run_runner(runner: Path, *args: object, timeout: int = 180) -> dict[str, Any]: + env = os.environ.copy() + env["CACTUS_NO_CLOUD_TELE"] = "1" + env.pop("CACTUS_KV_CACHE_FP16", None) + completed = subprocess.run( + [str(runner), *(str(arg) for arg in args)], + cwd=REPO_ROOT, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + if completed.returncode != 0: + pytest.fail( + f"integration runner failed with {completed.returncode}\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) + try: + return json.loads(completed.stdout) + except json.JSONDecodeError as exc: + pytest.fail(f"runner did not return JSON: {exc}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}") + + +def assert_completion(result: dict[str, Any]) -> None: + response = str(result.get("response") or result.get("text") or "").strip() + token_count = int(result.get("completion_tokens") or result.get("tokens_predicted") or result.get("decode_tokens") or 0) + assert response or token_count > 0, result + + +def assert_transcript(result: dict[str, Any], fragments: tuple[str, ...] = ("hello", "testing", "voice")) -> None: + transcript = str(result.get("response") or result.get("transcript") or "").strip().lower() + assert len(transcript) > 5, result + assert any(fragment in transcript for fragment in fragments), transcript diff --git a/local_integration_tests/native_runner.cpp b/local_integration_tests/native_runner.cpp new file mode 100644 index 000000000..8cba0ce08 --- /dev/null +++ b/local_integration_tests/native_runner.cpp @@ -0,0 +1,1110 @@ +#include "cactus_engine.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define private public +#include "engine.h" +#undef private +#include "cactus_kernels.h" +#include "stb_image_resize2.h" +#include "utils.h" +#include "wav.h" + +namespace { + +constexpr size_t kResponseBufferSize = 1 << 20; +constexpr uint64_t kFnvOffset = 1469598103934665603ull; +constexpr uint64_t kFnvPrime = 1099511628211ull; + +struct Args { + std::string command; + std::string model; + std::string prompt; + std::string image; + std::string audio; + std::string messages_json; + int max_tokens = 32; +}; + +std::string arg_value(int& i, int argc, char** argv) { + if (i + 1 >= argc) throw std::runtime_error(std::string("missing value for ") + argv[i]); + return argv[++i]; +} + +Args parse_args(int argc, char** argv) { + if (argc < 2) throw std::runtime_error("usage: integration_runner [options]"); + Args args; + args.command = argv[1]; + for (int i = 2; i < argc; ++i) { + std::string flag = argv[i]; + if (flag == "--model") args.model = arg_value(i, argc, argv); + else if (flag == "--prompt") args.prompt = arg_value(i, argc, argv); + else if (flag == "--image") args.image = arg_value(i, argc, argv); + else if (flag == "--audio") args.audio = arg_value(i, argc, argv); + else if (flag == "--messages-json") args.messages_json = arg_value(i, argc, argv); + else if (flag == "--max-tokens") args.max_tokens = std::max(0, std::stoi(arg_value(i, argc, argv))); + else throw std::runtime_error("unknown flag: " + flag); + } + if (args.model.empty()) throw std::runtime_error("--model is required"); + return args; +} + +std::string escape_json(const std::string& value) { + std::ostringstream out; + for (char ch : value) { + switch (ch) { + case '\\': out << "\\\\"; break; + case '"': out << "\\\""; break; + case '\n': out << "\\n"; break; + case '\r': out << "\\r"; break; + case '\t': out << "\\t"; break; + default: + if (static_cast(ch) < 0x20) { + out << "\\u00"; + const char* hex = "0123456789abcdef"; + out << hex[(ch >> 4) & 0x0f] << hex[ch & 0x0f]; + } else { + out << ch; + } + } + } + return out.str(); +} + +template +std::string json_array(const std::vector& values, size_t limit = std::numeric_limits::max()) { + std::ostringstream out; + out << "["; + const size_t n = std::min(values.size(), limit); + for (size_t i = 0; i < n; ++i) { + if (i) out << ","; + out << values[i]; + } + out << "]"; + return out.str(); +} + +std::string json_float_array(const std::vector& values, size_t limit = std::numeric_limits::max()) { + std::ostringstream out; + out << "["; + const size_t n = std::min(values.size(), limit); + for (size_t i = 0; i < n; ++i) { + if (i) out << ","; + if (std::isfinite(values[i])) out << values[i]; + else out << "null"; + } + out << "]"; + return out.str(); +} + +uint64_t fnv1a_bytes(const void* data, size_t size) { + const auto* bytes = static_cast(data); + uint64_t hash = kFnvOffset; + for (size_t i = 0; i < size; ++i) { + hash ^= static_cast(bytes[i]); + hash *= kFnvPrime; + } + return hash; +} + +std::string hex64(uint64_t value) { + std::ostringstream out; + out << std::hex << std::setw(16) << std::setfill('0') << value; + return out.str(); +} + +template +std::vector first_values(const std::vector& values, size_t n) { + return std::vector(values.begin(), values.begin() + std::min(values.size(), n)); +} + +std::vector last_values(const std::vector& values, size_t n) { + if (values.size() <= n) return values; + return std::vector(values.end() - static_cast(n), values.end()); +} + +int run_e002_tokens(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for token probe: " + args.model); + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() ? "Respond briefly." : args.prompt; + message.images.push_back(args.image); + std::vector messages{message}; + + const std::string full_prompt = tokenizer->format_chat_prompt(messages, true); + const std::string no_generation_prompt = tokenizer->format_chat_prompt(messages, false); + const std::vector tokens = tokenizer->encode(full_prompt); + const std::vector prefix_tokens = tokenizer->encode(no_generation_prompt); + const uint32_t image_token_id = model->get_config().image_token_id ? model->get_config().image_token_id : tokenizer->get_image_token_id(); + + size_t image_start = tokens.size(); + size_t image_count = 0; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i] == image_token_id) { + if (image_count == 0) image_start = i; + ++image_count; + } + } + + std::ostringstream out; + out << "{" + << "\"token_count\":" << tokens.size() << "," + << "\"assistant_generation_start\":" << prefix_tokens.size() << "," + << "\"image_token_id\":" << image_token_id << "," + << "\"image_token_start\":" << (image_count ? static_cast(image_start) : -1) << "," + << "\"image_token_count\":" << image_count << "," + << "\"first_20\":" << json_array(first_values(tokens, 20)) << "," + << "\"last_20\":" << json_array(last_values(tokens, 20)) << "," + << "\"input_ids\":" << json_array(tokens) << "," + << "\"prompt_prefix\":\"" << escape_json(full_prompt.substr(0, 320)) << "\"" + << "}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_preprocess(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + cactus::engine::Config config; + if (!config.from_json(args.model + "/config.txt")) { + throw std::runtime_error("failed to read config.txt for preprocess probe: " + args.model); + } + auto prep = cactus::engine::preprocess_gemma4_image(args.image, config); + double sum = 0.0; + float min_v = prep.pixel_values.empty() ? 0.0f : prep.pixel_values[0]; + float max_v = min_v; + for (float v : prep.pixel_values) { + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + size_t valid_positions = 0; + for (size_t i = 0; i + 1 < prep.pixel_position_ids.size(); i += 2) { + if (prep.pixel_position_ids[i] >= 0 && prep.pixel_position_ids[i + 1] >= 0) ++valid_positions; + } + std::vector first_positions; + const size_t position_rows = std::min(16, prep.pixel_position_ids.size() / 2); + first_positions.reserve(position_rows * 2); + for (size_t i = 0; i < position_rows * 2; ++i) first_positions.push_back(prep.pixel_position_ids[i]); + + std::ostringstream out; + out << std::setprecision(9) + << "{" + << "\"pixel_values_shape\":[1," << prep.max_patches << "," << prep.patch_dim << "]," + << "\"pixel_position_ids_shape\":[1," << prep.max_patches << ",2]," + << "\"num_patches\":" << prep.num_patches << "," + << "\"max_patches\":" << prep.max_patches << "," + << "\"patch_dim\":" << prep.patch_dim << "," + << "\"pixel_values_min\":" << min_v << "," + << "\"pixel_values_max\":" << max_v << "," + << "\"pixel_values_mean\":" << (prep.pixel_values.empty() ? 0.0 : sum / static_cast(prep.pixel_values.size())) << "," + << "\"pixel_values_hash\":\"" << hex64(fnv1a_bytes(prep.pixel_values.data(), prep.pixel_values.size() * sizeof(float))) << "\"," + << "\"pixel_position_ids_hash\":\"" << hex64(fnv1a_bytes(prep.pixel_position_ids.data(), prep.pixel_position_ids.size() * sizeof(int64_t))) << "\"," + << "\"valid_position_count\":" << valid_positions << "," + << "\"first_pixel_values\":" << json_array(first_values(prep.pixel_values, 32)) << "," + << "\"first_position_rows_flat\":" << json_array(first_positions) << "," + << "\"pixel_values\":" << json_array(prep.pixel_values) << "," + << "\"pixel_position_ids\":" << json_array(prep.pixel_position_ids) + << "}\n"; + std::cout << out.str(); + return 0; +} + +struct E003Variant { + std::string name; + std::vector resized; + std::vector pixel_values; +}; + +void resize_float_default(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { + std::vector src_float(static_cast(width) * height * 3); + for (size_t i = 0; i < src_float.size(); ++i) src_float[i] = static_cast(raw[i]); + if (!stbir_resize_float_linear(src_float.data(), width, height, 0, resized.data(), target_w, target_h, 0, STBIR_RGB)) { + throw std::runtime_error("stbir_resize_float_linear failed"); + } +} + +void resize_uint8_default(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { + std::vector tmp(static_cast(target_w) * target_h * 3); + if (!stbir_resize_uint8_linear(raw, width, height, 0, tmp.data(), target_w, target_h, 0, STBIR_RGB)) { + throw std::runtime_error("stbir_resize_uint8_linear failed"); + } + for (size_t i = 0; i < tmp.size(); ++i) resized[i] = static_cast(tmp[i]); +} + +void resize_float_triangle(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { + std::vector src_float(static_cast(width) * height * 3); + for (size_t i = 0; i < src_float.size(); ++i) src_float[i] = static_cast(raw[i]); + STBIR_RESIZE resize; + stbir_resize_init(&resize, src_float.data(), width, height, 0, resized.data(), target_w, target_h, 0, STBIR_RGB, STBIR_TYPE_FLOAT); + stbir_set_filters(&resize, STBIR_FILTER_TRIANGLE, STBIR_FILTER_TRIANGLE); + if (!stbir_resize_extended(&resize)) throw std::runtime_error("triangle float resize failed"); +} + +void resize_uint8_triangle(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { + std::vector tmp(static_cast(target_w) * target_h * 3); + STBIR_RESIZE resize; + stbir_resize_init(&resize, raw, width, height, 0, tmp.data(), target_w, target_h, 0, STBIR_RGB, STBIR_TYPE_UINT8); + stbir_set_filters(&resize, STBIR_FILTER_TRIANGLE, STBIR_FILTER_TRIANGLE); + if (!stbir_resize_extended(&resize)) throw std::runtime_error("triangle uint8 resize failed"); + for (size_t i = 0; i < tmp.size(); ++i) resized[i] = static_cast(tmp[i]); +} + +std::vector pack_gemma4_patches(const std::vector& resized, int target_w, int target_h, int patch_size, + size_t max_patches, size_t patch_dim, float rescale_factor) { + const int patch_h = target_h / patch_size; + const int patch_w = target_w / patch_size; + std::vector pixel_values(max_patches * patch_dim, 0.0f); + for (int py = 0; py < patch_h; ++py) { + for (int px = 0; px < patch_w; ++px) { + const size_t patch_idx = static_cast(py) * patch_w + px; + float* dst = pixel_values.data() + patch_idx * patch_dim; + for (int y = 0; y < patch_size; ++y) { + const int img_y = py * patch_size + y; + for (int x = 0; x < patch_size; ++x) { + const int img_x = px * patch_size + x; + const size_t src_off = (static_cast(img_y) * target_w + img_x) * 3; + const size_t dst_off = (static_cast(y) * patch_size + x) * 3; + dst[dst_off + 0] = resized[src_off + 0] * rescale_factor; + dst[dst_off + 1] = resized[src_off + 1] * rescale_factor; + dst[dst_off + 2] = resized[src_off + 2] * rescale_factor; + } + } + } + } + return pixel_values; +} + +void write_variant_json(std::ostream& out, const E003Variant& variant, bool comma) { + double sum = 0.0; + float min_v = variant.pixel_values.empty() ? 0.0f : variant.pixel_values[0]; + float max_v = min_v; + for (float v : variant.pixel_values) { + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + float resized_min = variant.resized.empty() ? 0.0f : variant.resized[0]; + float resized_max = resized_min; + for (float v : variant.resized) { + resized_min = std::min(resized_min, v); + resized_max = std::max(resized_max, v); + } + if (comma) out << ","; + out << "\"" << variant.name << "\":{" + << "\"resized_min\":" << resized_min << "," + << "\"resized_max\":" << resized_max << "," + << "\"first_resized_values\":" << json_array(first_values(variant.resized, 32)) << "," + << "\"pixel_values_min\":" << min_v << "," + << "\"pixel_values_max\":" << max_v << "," + << "\"pixel_values_mean\":" << (variant.pixel_values.empty() ? 0.0 : sum / static_cast(variant.pixel_values.size())) << "," + << "\"first_pixel_values\":" << json_array(first_values(variant.pixel_values, 32)) << "," + << "\"pixel_values\":" << json_array(variant.pixel_values) + << "}"; +} + +std::string precision_name(Precision precision) { + switch (precision) { + case Precision::FP32: return "FP32"; + case Precision::FP16: return "FP16"; + case Precision::INT8: return "INT8"; + default: return "UNKNOWN"; + } +} + +std::string op_type_name(OpType op) { + switch (op) { + case OpType::INPUT: return "INPUT"; + case OpType::PRECISION_CAST: return "PRECISION_CAST"; + case OpType::ADD: return "ADD"; + case OpType::ADD_CLIPPED: return "ADD_CLIPPED"; + case OpType::SUBTRACT: return "SUBTRACT"; + case OpType::MULTIPLY: return "MULTIPLY"; + case OpType::DIVIDE: return "DIVIDE"; + case OpType::ABS: return "ABS"; + case OpType::POW: return "POW"; + case OpType::FLATTEN: return "FLATTEN"; + case OpType::VIEW: return "VIEW"; + case OpType::MATMUL: return "MATMUL"; + case OpType::TRANSPOSE: return "TRANSPOSE"; + case OpType::RESHAPE: return "RESHAPE"; + case OpType::SLICE: return "SLICE"; + case OpType::GATHER: return "GATHER"; + case OpType::EMBEDDING: return "EMBEDDING"; + case OpType::BILINEAR_INTERPOLATION: return "BILINEAR_INTERPOLATION"; + case OpType::SUM: return "SUM"; + case OpType::MEAN: return "MEAN"; + case OpType::VARIANCE: return "VARIANCE"; + case OpType::MIN: return "MIN"; + case OpType::MAX: return "MAX"; + case OpType::CUMSUM: return "CUMSUM"; + case OpType::RMS_NORM: return "RMS_NORM"; + case OpType::ROPE: return "ROPE"; + case OpType::ROPE_GPTJ: return "ROPE_GPTJ"; + case OpType::SOFTMAX: return "SOFTMAX"; + case OpType::ATTENTION: return "ATTENTION"; + case OpType::ATTENTION_INT8_HYBRID: return "ATTENTION_INT8_HYBRID"; + case OpType::REL_POS_BIAS: return "REL_POS_BIAS"; + case OpType::CONV1D_CAUSAL: return "CONV1D_CAUSAL"; + case OpType::CONV1D_K3: return "CONV1D_K3"; + case OpType::CONV1D_K7S3: return "CONV1D_K7S3"; + case OpType::CONV1D: return "CONV1D"; + case OpType::CONV1D_SAME_DEPTHWISE_K9: return "CONV1D_SAME_DEPTHWISE_K9"; + case OpType::CONV1D_POINTWISE: return "CONV1D_POINTWISE"; + case OpType::CONV2D_K3S2P1: return "CONV2D_K3S2P1"; + case OpType::CONV2D_DEPTHWISE_K3S2P1: return "CONV2D_DEPTHWISE_K3S2P1"; + case OpType::CONV2D_POINTWISE_1X1: return "CONV2D_POINTWISE_1X1"; + case OpType::GLU: return "GLU"; + case OpType::BATCHNORM: return "BATCHNORM"; + case OpType::SCALAR_ADD: return "SCALAR_ADD"; + case OpType::SCALAR_SUBTRACT: return "SCALAR_SUBTRACT"; + case OpType::SCALAR_MULTIPLY: return "SCALAR_MULTIPLY"; + case OpType::SCALAR_DIVIDE: return "SCALAR_DIVIDE"; + case OpType::SCALAR_EXP: return "SCALAR_EXP"; + case OpType::SCALAR_SQRT: return "SCALAR_SQRT"; + case OpType::SCALAR_COS: return "SCALAR_COS"; + case OpType::SCALAR_SIN: return "SCALAR_SIN"; + case OpType::SCALAR_LOG: return "SCALAR_LOG"; + case OpType::RELU: return "RELU"; + case OpType::SILU: return "SILU"; + case OpType::GELU: return "GELU"; + case OpType::GELU_ERF: return "GELU_ERF"; + case OpType::SIGMOID: return "SIGMOID"; + case OpType::TANH: return "TANH"; + case OpType::SAMPLE: return "SAMPLE"; + case OpType::CONCAT: return "CONCAT"; + case OpType::CAT: return "CAT"; + case OpType::SCATTER_TOPK: return "SCATTER_TOPK"; + case OpType::TOPK: return "TOPK"; + case OpType::LAYERNORM: return "LAYERNORM"; + case OpType::GROUPNORM: return "GROUPNORM"; + case OpType::MOE_LAYER: return "MOE_LAYER"; + case OpType::INDEX: return "INDEX"; + case OpType::PERSISTENT: return "PERSISTENT"; + case OpType::LSTM_CELL: return "LSTM_CELL"; + case OpType::GATED_DELTANET_DECODE: return "GATED_DELTANET_DECODE"; + case OpType::GATED_DELTANET_PREFILL: return "GATED_DELTANET_PREFILL"; + case OpType::STFT: return "STFT"; + case OpType::ALTUP_PREDICT: return "ALTUP_PREDICT"; + case OpType::ALTUP_CORRECT: return "ALTUP_CORRECT"; + case OpType::GAUSSIAN_TOPK: return "GAUSSIAN_TOPK"; + case OpType::MAXPOOL1D: return "MAXPOOL1D"; + case OpType::BILSTM_SEQUENCE: return "BILSTM_SEQUENCE"; + case OpType::LEAKY_RELU: return "LEAKY_RELU"; + case OpType::CONV2D_K3S1P1: return "CONV2D_K3S1P1"; + case OpType::STATS_POOL: return "STATS_POOL"; + case OpType::WEIGHTED_STATS_POOL: return "WEIGHTED_STATS_POOL"; + case OpType::KV_CACHE_STATE: return "KV_CACHE_STATE"; + case OpType::KV_CACHE_APPEND: return "KV_CACHE_APPEND"; + case OpType::ATTENTION_CACHED: return "ATTENTION_CACHED"; + case OpType::CONV_CACHE_STATE: return "CONV_CACHE_STATE"; + case OpType::CONV_CACHE_APPEND: return "CONV_CACHE_APPEND"; + case OpType::RFFT: return "RFFT"; + case OpType::IRFFT: return "IRFFT"; + case OpType::MEL_FILTER_BANK: return "MEL_FILTER_BANK"; + case OpType::SPECTROGRAM: return "SPECTROGRAM"; + case OpType::IMAGE_PREPROCESS: return "IMAGE_PREPROCESS"; + case OpType::CLAMP: return "CLAMP"; + case OpType::DENSE_MLP_TQ_FUSED: return "DENSE_MLP_TQ_FUSED"; + case OpType::NOT_EQUAL: return "NOT_EQUAL"; + case OpType::SCALAR_NOT_EQUAL: return "SCALAR_NOT_EQUAL"; + } + return "UNKNOWN"; +} + +std::vector bytes_to_floats(const std::vector& bytes, Precision precision) { + std::vector values; + if (precision == Precision::FP32) { + const size_t count = bytes.size() / sizeof(float); + values.resize(count); + const auto* src = reinterpret_cast(bytes.data()); + for (size_t i = 0; i < count; ++i) values[i] = src[i]; + } else if (precision == Precision::FP16) { + const size_t count = bytes.size() / sizeof(__fp16); + values.resize(count); + const auto* src = reinterpret_cast(bytes.data()); + for (size_t i = 0; i < count; ++i) values[i] = static_cast(src[i]); + } else if (precision == Precision::INT8) { + const size_t count = bytes.size(); + values.resize(count); + const auto* src = reinterpret_cast(bytes.data()); + for (size_t i = 0; i < count; ++i) values[i] = static_cast(src[i]); + } + return values; +} + +std::vector buffer_to_floats(const BufferDesc& desc) { + std::vector values; + const void* data = desc.get_data(); + if (!data) return values; + if (desc.precision == Precision::FP32) { + values.resize(desc.byte_size / sizeof(float)); + const auto* src = static_cast(data); + for (size_t i = 0; i < values.size(); ++i) values[i] = src[i]; + } else if (desc.precision == Precision::FP16) { + values.resize(desc.byte_size / sizeof(__fp16)); + const auto* src = static_cast(data); + for (size_t i = 0; i < values.size(); ++i) values[i] = static_cast(src[i]); + } else if (desc.precision == Precision::INT8) { + values.resize(desc.byte_size); + const auto* src = static_cast(data); + for (size_t i = 0; i < values.size(); ++i) values[i] = static_cast(src[i]); + } + return values; +} + +void write_float_feature_json(std::ostream& out, + const std::string& name, + const std::vector& values, + const std::vector& shape, + Precision precision, + bool comma) { + double sum = 0.0; + float min_v = values.empty() ? 0.0f : values[0]; + float max_v = min_v; + double l2 = 0.0; + size_t finite_count = 0; + size_t nonfinite_count = 0; + for (float v : values) { + if (!std::isfinite(v)) { + ++nonfinite_count; + continue; + } + if (finite_count == 0) { + min_v = v; + max_v = v; + } + ++finite_count; + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + l2 += static_cast(v) * static_cast(v); + } + if (comma) out << ","; + out << "\"" << escape_json(name) << "\":{" + << "\"shape\":" << json_array(shape) << "," + << "\"precision\":\"" << precision_name(precision) << "\"," + << "\"count\":" << values.size() << "," + << "\"finite_count\":" << finite_count << "," + << "\"nonfinite_count\":" << nonfinite_count << "," + << "\"min\":" << min_v << "," + << "\"max\":" << max_v << "," + << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," + << "\"l2\":" << std::sqrt(l2) << "," + << "\"first_values\":" << json_float_array(values, 32) << "," + << "\"values\":" << json_float_array(values) + << "}"; +} + +int run_e003_preprocess_variants(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + cactus::engine::Config config; + if (!config.from_json(args.model + "/config.txt")) { + throw std::runtime_error("failed to read config.txt for E003 probe: " + args.model); + } + + const int patch_size = static_cast(config.vision_patch_size ? config.vision_patch_size : 16); + const int pooling_k = static_cast(config.vision_pooling_kernel_size ? config.vision_pooling_kernel_size : 3); + const size_t max_patches = static_cast(config.vision_default_output_length ? config.vision_default_output_length : 280) * pooling_k * pooling_k; + const int side_multiple = pooling_k * patch_size; + const size_t patch_dim = static_cast(3) * patch_size * patch_size; + const float rescale_factor = config.rescale_factor > 0.0f ? config.rescale_factor : (1.0f / 255.0f); + + int width = 0, height = 0, channels = 0; + unsigned char* raw = cactus_image_load(args.image.c_str(), &width, &height, &channels, 3); + if (!raw) throw std::runtime_error("failed to load image for E003 probe: " + args.image); + + const double target_pixels = static_cast(max_patches) * patch_size * patch_size; + const double pixel_count = std::max(1.0, static_cast(width) * static_cast(height)); + const double factor = std::sqrt(target_pixels / pixel_count); + int target_h = static_cast(std::floor(factor * height / side_multiple)) * side_multiple; + int target_w = static_cast(std::floor(factor * width / side_multiple)) * side_multiple; + if (target_h == 0) target_h = side_multiple; + if (target_w == 0) target_w = side_multiple; + + std::vector variants; + for (const std::string& name : {"stb_default_float", "stb_default_uint8", "stb_triangle_float", "stb_triangle_uint8"}) { + E003Variant variant; + variant.name = name; + variant.resized.assign(static_cast(target_w) * target_h * 3, 0.0f); + if (name == "stb_default_float") resize_float_default(raw, width, height, variant.resized, target_w, target_h); + else if (name == "stb_default_uint8") resize_uint8_default(raw, width, height, variant.resized, target_w, target_h); + else if (name == "stb_triangle_float") resize_float_triangle(raw, width, height, variant.resized, target_w, target_h); + else resize_uint8_triangle(raw, width, height, variant.resized, target_w, target_h); + variant.pixel_values = pack_gemma4_patches(variant.resized, target_w, target_h, patch_size, max_patches, patch_dim, rescale_factor); + variants.push_back(std::move(variant)); + } + cactus_image_free(raw); + + std::ostringstream out; + out << std::setprecision(9) + << "{" + << "\"source_width\":" << width << "," + << "\"source_height\":" << height << "," + << "\"target_width\":" << target_w << "," + << "\"target_height\":" << target_h << "," + << "\"patch_size\":" << patch_size << "," + << "\"pooling_kernel_size\":" << pooling_k << "," + << "\"max_patches\":" << max_patches << "," + << "\"patch_dim\":" << patch_dim << "," + << "\"rescale_factor\":" << rescale_factor << "," + << "\"variants\":{"; + for (size_t i = 0; i < variants.size(); ++i) { + write_variant_json(out, variants[i], i > 0); + } + out << "}}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_vision(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for vision probe: " + args.model); + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() + ? "What animal is in this image? Reply in one complete short sentence." + : args.prompt; + message.images.push_back(args.image); + std::vector messages{message}; + std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); + if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { + throw std::runtime_error("chunk prefill path failed for vision probe"); + } + + std::ostringstream out; + out << std::setprecision(9) << "{\"inputs\":{"; + if (model->vision_encoder_) { + for (size_t i = 0; i < model->vision_encoder_->logical_inputs.size(); ++i) { + if (i) out << ","; + const std::string& input_name = model->vision_encoder_->logical_inputs[i]; + const size_t node_id = static_cast(model->vision_encoder_->runtime_input_node_ids[i]); + const auto& desc = model->vision_encoder_->graph->get_output_buffer(node_id); + out << "\"" << escape_json(input_name) << "\":{" + << "\"node_id\":" << node_id << "," + << "\"shape\":" << json_array(desc.shape) << "," + << "\"precision\":\"" << precision_name(desc.precision) << "\"," + << "\"byte_size\":" << desc.byte_size + << "}"; + } + } + out << "},\"features\":{"; + bool first = true; + for (const auto& kv : model->media_features_) { + const std::string& name = kv.first; + Precision precision = model->media_feature_precisions_[name]; + std::vector values = bytes_to_floats(kv.second, precision); + write_float_feature_json(out, name, values, model->media_feature_shapes_[name], precision, !first); + first = false; + } + out << "}}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_vision_node_summary(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for vision node summary: " + args.model); + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() + ? "What animal is in this image? Reply in one complete short sentence." + : args.prompt; + message.images.push_back(args.image); + std::vector messages{message}; + std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); + if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { + throw std::runtime_error("chunk prefill path failed for vision node summary"); + } + if (!model->vision_encoder_ || !model->vision_encoder_->graph) { + throw std::runtime_error("model has no vision encoder graph"); + } + + const auto& graph = *model->vision_encoder_->graph; + std::ostringstream out; + out << std::setprecision(9) << "{\"nodes\":["; + bool first = true; + for (const auto& node_ptr : graph.nodes_) { + const GraphNode& node = *node_ptr; + const auto& desc = node.output_buffer; + const bool interesting = + node.id < 25 || + (node.id >= 660 && node.id <= 710) || + desc.shape == std::vector{1, 2520} || + desc.shape == std::vector{1, 2520, 1} || + desc.shape == std::vector{1, 2520, 2} || + desc.shape == std::vector{1, 2520, 768} || + desc.shape == std::vector{1, 256, 768} || + desc.shape == std::vector{256, 768} || + desc.shape == std::vector{256, 1536}; + if (!interesting) continue; + std::vector values = buffer_to_floats(desc); + double sum = 0.0; + float min_v = values.empty() ? 0.0f : values[0]; + float max_v = min_v; + size_t finite_count = 0; + size_t nonfinite_count = 0; + for (float v : values) { + if (!std::isfinite(v)) { + ++nonfinite_count; + continue; + } + if (finite_count == 0) { + min_v = v; + max_v = v; + } + ++finite_count; + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + if (!first) out << ","; + first = false; + out << "{" + << "\"node_id\":" << node.id << "," + << "\"op\":\"" << op_type_name(node.op_type) << "\"," + << "\"inputs\":" << json_array(node.input_ids) << "," + << "\"shape\":" << json_array(desc.shape) << "," + << "\"precision\":\"" << precision_name(desc.precision) << "\"," + << "\"byte_size\":" << desc.byte_size << "," + << "\"value_count\":" << values.size() << "," + << "\"finite_count\":" << finite_count << "," + << "\"nonfinite_count\":" << nonfinite_count << "," + << "\"min\":" << min_v << "," + << "\"max\":" << max_v << "," + << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," + << "\"first_values\":" << json_float_array(values, 32) << "," + << "\"values\":" << json_float_array(values) + << "}"; + } + out << "]}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_vision_selected_nodes(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for selected vision nodes: " + args.model); + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() + ? "What animal is in this image? Reply in one complete short sentence." + : args.prompt; + message.images.push_back(args.image); + std::vector messages{message}; + std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); + if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { + throw std::runtime_error("chunk prefill path failed for selected vision nodes"); + } + if (!model->vision_encoder_ || !model->vision_encoder_->graph) { + throw std::runtime_error("model has no vision encoder graph"); + } + + const std::vector selected = { + 672, 675, 699, 700, + 752, 753, 756, 757, 789, 792, 793, 825, 828, 829, 846, 847, 874, 875, + 2757, 2758, 2804, 2805, 2806, 2807, 2812, 2813, 2814, 2815, 2816, + }; + const auto& graph = *model->vision_encoder_->graph; + std::ostringstream out; + out << std::setprecision(9) << "{\"nodes\":{"; + bool first = true; + for (size_t node_id : selected) { + auto it = graph.node_index_map_.find(node_id); + if (it == graph.node_index_map_.end()) continue; + const GraphNode& node = *graph.nodes_[it->second]; + const auto& desc = node.output_buffer; + std::vector values = buffer_to_floats(desc); + if (!first) out << ","; + first = false; + out << "\"" << node_id << "\":{" + << "\"node_id\":" << node.id << "," + << "\"op\":\"" << op_type_name(node.op_type) << "\"," + << "\"inputs\":" << json_array(node.input_ids) << "," + << "\"shape\":" << json_array(desc.shape) << "," + << "\"precision\":\"" << precision_name(desc.precision) << "\"," + << "\"values\":" << json_float_array(values) + << "}"; + } + out << "}}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_audio_summary(const Args& args) { + if (args.audio.empty()) throw std::runtime_error("--audio is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for audio summary: " + args.model); + } + AudioFP32 wav = load_wav(args.audio); + std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); + auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); + model->run_audio_encoder(prep.features); + + std::ostringstream out; + out << std::setprecision(9) << "{" + << "\"num_frames\":" << prep.num_frames << "," + << "\"num_soft_tokens\":" << prep.num_soft_tokens << "," + << "\"feature_count\":" << prep.features.size() << "," + << "\"feature_first_values\":" << json_float_array(prep.features, 32) << "," + << "\"media_features\":{"; + bool first_feature = true; + for (const auto& kv : model->media_features_) { + const std::string& name = kv.first; + Precision precision = model->media_feature_precisions_[name]; + std::vector values = bytes_to_floats(kv.second, precision); + double sum = 0.0; + float min_v = values.empty() ? 0.0f : values[0]; + float max_v = min_v; + size_t finite_count = 0; + size_t nonfinite_count = 0; + for (float v : values) { + if (!std::isfinite(v)) { + ++nonfinite_count; + continue; + } + if (finite_count == 0) { + min_v = v; + max_v = v; + } + ++finite_count; + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + if (!first_feature) out << ","; + first_feature = false; + out << "\"" << name << "\":{" + << "\"shape\":" << json_array(model->media_feature_shapes_[name]) << "," + << "\"precision\":\"" << precision_name(precision) << "\"," + << "\"value_count\":" << values.size() << "," + << "\"finite_count\":" << finite_count << "," + << "\"nonfinite_count\":" << nonfinite_count << "," + << "\"min\":" << min_v << "," + << "\"max\":" << max_v << "," + << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," + << "\"first_values\":" << json_float_array(values, 32) << "," + << "\"values\":" << json_float_array(values) + << "}"; + } + out << "}}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_audio_node_summary(const Args& args) { + if (args.audio.empty()) throw std::runtime_error("--audio is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for audio node summary: " + args.model); + } + AudioFP32 wav = load_wav(args.audio); + std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); + auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); + model->run_audio_encoder(prep.features); + if (!model->audio_encoder_ || !model->audio_encoder_->graph) { + throw std::runtime_error("model has no audio encoder graph"); + } + + const auto& graph = *model->audio_encoder_->graph; + std::ostringstream out; + out << std::setprecision(9) << "{\"nodes\":["; + bool first = true; + for (size_t order = 0; order < graph.nodes_.size(); ++order) { + const GraphNode& node = *graph.nodes_[order]; + const auto& desc = node.output_buffer; + const auto& shape = desc.shape; + bool interesting = node.op_type != OpType::INPUT; + if (shape.empty()) interesting = false; + if (!interesting) continue; + + std::vector values = buffer_to_floats(desc); + double sum = 0.0; + float min_v = values.empty() ? 0.0f : values[0]; + float max_v = min_v; + size_t finite_count = 0; + size_t nonfinite_count = 0; + for (float v : values) { + if (!std::isfinite(v)) { + ++nonfinite_count; + continue; + } + if (finite_count == 0) { + min_v = v; + max_v = v; + } + ++finite_count; + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + if (!first) out << ","; + first = false; + out << "{" + << "\"order\":" << order << "," + << "\"node_id\":" << node.id << "," + << "\"op\":\"" << op_type_name(node.op_type) << "\"," + << "\"inputs\":" << json_array(node.input_ids) << "," + << "\"shape\":" << json_array(shape) << "," + << "\"precision\":\"" << precision_name(desc.precision) << "\"," + << "\"value_count\":" << values.size() << "," + << "\"finite_count\":" << finite_count << "," + << "\"nonfinite_count\":" << nonfinite_count << "," + << "\"min\":" << min_v << "," + << "\"max\":" << max_v << "," + << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," + << "\"first_values\":" << json_float_array(values, 16) + << "}"; + } + out << "]}\n"; + std::cout << out.str(); + return 0; +} + +int run_gemma_first_token(const Args& args) { + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for first token probe: " + args.model); + } + if (args.command == "gemma-first-token-media-step") { + model->lm_encoder_ = nullptr; + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() ? "Respond briefly." : args.prompt; + if (!args.image.empty()) message.images.push_back(args.image); + + std::vector audio_features; + size_t audio_num_frames = 0; + if (!args.audio.empty()) { + message.audio.push_back(args.audio); + AudioFP32 wav = load_wav(args.audio); + std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); + auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); + audio_features = std::move(prep.features); + audio_num_frames = prep.num_frames; + message.audio_soft_token_count = prep.num_soft_tokens; + } + + std::vector messages{message}; + std::string prompt = tokenizer->format_chat_prompt(messages, true); + std::vector tokens = tokenizer->encode(prompt); + if (tokens.empty()) throw std::runtime_error("first token probe produced empty prompt"); + std::vector prefill_tokens(tokens.begin(), tokens.end() - 1); + if (!prefill_tokens.empty()) { + std::vector> audio_messages; + if (!audio_features.empty()) audio_messages.push_back(std::move(audio_features)); + model->prefill_with_media(prefill_tokens, args.image.empty() ? std::vector{} : std::vector{args.image}, audio_messages); + } + uint32_t first = model->decode({tokens.back()}, 0.0f, 1.0f, 1); + std::string text = tokenizer->decode({first}); + std::vector> top_logits; + if (model->decoder_ && !model->decoder_->output_node_ids.empty()) { + size_t out_node = static_cast(model->decoder_->output_node_ids[0]); + const auto& desc = model->decoder_->graph->get_output_buffer(out_node); + std::vector logits = buffer_to_floats(desc); + if (!desc.shape.empty() && !logits.empty()) { + size_t vocab = desc.shape.back(); + size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; + size_t row = seq > 0 ? seq - 1 : 0; + size_t start = row * vocab; + size_t end = std::min(start + vocab, logits.size()); + for (size_t i = start; i < end; ++i) top_logits.emplace_back(i - start, logits[i]); + size_t keep = std::min(10, top_logits.size()); + std::partial_sort( + top_logits.begin(), + top_logits.begin() + keep, + top_logits.end(), + [](const auto& a, const auto& b) { return a.second > b.second; }); + top_logits.resize(keep); + } + } + + const uint32_t image_token_id = model->get_config().image_token_id ? model->get_config().image_token_id : tokenizer->get_image_token_id(); + const uint32_t audio_token_id = model->get_config().audio_token_id; + size_t image_token_count = 0; + size_t audio_token_count = 0; + for (uint32_t token : tokens) { + if (image_token_id != 0 && token == image_token_id) ++image_token_count; + if (audio_token_id != 0 && token == audio_token_id) ++audio_token_count; + } + + std::ostringstream out; + out << "{" + << "\"prompt_token_count\":" << tokens.size() << "," + << "\"prefill_token_count\":" << prefill_tokens.size() << "," + << "\"last_prompt_token\":" << tokens.back() << "," + << "\"first_token\":" << first << "," + << "\"first_text\":\"" << escape_json(text) << "\"," + << "\"eos_token\":" << tokenizer->get_eos_token() << "," + << "\"is_eos\":" << (first == tokenizer->get_eos_token() ? "true" : "false") << "," + << "\"image_token_count\":" << image_token_count << "," + << "\"audio_token_id\":" << audio_token_id << "," + << "\"audio_token_count\":" << audio_token_count << "," + << "\"audio_num_frames\":" << audio_num_frames << "," + << "\"top_logits\":["; + for (size_t i = 0; i < top_logits.size(); ++i) { + if (i) out << ","; + out << "{\"token\":" << top_logits[i].first + << ",\"score\":" << top_logits[i].second + << ",\"text\":\"" << escape_json(tokenizer->decode({static_cast(top_logits[i].first)})) << "\"}"; + } + out << "]," + << "\"prompt_prefix\":\"" << escape_json(prompt.substr(0, 320)) << "\"" + << "}\n"; + std::cout << out.str(); + return 0; +} + +std::string build_messages(const Args& args) { + if (!args.messages_json.empty()) return args.messages_json; + std::ostringstream out; + out << "[{\"role\":\"user\",\"content\":\"" << escape_json(args.prompt.empty() ? "Respond briefly." : args.prompt) << "\""; + if (!args.image.empty()) out << ",\"images\":[\"" << escape_json(args.image) << "\"]"; + if (!args.audio.empty()) out << ",\"audio\":[\"" << escape_json(args.audio) << "\"]"; + out << "}]"; + return out.str(); +} + +std::string complete_options(int max_tokens) { + std::ostringstream out; + out << "{" + << "\"max_tokens\":" << max_tokens << "," + << "\"temperature\":0.0," + << "\"top_p\":1.0," + << "\"top_k\":1," + << "\"telemetry_enabled\":false," + << "\"auto_handoff\":false," + << "\"confidence_threshold\":0.0," + << "\"stop_sequences\":[\"<|im_end|>\",\"\"]" + << "}"; + return out.str(); +} + +std::vector read_wav_pcm16(const std::string& path) { + std::ifstream in(path, std::ios::binary); + if (!in) throw std::runtime_error("failed to open wav: " + path); + in.seekg(0, std::ios::end); + std::streamoff size = in.tellg(); + if (size <= 44) throw std::runtime_error("wav file too small: " + path); + in.seekg(44, std::ios::beg); + std::vector pcm(static_cast(size - 44)); + in.read(reinterpret_cast(pcm.data()), static_cast(pcm.size())); + if (!in) throw std::runtime_error("failed to read wav pcm: " + path); + return pcm; +} + +int run_complete(const Args& args) { + cactus_model_t model = cactus_init(args.model.c_str(), nullptr, false); + if (!model) throw std::runtime_error("cactus_init failed for " + args.model); + std::vector response(kResponseBufferSize, 0); + std::string messages = build_messages(args); + std::string options = complete_options(args.max_tokens); + int rc = cactus_complete( + model, + messages.c_str(), + response.data(), + response.size(), + options.c_str(), + nullptr, + nullptr, + nullptr, + nullptr, + 0); + cactus_destroy(model); + std::cout << response.data() << "\n"; + return rc > 0 ? 0 : 2; +} + +int run_transcribe(const Args& args, bool pcm) { + if (args.audio.empty()) throw std::runtime_error("--audio is required"); + cactus_model_t model = cactus_init(args.model.c_str(), nullptr, false); + if (!model) throw std::runtime_error("cactus_init failed for " + args.model); + std::vector response(kResponseBufferSize, 0); + std::vector pcm_data; + const char* audio_path = args.audio.c_str(); + const uint8_t* pcm_buffer = nullptr; + size_t pcm_size = 0; + if (pcm) { + pcm_data = read_wav_pcm16(args.audio); + audio_path = nullptr; + pcm_buffer = pcm_data.data(); + pcm_size = pcm_data.size(); + } + int rc = cactus_transcribe( + model, + audio_path, + nullptr, + response.data(), + response.size(), + "{\"max_tokens\":200,\"telemetry_enabled\":false,\"auto_handoff\":false}", + nullptr, + nullptr, + pcm_buffer, + pcm_size); + cactus_destroy(model); + std::cout << response.data() << "\n"; + return rc > 0 ? 0 : 2; +} + +} // namespace + +int main(int argc, char** argv) { + try { + Args args = parse_args(argc, argv); + if (args.command == "e002-tokens") return run_e002_tokens(args); + if (args.command == "e002-preprocess") return run_e002_preprocess(args); + if (args.command == "e002-vision") return run_e002_vision(args); + if (args.command == "e002-vision-node-summary") return run_e002_vision_node_summary(args); + if (args.command == "e002-vision-selected-nodes") return run_e002_vision_selected_nodes(args); + if (args.command == "e002-audio-summary") return run_e002_audio_summary(args); + if (args.command == "e002-audio-node-summary") return run_e002_audio_node_summary(args); + if (args.command == "gemma-first-token") return run_gemma_first_token(args); + if (args.command == "gemma-first-token-media-step") return run_gemma_first_token(args); + if (args.command == "e003-preprocess-variants") return run_e003_preprocess_variants(args); + if (args.command == "complete") return run_complete(args); + if (args.command == "transcribe") return run_transcribe(args, false); + if (args.command == "transcribe-pcm") return run_transcribe(args, true); + throw std::runtime_error("unknown command: " + args.command); + } catch (const std::exception& exc) { + std::cerr << exc.what() << "\n"; + return 1; + } +} diff --git a/local_integration_tests/test_modalities.py b/local_integration_tests/test_modalities.py new file mode 100644 index 000000000..62251f3e2 --- /dev/null +++ b/local_integration_tests/test_modalities.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from conftest import assert_completion, assert_transcript, model_path, run_runner + + +def long_prompt() -> str: + sentence = "Summarize this integration test context in one concise sentence. " + return sentence * 40 + + +@pytest.mark.smoke +def test_gemma4_e2b_text_image_and_audio(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + text_model = model_path( + weights_root, + "gemma-4-e2b-it-sharedkv", + "gemma-4-e2b-it-sharedkv-c128", + "gemma-4-e2b-it", + native_config=True, + ) + media_model = model_path( + weights_root, + "gemma-4-e2b-it-alt-fixed-rowtuned-mm", + "gemma-4-e2b-it", + native_config=True, + ) + cases = [ + [text_model, "--prompt", "Reply with one short sentence about local inference."], + [media_model, "--prompt", "Describe this image in one short sentence.", "--image", assets["image_monkey"]], + [media_model, "--prompt", "Transcribe or summarize this audio in one short sentence.", "--audio", assets["audio"]], + ] + for case in cases: + model, *args = case + result = run_runner(runner, "complete", "--model", model, "--max-tokens", 24, *args, timeout=240) + assert_completion(result) + + +@pytest.mark.smoke +def test_gemma4_e2b_combined_image_audio( + runner: Path, + weights_root: Path, + assets: dict[str, Path], +) -> None: + model = model_path( + weights_root, + "gemma-4-e2b-it-alt-fixed-rowtuned-mm", + "gemma-4-e2b-it", + native_config=True, + ) + result = run_runner( + runner, + "complete", + "--model", + model, + "--prompt", + "Describe the image and summarize the audio in one short sentence.", + "--image", + assets["image_thing"], + "--audio", + assets["audio"], + "--max-tokens", + 24, + timeout=240, + ) + assert_completion(result) + + +@pytest.mark.smoke +def test_gemma4_e2b_multiturn_text_reuse(runner: Path, weights_root: Path) -> None: + model = model_path( + weights_root, + "gemma-4-e2b-it-sharedkv", + "gemma-4-e2b-it-sharedkv-c128", + "gemma-4-e2b-it", + native_config=True, + ) + messages = [ + {"role": "system", "content": "Be brief."}, + {"role": "user", "content": "Remember the name Henry."}, + {"role": "assistant", "content": "I will remember the name Henry."}, + {"role": "user", "content": "What name should you remember?"}, + ] + result = run_runner( + runner, + "complete", + "--model", + model, + "--messages-json", + json.dumps(messages), + "--max-tokens", + 24, + timeout=240, + ) + assert_completion(result) + + +@pytest.mark.full +def test_gemma4_e4b_all_modalities_when_available(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + model = model_path(weights_root, "gemma-4-e4b-it", required=False, native_config=True) + result = run_runner( + runner, + "complete", + "--model", + model, + "--prompt", + "Describe the image and audio briefly.", + "--image", + assets["image_monkey"], + "--audio", + assets["audio"], + "--max-tokens", + 24, + timeout=360, + ) + assert_completion(result) + + +@pytest.mark.smoke +def test_qwen3_vl_text_and_chunked_prefill(runner: Path, weights_root: Path) -> None: + model = model_path( + weights_root, + "qwen3-vl-2b-instruct-reconvert", + "qwen3-vl-2b-instruct-reconvert-c256", + "qwen3-vl-2b-instruct-reconvert-chunks", + "qwen3-vl-2b-instruct", + "qwen3-vl-2b", + native_config=True, + ) + for prompt in ("Reply with one short sentence about cactus.", long_prompt()): + result = run_runner(runner, "complete", "--model", model, "--prompt", prompt, "--max-tokens", 16, timeout=300) + assert_completion(result) + + +@pytest.mark.smoke +def test_qwen3_vl_image(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + model = model_path( + weights_root, + "qwen3-vl-2b-instruct-reconvert", + "qwen3-vl-2b-instruct-reconvert-c256", + "qwen3-vl-2b-instruct-reconvert-c512", + native_config=True, + ) + result = run_runner( + runner, + "complete", + "--model", + model, + "--prompt", + "Describe this image in one short sentence.", + "--image", + assets["image_monkey"], + "--max-tokens", + 16, + timeout=300, + ) + assert_completion(result) + + +@pytest.mark.smoke +def test_lfm2_vl_text_image_and_chunked_prefill(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + model = model_path( + weights_root, + "lfm2.5-vl-1.6b-retranspile-4096", + "lfm2.5-vl-1.6b-rechunk-c256", + "lfm2.5-vl-1.6b", + "lfm2.5-vl-1.6b-rechunk-c128", + "lfm2-vl-450m", + native_config=True, + ) + for prompt, image in ( + ("Reply with one short sentence about local model testing.", None), + ("Describe this image in one short sentence.", assets["image_thing"]), + (long_prompt(), None), + ): + args: list[object] = ["complete", "--model", model, "--prompt", prompt, "--max-tokens", 16] + if image is not None: + args.extend(["--image", image]) + result = run_runner(runner, *args, timeout=300) + assert_completion(result) + + +@pytest.mark.smoke +def test_parakeet_tdt_file_and_pcm_transcription(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + model = model_path( + weights_root, + "parakeet-tdt-0.6b-v3-transpiled", + "parakeet-tdt-0.6b-v3", + native_config=True, + ) + file_result = run_runner(runner, "transcribe", "--model", model, "--audio", assets["audio"], timeout=240) + pcm_result = run_runner(runner, "transcribe-pcm", "--model", model, "--audio", assets["audio"], timeout=240) + assert_transcript(file_result) + assert_transcript(pcm_result) + + +@pytest.mark.full +def test_whisper_small_file_and_pcm_transcription_when_available( + runner: Path, + weights_root: Path, + assets: dict[str, Path], +) -> None: + model = model_path(weights_root, "whisper-small", required=False, native_config=True) + file_result = run_runner(runner, "transcribe", "--model", model, "--audio", assets["audio"], timeout=300) + pcm_result = run_runner(runner, "transcribe-pcm", "--model", model, "--audio", assets["audio"], timeout=300) + assert_transcript(file_result) + assert_transcript(pcm_result) diff --git a/local_investigations/gemma4_multimodal/BACKGROUND.md b/local_investigations/gemma4_multimodal/BACKGROUND.md new file mode 100644 index 000000000..8a7cf2031 --- /dev/null +++ b/local_investigations/gemma4_multimodal/BACKGROUND.md @@ -0,0 +1,53 @@ +# Background + +## Local artifact inventory + +Multimodal Gemma candidates: +- `gemma-4-e2b-it` +- `gemma-4-e2b-it-alt-fixed-rowtuned-mm` +- `gemma-4-e2b-it-fresh-mm` + +Text-only/shared-KV Gemma candidates: +- `gemma-4-e2b-it-sharedkv` +- `gemma-4-e2b-it-sharedkv-c128` +- `gemma-4-e2b-it-sharedkv-c256` +- `gemma-4-e2b-it-sharedkv-c512` + +The shared-KV bundles do not include `vision_encoder` or `audio_encoder`, so they are not valid vision/audio quality candidates. + +## Multimodal component shape facts + +Both local multimodal Gemma E2B bundles point at `google/gemma-4-E2B-it` and use the same local HF snapshot. + +The two older multimodal bundles point at missing HF snapshot `b324173c7d5721c2baba7f3b17b3b9b3d34ab1e9`. + +The fresh bundle points at current cached HF snapshot `905e84b50c4d2a365ebde34e685027578e6728db`. + +Component order: +- `audio_encoder` +- `vision_encoder` +- `lm_encoder` +- `decoder_prefill_chunk` +- `lm_encoder_step` +- `lm_encoder_media_step` +- `decoder_step` + +Important logical inputs: +- `vision_encoder`: `pixel_values`, `pixel_position_ids` +- `audio_encoder`: `input_features`, `input_features_mask` +- `lm_encoder`: `input_ids`, `image_features`, `audio_features` +- `lm_encoder_media_step`: `inputs_embeds`, `input_ids`, `position_ids` + +## Controls + +The paired audio created from `../cactus/what_is_in_this_image.mp3` was converted to RIFF/WAV because `cactus_complete` currently rejects MP3 with `Not RIFF`. + +Parakeet transcribes the converted WAV as: + +```text +What is in this image? +``` + +Qwen3-VL identifies `test_monkey.png` as a proboscis monkey, so the asset itself is a good vision control. + +HF reference on the current cached snapshot identifies the same image with the same general prompt as an orangutan. HF also works when using the Cactus-native expanded image prompt/tensors after renaming `pixel_position_ids` to HF's `image_position_ids`. diff --git a/local_investigations/gemma4_multimodal/EXPERIMENTS.md b/local_investigations/gemma4_multimodal/EXPERIMENTS.md new file mode 100644 index 000000000..0e5f36f6d --- /dev/null +++ b/local_investigations/gemma4_multimodal/EXPERIMENTS.md @@ -0,0 +1,80 @@ +# Experiments + +## E1: Paired audio validation + +Purpose: +- Verify that the new audio fixture says what the user intended before using it for Gemma mixed testing. + +Inputs: +- `../cactus/what_is_in_this_image.mp3` +- Converted WAV: `local_integration_tests/.tmp/what_is_in_this_image.wav` +- Parakeet model: `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` + +Observed: + +```text +What is in this image? +``` + +Interpretation: +- The audio fixture is good. +- The native completion path currently requires WAV input, not MP3. + +## E2: Gemma and Qwen image controls + +Purpose: +- Confirm whether the monkey image is a valid vision test and whether the Gemma failure is model/path-specific. + +Observed: +- Qwen3-VL identifies the image as a proboscis monkey. +- Gemma multimodal artifacts fail to identify the monkey in image-only prompts. + +Interpretation: +- The asset is a valid vision control. +- The failure is specific to Gemma artifact/code path behavior, not the image itself. + +## E2b: HF reference with Cactus-native prompt style + +Purpose: +- Check whether Cactus-native expanded media prompts and native image tensors are themselves valid. + +Observed: +- HF current snapshot with normal processor chat template answered: `The animal in this image is an orangutan.` +- HF current snapshot with Cactus-native expanded prompt/tensors also answered: `The animal in this image is an orangutan.` + +Interpretation: +- Prompt style and native image tensor preparation are not the high-level cause. +- The remaining fault is in the transpiled/C++ component execution path or artifact lowering/runtime behavior. + +## E3: Force Gemma media through media-step path + +Purpose: +- Test whether Gemma image-only quality fails because arbitrary prompts are incorrectly routed through the static `lm_encoder` chunk path. + +Change tested: +- Temporarily disabled chunk media prefill for Gemma so `prefill_with_media` used the `lm_encoder_media_step` loop. + +Observed: +- Gemma image-only monkey prompt produced malformed/gibberish output instead of identifying the image. +- Repeating the same temporary routing change with `gemma-4-e2b-it-fresh-mm` also produced malformed/gibberish output. + +Interpretation: +- The simple routing fallback is not valid. +- The investigation should compare actual component contracts and traced prompt assumptions before changing routing further. + +## E4: Fresh reconvert artifact + +Purpose: +- Determine whether stale artifacts caused the Gemma vision failure. + +Artifact: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +Observed: +- Fresh artifact points at current cached snapshot `905e84b50c4d2a365ebde34e685027578e6728db`. +- Image-only monkey prompt produced: `The image shows a close-up of a human nose.` +- Similar prompt produced: `The image shows a close-up of a fawn's nose.` + +Interpretation: +- Reconvert was necessary to remove stale-artifact uncertainty. +- Reconvert alone does not solve Gemma native vision quality. diff --git a/local_investigations/gemma4_multimodal/FIXES.md b/local_investigations/gemma4_multimodal/FIXES.md new file mode 100644 index 000000000..854813740 --- /dev/null +++ b/local_investigations/gemma4_multimodal/FIXES.md @@ -0,0 +1,52 @@ +# Fixes + +## F1: Audio incremental decode prompt replay + +Status: applied before this investigation workspace was created + +Rationale: +- `cactus_complete` passed the entire processed prompt into `decode_with_audio` on each generated token. +- `decode_with_audio` currently delegates to incremental `decode`, so this replayed the prompt into KV cache repeatedly. + +Observed effect: +- Mixed-media generation speed improved. +- Gemma audio semantics remained poor, so this was not the full root cause. + +## F2: Audio encoder mask valid-frame handling + +Status: applied before this investigation workspace was created + +Rationale: +- The audio frontend pads feature buffers to graph capacity. +- The mask should mark only valid frames, not all padded frames. + +Observed effect: +- Build and smoke tests still pass. +- Gemma audio semantics remained poor, so this was not the full root cause. + +## F3: Force Gemma media-step fallback + +Status: reverted + +Rationale: +- Tested whether the static chunk `lm_encoder` path was the immediate cause of Gemma image-only failure. + +Observed effect: +- Output became gibberish. +- The change was reverted. +- The same experiment was repeated with the fresh artifact and was still gibberish. + +## F4: Fresh Gemma4 multimodal reconvert + +Status: created for investigation + +Artifact: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +Rationale: +- Existing multimodal manifests referenced a missing older HF snapshot. +- The current cached HF snapshot produces a valid HF reference answer. + +Observed effect: +- Fresh artifact no longer depends on the missing snapshot. +- Native Cactus vision quality is still wrong, so further runtime/component investigation is required. diff --git a/local_investigations/gemma4_multimodal/HYPOTHESES.md b/local_investigations/gemma4_multimodal/HYPOTHESES.md new file mode 100644 index 000000000..c90541186 --- /dev/null +++ b/local_investigations/gemma4_multimodal/HYPOTHESES.md @@ -0,0 +1,71 @@ +# Hypotheses + +## H1: Current multimodal artifact is stale or malformed + +Status: supported but not sufficient + +Evidence so far: +- Both local multimodal Gemma E2B bundles have full multimodal components. +- Both reference the same HF snapshot. +- Shared-KV artifacts are text-only and should not be used for vision/audio quality. +- The older multimodal bundles reference a missing local HF snapshot. +- A fresh bundle was reconverted from the current cached HF snapshot. +- The fresh bundle improves the failure mode but still does not match HF vision quality. + +Next evidence needed: +- Keep using `gemma-4-e2b-it-fresh-mm` as the best Cactus artifact for further debugging. +- Do not assume reconversion alone fixes the native runtime path. + +## H2: Gemma native image preprocessing diverges from Python/HF preprocessing + +Status: open + +Evidence so far: +- C++ and Python both appear to use Gemma4-style resizing, patching, and position IDs. +- No tensor-level diff has been run yet. + +Next evidence needed: +- Compare `pixel_values` and `pixel_position_ids` between C++ and Python for the same image. + +## H3: Gemma multimodal chunk prefill uses a static traced merge plan that is invalid for arbitrary prompts + +Status: partially supported, simple fallback refuted + +Evidence so far: +- The `lm_encoder` component was traced with a fixed processor prompt embedded in the graph metadata. +- The Python adapter builds a native merge plan during transpile from the example `input_ids`. +- C++ currently sends arbitrary prompt tokens through this `lm_encoder` for media prefill. +- Gemma image-only fails even though Qwen identifies the same image. + +Experiment: +- Route Gemma media prefill through `lm_encoder_media_step` instead of the chunk `lm_encoder` path, then re-run image-only quality probes. + +Result: +- The simple dynamic fallback produced obvious gibberish for image-only monkey prompts. +- The same result reproduced with the fresh artifact. +- This suggests `lm_encoder_media_step` is not a drop-in prefill replacement as currently wired, or it needs additional exact contract handling. +- The static-merge concern remains plausible, but the fix is not simply disabling chunk prefill for all Gemma media. + +## H5: HF/native prompt and preprocessing are sound; the divergence is in transpiled/C++ execution + +Status: supported + +Evidence so far: +- HF with the normal processor chat template identifies the monkey image as an orangutan. +- HF with Cactus-native expanded prompt/tensors also identifies the image as an orangutan when `pixel_position_ids` are passed as HF `image_position_ids`. +- Fresh Cactus native with the same image says the image is a human/fawn nose. + +Next evidence needed: +- Compare component-level outputs or implement/test the intended dynamic Gemma media chunk path using `lm_encoder_text_chunk` and `lm_encoder_media_chunk`. + +## H4: Audio frontend or audio mask is wrong + +Status: open + +Evidence so far: +- The completion decode loop previously replayed the whole prompt for audio generation; this was fixed. +- The audio encoder mask was changed to mark only valid frames. +- Audio-only still does not show evidence of understanding the spoken content. + +Next evidence needed: +- Work on audio only after vision-only is explained or fixed. diff --git a/local_investigations/gemma4_multimodal/PLAN.md b/local_investigations/gemma4_multimodal/PLAN.md new file mode 100644 index 000000000..5fc3bdad7 --- /dev/null +++ b/local_investigations/gemma4_multimodal/PLAN.md @@ -0,0 +1,29 @@ +# Plan + +## Current phase + +Isolate Gemma vision-only correctness before treating mixed image+audio as a product behavior. + +## Method + +Use a hypothesis-driven loop: +1. Record a hypothesis in `HYPOTHESES.md`. +2. Run the smallest experiment that can support or refute it. +3. Record exact output and interpretation in `EXPERIMENTS.md`. +4. If supported, make the smallest fix and document it in `FIXES.md`. +5. Re-run the focused quality probe and the nearest regression suite. + +## Layer order + +1. Artifact validity and freshness. +2. Prompt/token construction. +3. Image preprocessing. +4. Vision encoder output. +5. LM media merge behavior. +6. Decoder/cache behavior. +7. Audio preprocessing and encoder behavior. +8. Mixed image+audio behavior. + +## Current decision + +Investigate whether Gemma multimodal prefill is incorrectly using the static chunk `lm_encoder` path for arbitrary prompts instead of the dynamic `lm_encoder_media_step` path. diff --git a/local_investigations/gemma4_multimodal/README.md b/local_investigations/gemma4_multimodal/README.md new file mode 100644 index 000000000..0cde142cb --- /dev/null +++ b/local_investigations/gemma4_multimodal/README.md @@ -0,0 +1,23 @@ +# Gemma4 multimodal investigation + +Local, uncommitted workspace for investigating Gemma4 native vision/audio quality. + +Current status: +- Vision-only Gemma native does not reliably identify `cactus-engine/tests/assets/test_monkey.png`. +- Audio-only Gemma native does not yet show evidence that it is using the audio content. +- Mixed image+audio with `what_is_in_this_image.wav` currently fails because the underlying unimodal paths are not known-good. +- Qwen3-VL and Parakeet are useful controls: Qwen identifies the monkey, and Parakeet transcribes the paired audio. + +Primary artifacts: +- `../cactus/weights/gemma-4-e2b-it` +- `../cactus/weights/gemma-4-e2b-it-alt-fixed-rowtuned-mm` +- `../cactus/weights/qwen3-vl-2b-instruct-reconvert` +- `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` + +Read in order: +1. `BACKGROUND.md` +2. `PLAN.md` +3. `HYPOTHESES.md` +4. `EXPERIMENTS.md` +5. `FIXES.md` +6. `SUMMARY.md` diff --git a/local_investigations/gemma4_multimodal/SUMMARY.md b/local_investigations/gemma4_multimodal/SUMMARY.md new file mode 100644 index 000000000..e39aa24e0 --- /dev/null +++ b/local_investigations/gemma4_multimodal/SUMMARY.md @@ -0,0 +1,9 @@ +# Summary + +Investigation is in progress. + +Current leading hypothesis: +- Gemma media prefill has a component-contract mismatch or lowering/runtime mismatch after HF-native prompt/preprocessing, likely around transpiled media encoder/LM merge execution. + +Current next step: +- Compare component-level outputs or implement/test the intended dynamic Gemma media chunk path using `lm_encoder_text_chunk` and `lm_encoder_media_chunk`. diff --git a/local_investigations/gemma4_multimodal_v2/00_INDEX.md b/local_investigations/gemma4_multimodal_v2/00_INDEX.md new file mode 100644 index 000000000..71ed21eb7 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/00_INDEX.md @@ -0,0 +1,48 @@ +# Gemma4 Multimodal V2 Index + +Status: Gemma4 image-only, audio-only, and mixed image+audio now pass local end-to-end probes after the dynamic audio merge fix; Qwen, Parakeet, policy, and smoke controls pass + +Active phase: post-fix review / audit. Do not commit or push until the user reviews. + +Default artifact under investigation: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +Key controls: +- HF reference snapshot: `905e84b50c4d2a365ebde34e685027578e6728db` +- Qwen image control: `../cactus/weights/qwen3-vl-2b-instruct-reconvert` +- Parakeet audio control: `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` + +Read order: +1. `09_STATUS_FOR_NEXT_AGENT.md` +2. `10_CONTINUATION_PLAN.md` +3. `01_BACKGROUND.md` +4. `02_QUESTIONS.md` +5. `03_HYPOTHESES.md` +6. `04_EXPERIMENTS.md` +7. `05_DECISIONS.md` +8. `06_FIX_LOG.md` +9. `07_ACCEPTANCE.md` +10. `08_HANDOFF.md` + +Latest proven artifacts: + +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json`: image preprocessing parity recorded as `status: pass`, max abs diff `4.987010955859184e-07`. +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json`: latest recorded vision encoder output is FP16-close to the Python/HF reference for `pooler_output`, cosine `0.9999955754245847`, mean abs diff `0.0015996834738693023`. +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json`: image-only Cactus Gemma answered `The animal in the image is a golden monkey.` +- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json`: after the Gemma4 audio bias FP16 fix, first 64 audio rows are FP16-close to Python, cosine `0.9978210926`, mean abs diff `0.00645508`. +- `artifacts/E029_first_token_after_dynamic_merge/`: dynamic merge first-token probes now produce normal text starts, audio `Please`, mixed `This`. +- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json`: audio-only Cactus Gemma responds coherently: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json`: mixed Cactus Gemma answers: `This image is of a golden monkey.` +- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json`: image-only still answers: `The animal in the image is a golden monkey.` +- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json`: Qwen image control identifies a proboscis monkey. +- `artifacts/E034_parakeet_regression_after_dynamic_merge/`: Parakeet file and PCM controls both transcribe `What is in this image?` +- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json`: full emitted audio tensor matches the E023 post-bias tensor exactly, max diff `0`. + +Latest test commands: +- `source ../cactus/venv/bin/activate && ./cactus/build.sh` +- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q` -> `16 passed` +- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s` -> `7 passed, 2 deselected` + +Continuation contract: +- Follow `10_CONTINUATION_PLAN.md`. +- The acceptance probes now pass locally. Next work is review, cleanup, and any additional audit the user requests. diff --git a/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md b/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md new file mode 100644 index 000000000..a4336c5f5 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md @@ -0,0 +1,48 @@ +# Background + +## Stable facts copied from V1 + +The older local Gemma multimodal bundles are: +- `../cactus/weights/gemma-4-e2b-it` +- `../cactus/weights/gemma-4-e2b-it-alt-fixed-rowtuned-mm` + +Both older bundles reference missing HF snapshot: +- `b324173c7d5721c2baba7f3b17b3b9b3d34ab1e9` + +The fresh investigation bundle is: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +The fresh bundle references current cached HF snapshot: +- `905e84b50c4d2a365ebde34e685027578e6728db` + +The shared-KV Gemma artifacts are text-only for this investigation because they do not include `vision_encoder` or `audio_encoder`. + +## Confirmed prior observations + +- HF reference identifies `test_monkey.png` as an orangutan/monkey-class animal. +- HF still works when using Cactus-native expanded image prompt/tensors with `pixel_position_ids` passed as HF `image_position_ids`. +- Fresh Cactus native Gemma originally failed image quality in E001, describing the monkey image as a human/fawn nose. +- Later current artifacts record image preprocessing parity and image-only generation success: + - `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` + - `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` +- Later artifacts record audio-only and mixed image+audio passing after the audio bias FP16 fix and dynamic Gemma4 audio merge: + - `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` + - `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` +- Qwen3-VL identifies the same image as a proboscis monkey. +- Parakeet transcribes the paired audio as: `What is in this image?` +- Direct MP3 input to native completion fails with `Not RIFF`; the paired audio must be converted to WAV. + +## Local paths + +Image: +- `cactus-engine/tests/assets/test_monkey.png` + +Audio: +- Source MP3: `../cactus/what_is_in_this_image.mp3` +- Converted WAV when needed: `local_integration_tests/.tmp/what_is_in_this_image.wav` + +Harness: +- `local_investigations/gemma4_multimodal_v2/scripts/run_probe.py` + +Artifacts: +- `local_investigations/gemma4_multimodal_v2/artifacts/` diff --git a/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md b/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md new file mode 100644 index 000000000..46965b759 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md @@ -0,0 +1,41 @@ +# Questions + +## Q1: Where is the first bad boundary? + +Status: answered for vision and audio in the current artifact set + +Candidate boundaries: +- Prompt/token IDs and media spans. +- Image preprocessing tensors. +- Vision encoder output. +- LM media merge output. +- Decoder first-token logits. +- Autoregressive decode loop. + +Resolution: +- For vision, E002/E003/E014/E015 have current artifacts and image-only is recorded as passing. +- For audio, E022/E023 localized and fixed the audio encoder bias issue, and E027/E029 localized and fixed the static media merge issue. + +## Q2: Is the fresh Gemma artifact good enough to be the default investigation artifact? + +Status: yes for current investigation + +Known evidence: +- It points at the current cached HF snapshot. +- E001 confirmed HF on the same snapshot worked while Cactus image-only originally did not. +- Later artifacts record Cactus image-only success on the fresh artifact. +- E030/E031 record audio-only and mixed image+audio passing on the fresh artifact. + +Next falsifier: +- If a fresh rerun contradicts E030/E031, inspect the changed artifact/runtime state before changing prompts. + +## Q3: Is the previous media-step fallback worth pursuing? + +Status: no as a wholesale fallback; dynamic media-step use inside chunked prefill is accepted. + +Known evidence: +- Forcing all Gemma media through the old step fallback produced gibberish on mixed prompts. +- The accepted fix uses `lm_encoder_media_step` only to assemble real prompt-position outputs before chunked decoder prefill. + +Next falsifier: +- Only revisit after proving the correct `lm_encoder_media_chunk`/step contract and how cache copies should work. diff --git a/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md b/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md new file mode 100644 index 000000000..46eacf99b --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md @@ -0,0 +1,66 @@ +# Hypotheses + +## H1: Artifact staleness caused the original failure + +Status: supported but not sufficient; not the active boundary + +Evidence: +- Older multimodal artifacts reference a missing HF snapshot. +- Fresh artifact was reconverted from current cached snapshot. +- Fresh artifact originally failed native Gemma vision quality in E001. +- Later artifacts show image-only now passes after targeted fixes. + +Decision: +- Use the fresh artifact for V2 unless a stronger artifact is produced for a specific experiment. + +## H2: Prompt style or native image preprocessing is invalid + +Status: refuted for current image-only path + +Evidence: +- HF works with the normal processor prompt. +- HF also works with Cactus-native expanded prompt/tensors. +- E001 stores both records under `artifacts/E001/`. +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records image preprocessing parity as `status: pass`. +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records image-only output `The animal in the image is a golden monkey.` + +Resolution: +- Audio and mixed image+audio now pass in E030/E031; do not reopen image preprocessing without contradictory fresh evidence. + +## H3: The transpiled/C++ component path diverges after preprocessing + +Status: historically true for vision and audio; currently fixed in local artifacts + +Evidence: +- Vision had later graph/runtime/conversion fixes recorded in `06_FIX_LOG.md`. +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` records FP16-close vision output agreement. +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` records a substantial emitted `audio_features` mismatch against the Python native-like reference. +- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` records audio features as FP16-close after the audio bias fix. + +Next test: +- Rerun E030/E031 if the artifact or runtime changes. + +## H4: The C++ media merge/cache path is using the wrong component contract + +Status: supported and fixed for Gemma4 prompts containing audio + +Evidence: +- Simple `lm_encoder_media_step` fallback produced gibberish. +- Fresh artifact includes `lm_encoder_text_chunk`, `lm_encoder_media_step`, and `lm_encoder_media_chunk`, but C++ currently uses the monolithic `lm_encoder` chunk path for media. +- E025/E026 previously returned empty responses after the audio encoder fix, which made media merge/cache the next boundary. +- After the audio encoder fix, E027 showed the static `lm_encoder` chunk path selected `` for audio/mixed first tokens. +- E029/E030/E031 show dynamic prompt-position media merge fixes first-token and generation behavior. + +Next test: +- Keep dynamic merge active for Gemma4 prompts with audio and rerun regressions after runtime changes. + +## H5: Audio should remain blocked until vision is explained + +Status: retired + +Evidence: +- Vision-only has a recorded passing artifact in `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json`. +- Audio-only and mixed image+audio pass in E030/E031. + +Next test: +- None unless a fresh rerun contradicts the passing artifacts. diff --git a/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md b/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md new file mode 100644 index 000000000..37ef0129d --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md @@ -0,0 +1,356 @@ +# Experiments + +## Experiment format + +Each experiment must include: +- ID +- Question +- Setup +- Command +- Raw result path +- Interpretation +- Next decision + +## E001: Baseline ledger + +Question: +- Can the harness reproduce the known HF/Cactus/control outputs and store them in a comparable format? + +Setup: +- Fresh Gemma artifact: `../cactus/weights/gemma-4-e2b-it-fresh-mm` +- Image: `cactus-engine/tests/assets/test_monkey.png` +- Audio: converted `what_is_in_this_image.wav` + +Command: + +```bash +python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case all-baseline --experiment E001 +``` + +Raw result path: +- `artifacts/E001/` + +Interpretation: +- Completed. +- HF Gemma normal prompt: `The animal in this image is an orangutan.` +- HF Gemma Cactus-native expanded prompt/tensors: `The animal in this image is an orangutan.` +- Cactus fresh Gemma native: `The image shows a close-up of a human nose.` +- Cactus Qwen control identifies the proboscis monkey. +- Parakeet file and PCM controls both transcribe: `What is in this image?` + +Next decision: +- Move to E002 component-boundary comparison. +- The next probe must localize the first bad boundary after HF-native prompt/preprocessing, not attempt another runtime fix. + +## E002: Component-boundary comparison + +Question: +- At which component boundary does fresh Cactus native first diverge from the HF/Python reference? + +Setup: +- Same artifact, prompt, and image as E001. +- Compare in this order: input IDs/media spans, image tensors, vision encoder output, LM merge output, decoder first-token logits. + +Command: +```bash +source ../cactus/venv/bin/activate +./cactus/build.sh +python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case e002-boundary --experiment E002 +``` + +Raw result path: +- `artifacts/E002/` +- `artifacts/E002/token_spans.json` +- `artifacts/E002/image_preprocess_compare.json` +- `artifacts/E002/hf_native_inputs.json` +- `artifacts/E002/cactus_token_probe.json` +- `artifacts/E002/cactus_preprocess.json` + +Interpretation: +- Completed 2026-05-20. +- Gate A passed: HF/Python native and C++ Cactus native `input_ids` match exactly. +- Gate A image span also matches exactly: token count `282`, assistant generation start `279`, image token id `258880`, image start `5`, image token count `256`. +- Gate B failed at `pixel_values`. +- `pixel_values` shape matches: `[1, 2520, 768]`. +- `pixel_position_ids` shape and values match, with valid position count `2304`. +- `pixel_values` differ immediately at element `0`; max absolute difference is `0.1298471341896057`, mean absolute difference is `0.002023158910580417`. +- HF/Python pixel range is `[0.0, 1.0]`; C++ pixel range is `[-0.0213023126, 1.00900519]`. +- The first divergent boundary is Gemma4 native image preprocessing, before the vision encoder. + +Next decision: +- Stop E002 at Gate B for now, then continue with E003. Do not run vision encoder, LM merge, or decoder comparison until preprocessing parity is fixed or a stricter preprocessing reference proves the current C++ tensor is acceptable. +- The next fix should be scoped to `preprocess_gemma4_image` parity with `_prepare_gemma4_native_image_tensors`, including resize/interpolation and numeric range behavior. +- This is not final acceptance; the mission continues until Gemma image-only, audio-only, and mixed image+audio pass end to end. + +Superseded by later artifacts below: +- E003 / `E002_after_E003_pillow_resize` records preprocessing parity as passing. +- E015 records image-only generation as passing. +- Audio-only and mixed image+audio are now the active failures. + +## Current status correction: later artifacts after E002 + +This section records the newer artifacts currently present in the worktree. It does not erase E001/E002 history; it updates the active boundary for future agents. + +## E003 / E002_after_E003_pillow_resize: Image preprocessing parity + +Question: +- Does the C++ Gemma4 image preprocessing output match the Python/HF native reference after the Pillow-compatible resize path? + +Raw result paths: +- `artifacts/E003/preprocess_operation_compare.json` +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` + +Interpretation: +- Completed in current artifacts. +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. +- `pixel_values` max abs diff is `4.987010955859184e-07`. +- `pixel_values` mean abs diff is `2.2151478535820782e-07`. +- HF/Python range is `[0.0, 1.0]`; C++ range is `[0, 1]`. + +Next decision: +- Treat image preprocessing as fixed unless a fresh rerun contradicts this artifact. +- Continue to the vision encoder boundary. + +## E014: Latest recorded vision encoder comparison + +Question: +- Does the Cactus `vision_encoder` output match the Python/HF reference closely enough after the recorded graph/runtime/conversion fixes? + +Raw result path: +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` + +Interpretation: +- The latest recorded `pooler_output` comparison has matching shape `[256, 1536]`. +- Cosine similarity is `0.9999955754245847`. +- Max abs diff is `0.03515437499999985`. +- Mean abs diff is `0.0015996834738693023`. +- This is recorded as FP16-close. The artifact does not by itself prove every later LM merge/logit boundary, so do not overstate it as full multimodal acceptance. + +Next decision: +- Use image-only generation quality as the next acceptance check for the vision path. + +## E015: Gemma image-only quality check after vision fixes + +Question: +- Does fresh Cactus Gemma image-only generation identify `test_monkey.png` as a monkey-class animal? + +Raw result path: +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` + +Interpretation: +- Cactus returned success. +- Response: `The animal in the image is a golden monkey.` +- `prefill_tokens: 282`. +- `decode_tokens: 10`. +- This satisfies the image-only semantic acceptance class recorded in `07_ACCEPTANCE.md`. + +Next decision: +- Treat Gemma image-only as passing in the current artifact set. +- Continue to Gemma audio-only and mixed image+audio. + +## E016 / E020: Gemma audio-only quality checks + +Question: +- Does fresh Cactus Gemma audio-only generation produce coherent text using the paired audio prompt? + +Raw result paths: +- `artifacts/E016_gemma_audio_after_vision_gate/cactus_gemma_audio.json` +- `artifacts/E020_gemma_audio_after_audio_retranspile/cactus_gemma_audio.json` + +Interpretation: +- Both recorded Cactus runs returned success but no generated text. +- E020 response is empty: `response: ""`. +- E020 has `prefill_tokens: 83` and `decode_tokens: 0`. +- Retranspiling the audio encoder in the recorded E019/E020 sequence did not resolve audio-only generation. + +Next decision: +- Audio-only remains an active failure. +- Continue with audio boundary comparison before changing prompts or decoder settings. + +## E017: Gemma mixed image+audio quality check + +Question: +- Does fresh Cactus Gemma mixed image+audio generation answer the spoken question about `test_monkey.png`? + +Raw result path: +- `artifacts/E017_gemma_mixed_after_vision_gate/cactus_gemma_mixed.json` + +Interpretation: +- Cactus returned success but no generated text. +- Response is empty: `response: ""`. +- `prefill_tokens: 349`. +- `decode_tokens: 0`. + +Next decision: +- Mixed image+audio remains an active failure. +- Do not treat image-only success as full acceptance. + +## E018 / E019: Audio summary before and after audio retranspile + +Question: +- Do the recorded Cactus audio preprocessing and emitted audio feature summaries change after retranspiling the audio encoder? + +Raw result paths: +- `artifacts/E018_audio_summary/audio_summary.json` +- `artifacts/E019_audio_summary_after_retranspile/audio_summary.json` + +Interpretation: +- Both artifacts record `num_frames: 255`, `num_soft_tokens: 64`, and `feature_count: 32640`. +- Both artifacts record emitted `audio_features` shape `[1, 753, 1536]` with precision `FP16`. +- The first emitted values are unchanged across E018 and E019. +- These summaries are not enough to prove audio preprocessing parity because they only include prefixes and aggregate statistics. + +Next decision: +- Use full-tensor comparison before selecting an audio fix. + +## E021: Full Cactus-vs-Python audio feature comparison + +Question: +- Do the Cactus emitted `audio_features` match the Python native-like audio feature reference? + +Raw result paths: +- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` + +Interpretation: +- `cactus_audio_summary.json` now includes full `audio_features.values`. +- Cactus and Python output shapes match: `[1, 753, 1536]`. +- Python prepared feature shape is `[1, 3012, 128]`; Python input feature mask sum is `255`. +- For the first 64 rows, cosine similarity is `0.36957991123199463`, max abs diff is `2.796870708465576`, and mean abs diff is `0.16708998382091522`. +- For all 753 rows, cosine similarity is `0.7203282713890076`, max abs diff is `2.796870708465576`, and mean abs diff is `0.10658295452594757`. +- The `python_audio_token_count` field in this artifact is not used for conclusions because the comparison script counted a token id that is not validated here. + +Next decision: +- The current best proven active boundary is the audio encoder output / emitted audio feature boundary. +- Next work should localize the first divergent internal audio encoder operation or weight. Do not infer a specific internal cause from E021 alone. + +## E022 / E023 / E024: Audio encoder localization and FP16 bias fix + +Question: +- What caused the emitted Gemma4 `audio_features` mismatch from E021? + +Raw result paths: +- `artifacts/E022_audio_node_localization/` +- `artifacts/E023_audio_after_fp16_bias/` +- `artifacts/E024_retranspile_audio_fp16_bias/` + +Interpretation: +- E022 localized the first important divergence to the audio output projection bias handling. +- The active artifact had stored `audio_output_proj.bias` as an INT8 fallback, which made Cactus add raw values like `[1, -2, 0, 70]` instead of HF FP16-like bias values. +- After changing Gemma4 audio bias policy to FP16 and updating the active artifact, E023 records first 64 audio rows as FP16-close to Python: cosine `0.9978210926`, mean abs diff `0.00645508`. + +Next decision: +- Keep Gemma4 audio bias tensors as FP16. +- Continue to generation and first-token probes because audio-only and mixed still returned empty strings after E023. + +## E025 / E026 / E027: Post-audio-encoder generation and first-token failure + +Question: +- After fixing the audio encoder boundary, why do audio-only and mixed generation still fail? + +Raw result paths: +- `artifacts/E025_gemma_audio_after_fp16_bias/cactus_gemma_audio.json` +- `artifacts/E026_gemma_mixed_after_fp16_bias/cactus_gemma_mixed.json` +- `artifacts/E027_first_token_after_audio_encoder_fix/` + +Interpretation: +- Audio-only and mixed still returned empty responses after the audio encoder fix. +- First-token probes showed the static chunk path selected `` for both audio-only and mixed prompts, unlike HF references (`The` / `This` starts). +- Forcing the old media-step path improved audio-only first token to `Please`, but mixed produced whitespace, so wholesale fallback was not acceptable. +- The static `lm_encoder` graph was captured with stale media span assumptions and could insert audio at the wrong positions/counts for the current prompt. + +Next decision: +- Use a Gemma4-audio-only dynamic media merge inside the chunked prefill path instead of relying on stale static `lm_encoder` merge captures. + +## E029: First-token probes after dynamic merge + +Question: +- Does the dynamic Gemma4 audio merge fix first-token behavior for audio-only and mixed prompts? + +Raw result path: +- `artifacts/E029_first_token_after_dynamic_merge/` + +Interpretation: +- Audio-only first token is `Please`, with normal text alternatives in top logits. +- Mixed image+audio first token is `This`, matching the HF-style first-token direction. +- Neither probe selects ``, ``, or whitespace-only as the top token. + +Next decision: +- Continue to full generation quality checks. + +## E030: Gemma audio-only after dynamic merge + +Question: +- Does audio-only Gemma use the paired spoken prompt coherently? + +Raw result path: +- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` + +Interpretation: +- Cactus generated: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- This is coherent with the audio prompt saying `what is in this image` and no image being supplied. +- `decode_tokens: 21`. + +Next decision: +- Audio-only quality gate passes. Continue to mixed image+audio. + +## E031: Gemma mixed image+audio after dynamic merge + +Question: +- Does mixed Gemma answer the spoken question about `test_monkey.png`? + +Raw result path: +- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` + +Interpretation: +- Cactus generated: `This image is of a golden monkey.` +- This satisfies the mixed acceptance class. +- `decode_tokens: 8`. + +Next decision: +- Mixed gate passes. Run regressions. + +## E032 / E033 / E034 / E035: Regression checks after dynamic merge + +Question: +- Did the dynamic merge fix regress image-only Gemma, Qwen, Parakeet, or the audio encoder boundary? + +Raw result paths: +- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` +- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` +- `artifacts/E034_parakeet_regression_after_dynamic_merge/` +- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` + +Interpretation: +- Gemma image-only still generated: `The animal in the image is a golden monkey.` +- Qwen image control generated a proboscis-monkey answer. +- Parakeet file and PCM controls both transcribed: `What is in this image?` +- E035 shows the emitted Cactus audio tensor is unchanged from E023, with full-tensor max diff `0`. +- Policy tests passed: `16 passed`. +- Smoke tests passed: `7 passed, 2 deselected`. + +Next decision: +- The planned local acceptance gates pass. Keep the working tree uncommitted for user audit. + +## E036 / E037 / E038 / E039 / E040: Cleanup verification probes + +Question: +- After ignoring/removing generated local outputs, do the required quality probes still regenerate needed files and pass? + +Raw result paths: +- `artifacts/E036_cleanup_gemma_image/cactus_gemma_image.json` +- `artifacts/E037_cleanup_gemma_audio/cactus_gemma_audio.json` +- `artifacts/E038_cleanup_gemma_mixed/cactus_gemma_mixed.json` +- `artifacts/E039_cleanup_qwen_image/cactus_qwen_image.json` +- `artifacts/E040_cleanup_parakeet/` + +Interpretation: +- Gemma image-only still generated: `The animal in the image is a golden monkey.` +- Gemma audio-only still generated: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- Gemma mixed still generated: `This image is of a golden monkey.` +- Qwen image still generated a proboscis-monkey answer. +- Parakeet file and PCM still transcribed: `What is in this image?` +- The ignored `.tmp` directory was recreated successfully by the harness. + +Next decision: +- Cleanup did not regress the acceptance probes or remove benchmark/test harness source. diff --git a/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md b/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md new file mode 100644 index 000000000..922eef32b --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md @@ -0,0 +1,118 @@ +# Decisions + +## D1: Do not use older multimodal Gemma artifacts as the default + +Decision: +- Use `gemma-4-e2b-it-fresh-mm` for V2 baseline work. + +Reason: +- Older artifacts reference a missing HF snapshot. + +## D2: Do not continue the simple media-step fallback + +Decision: +- Do not disable Gemma chunk prefill wholesale to force `lm_encoder_media_step`. + +Reason: +- It produced gibberish on old and fresh artifacts. + +## D3: Treat quality probes as acceptance, not root-cause tools + +Decision: +- Use quality probes to confirm behavior, but debug via component boundary comparison. + +Reason: +- Human text output identified the issue but did not localize the first bad boundary. + +## D4: First bad boundary is Gemma4 image preprocessing + +Decision: +- Treat E002 Gate B as the first proven divergence. +- Do not debug vision encoder, LM merge, or decoder logits until Gemma4 native preprocessing matches the HF/Python reference or a stronger reference refutes the HF/Python preprocessing tensor as the correct target. + +Reason: +- E002 Gate A proves prompt `input_ids` and image token spans match exactly between HF/Python native and C++ Cactus native. +- E002 Gate B proves `pixel_position_ids` match, but `pixel_values` differ immediately and outside tiny tolerance. +- The mismatch exists before any graph runtime, media merge, cache, or decoder behavior is involved. + +## D5: Localization is not acceptance + +Decision: +- Boundary localization is now treated only as a checkpoint. +- The investigation remains active until Gemma4 native works end to end across image-only, audio-only, and mixed image+audio, with controls and smoke still passing. + +Reason: +- The user explicitly raised the acceptance bar from identifying the cause to making the model fully work end to end. + +## D6: Image preprocessing is no longer the active boundary + +Decision: +- Treat Gemma4 image preprocessing parity as fixed in the current artifact set. + +Reason: +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. +- The recorded `pixel_values` max abs diff is `4.987010955859184e-07`, and mean abs diff is `2.2151478535820782e-07`. + +## D7: Image-only generation is currently accepted, but full multimodal is not + +Decision: +- Treat Gemma image-only as passing in the current artifact set. +- Do not treat the investigation as complete. + +Reason: +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records the response `The animal in the image is a golden monkey.` +- `07_ACCEPTANCE.md` allows monkey/orangutan/proboscis monkey class answers for image-only. +- Audio-only and mixed image+audio still return empty responses in the latest recorded artifacts. + +## D8: Superseded historical audio / mixed failure + +Decision: +- This was the correct decision at E021, but it is superseded by D10 and D11 after E023/E029/E030/E031. +- Do not return to prompt tweaks or decoder parameters unless a fresh rerun contradicts the passing artifacts. + +Reason: +- `artifacts/E020_gemma_audio_after_audio_retranspile/cactus_gemma_audio.json` records `response: ""` and `decode_tokens: 0`. +- `artifacts/E017_gemma_mixed_after_vision_gate/cactus_gemma_mixed.json` records `response: ""` and `decode_tokens: 0`. +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` records matching Cactus/Python audio feature output shapes but poor first-64-row agreement: cosine `0.36957991123199463`, mean abs diff `0.16708998382091522`. + +## D9: E021 localizes a boundary, not an internal cause + +Decision: +- Use E021 only to justify investigating inside the audio encoder path. +- Do not claim a specific audio op, weight, runtime kernel, or conversion policy is wrong until a smaller comparison identifies it. + +Reason: +- E021 compares full emitted audio features against a Python native-like reference and proves a mismatch at that boundary. +- It does not identify the first divergent node inside the audio encoder. + +## D10: Gemma4 audio bias tensors must stay FP16 + +Decision: +- Keep Gemma4 audio component bias tensors as FP16 fallback tensors. + +Reason: +- E022 localized the audio encoder divergence to the output projection bias handling. +- E023 shows the FP16 bias policy brings the emitted audio features close to the Python reference. +- Storing these biases as INT8 caused Cactus to add raw integer-like values instead of the expected FP16 bias values. + +## D11: Use dynamic Gemma4 merge for prompts with audio + +Decision: +- For Gemma4 prompts with audio, do not use the static full `lm_encoder` merge graph. +- Dynamically walk the actual prompt tokens, route text through `lm_encoder_step`, route image/audio token rows through `lm_encoder_media_step`, then feed the assembled tensors to `decoder_prefill_chunk`. + +Reason: +- E027 showed the static chunk path selected `` for audio-only and mixed prompts after the audio encoder boundary was fixed. +- The static `lm_encoder` graph can encode stale media span assumptions from capture time. +- E029/E030/E031 show the dynamic merge produces normal first tokens and passing end-to-end audio/mixed outputs. + +## D12: Do not carry vision changes from `justin/v2-cpp-fix` + +Decision: +- Treat `origin/justin/v2-cpp-fix` as already absorbed baseline infrastructure. +- Carry over only the later dynamic Gemma4 audio merge idea from `77f808ed`, adapted to the current branch structure. + +Reason: +- The latest `justin/v2-cpp-fix` commit is already in this branch line. +- Vision already has passing local artifacts. +- The relevant audio fix pattern is the dynamic media walk, not a vision-path change. diff --git a/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md b/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md new file mode 100644 index 000000000..5f2ec53d7 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md @@ -0,0 +1,156 @@ +# Fix Log + +Current status: V2 has recorded fixes in the worktree. Do not use the old `No V2 fixes yet` state. + +Rules: +- A code change must cite a supported hypothesis and experiment ID. +- A failed fix must be reverted or explicitly documented as intentionally retained. +- Each fix record must include before/after quality and nearest regression result. + +Historical fixes from before V2: +- Audio incremental decode changed to feed only the last token. +- Audio encoder mask changed to mark valid frames only. +- Fresh Gemma artifact reconverted as `gemma-4-e2b-it-fresh-mm`. +- Simple media-step fallback was tested and reverted. + +## Local investigation tooling + +2026-05-20: +- Added local-only E002 probe commands to `local_integration_tests/native_runner.cpp`: + - `e002-tokens` + - `e002-preprocess` +- Updated `local_integration_tests/CMakeLists.txt` so the local runner can include internal engine headers. +- Extended `scripts/run_probe.py` with `--case e002-boundary`. +- These are investigation artifacts only, not runtime fixes. + +## F1: Gemma4 image preprocessing parity + +Supported by: +- `artifacts/E003/preprocess_operation_compare.json` +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` + +Change: +- `cactus-engine/src/engine_image.cpp` now has a Pillow-compatible uint8 bilinear resize path used by `preprocess_gemma4_image`. + +Before: +- `artifacts/E002/image_preprocess_compare.json` recorded `pixel_values` max abs diff `0.1298471341896057` and mean abs diff `0.002023158910580417`. +- C++ pixel range extended outside `[0, 1]`. + +After: +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. +- `pixel_values` max abs diff is `4.987010955859184e-07`. +- `pixel_values` mean abs diff is `2.2151478535820782e-07`. + +## F2: Vision graph/runtime/transpile fixes recorded by later artifacts + +Supported by: +- `artifacts/E006_gateC_after_fp16_embedding_indices/vision_encoder_compare.json` +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` + +Changes visible in the worktree: +- `cactus-graph/src/ops_tensor.cpp` handles FP16 embedding indices in `compute_embedding_node`. +- `cactus-graph/src/ops_math.cpp` adds FP32 binary and reduction support used by scalar/math lowering paths. +- `python/cactus/transpile/lower.py` preserves tensor dtype in `_ensure_scalar_math_tensor`. +- `python/cactus/transpile/runtime_compat.py` preserves tensor dtype for scalar ops, `abs`, and `pow`. +- `python/cactus/transpile/model_adapters.py` updates Gemma4 native-like vision pooling/projection behavior. + +Evidence: +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` records FP16-close `pooler_output` agreement: cosine `0.9999955754245847`, mean abs diff `0.0015996834738693023`. +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records image-only output: `The animal in the image is a golden monkey.` + +Limit: +- This does not prove audio-only or mixed image+audio acceptance. + +## F3: Gemma4 vision projection conversion policy and active artifact adjustment + +Supported by: +- `artifacts/E013_gateC_after_fp16_projection_graph/vision_encoder_compare.json` +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` + +Changes visible in the worktree: +- `python/cactus/convert/model_adapters/policy.py` makes `model.embed_vision.embedding_projection.weight` an FP16 fallback with reason `vision embedding projection scale-sensitive`. +- `python/cactus/convert/model_adapters/adapters.py` no longer overrides that policy back to CQ. +- `python/cactus/convert/model_adapters/naming.py` and `python/cactus/convert/cactus_adapters/tensor_io.py` remove `embed_vision_proj` from Gemma4 divide-scale handling. +- `python/cactus/convert/tests/test_policy.py` now expects FP16 for the Gemma4 vision projection policy. + +Evidence: +- The latest recorded vision comparison improves to FP16-close agreement in `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json`. + +## F4: Audio full-tensor diagnostic instrumentation + +Supported by: +- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` + +Change: +- `local_integration_tests/native_runner.cpp` now emits full `audio_features.values` for `e002-audio-summary`. + +Evidence: +- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` contains full emitted `audio_features.values`. +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` compares those values against Python native-like audio features. + +Limit: +- This is diagnostic instrumentation only. +- It proves the emitted audio feature boundary mismatches, but not which internal audio encoder operation or weight is the first cause. + +## F5: Gemma4 audio bias FP16 policy and artifact adjustment + +Supported by: +- `artifacts/E022_audio_node_localization/` +- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` +- `artifacts/E024_retranspile_audio_fp16_bias/` + +Changes: +- `python/cactus/convert/model_adapters/policy.py` keeps Gemma4 audio component bias tensors in FP16. +- `python/cactus/convert/tests/test_policy.py` expects FP16 for Gemma4 audio bias policy. +- The active fresh Gemma artifact has an FP16 `audio_output_proj.bias` and regenerated `audio_encoder` graph artifacts. + +Before: +- E021 first 64 emitted audio rows had cosine `0.3695799112` and mean abs diff `0.16708998` versus Python. + +After: +- E023 first 64 emitted audio rows have cosine `0.9978210926` and mean abs diff `0.00645508` versus Python. + +## F6: Dynamic Gemma4 audio/media merge for chunked prefill + +Supported by: +- `artifacts/E027_first_token_after_audio_encoder_fix/` +- `artifacts/E029_first_token_after_dynamic_merge/` +- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` +- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` + +Changes: +- `cactus-engine/src/model.cpp` now builds Gemma4 LM encoder outputs dynamically for prompts containing audio. +- The dynamic path walks actual prompt tokens, uses `lm_encoder_step` for text, uses `lm_encoder_media_step` for image/audio token rows, and feeds the assembled outputs into `decoder_prefill_chunk`. +- `cactus-engine/src/engine.h` declares the dynamic helper. +- The existing static `lm_encoder` merge path remains for non-audio cases. + +Before: +- E025/E026 still returned empty audio-only and mixed responses after the audio encoder fix. +- E027 first-token probes selected `` for both audio-only and mixed prompts on the static chunk path. + +After: +- E029 first-token probes produce audio `Please` and mixed `This`. +- E030 audio-only produces coherent text responding to the spoken image question. +- E031 mixed image+audio produces `This image is of a golden monkey.` +- E032/E033/E034/E035 and smoke/policy tests record no local regression. + +## F7: Cleanup of generated local outputs + +Supported by: +- `artifacts/E036_cleanup_gemma_image/cactus_gemma_image.json` +- `artifacts/E037_cleanup_gemma_audio/cactus_gemma_audio.json` +- `artifacts/E038_cleanup_gemma_mixed/cactus_gemma_mixed.json` +- `artifacts/E039_cleanup_qwen_image/cactus_qwen_image.json` +- `artifacts/E040_cleanup_parakeet/` + +Changes: +- `.gitignore` now ignores local integration `.tmp` output, local Python bytecode, and local investigation artifact directories. +- Removed rebuildable local `.tmp` and `__pycache__` directories. +- Benchmark source/results under `benchmarks/` were not changed. + +Evidence: +- Policy tests passed after cleanup: `16 passed`. +- Smoke tests passed after cleanup: `7 passed, 2 deselected`. +- E036-E040 quality probes passed after cleanup. diff --git a/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md b/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md new file mode 100644 index 000000000..3e6887129 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md @@ -0,0 +1,50 @@ +# Acceptance + +## End-to-end gate + +Gemma4 native multimodal work is accepted only when the full user path works: +- Image-only Gemma correctly identifies `test_monkey.png` as a monkey/orangutan/proboscis monkey class animal. +- Audio-only Gemma shows it uses the paired spoken prompt coherently. +- Mixed image+audio Gemma answers the spoken question about `test_monkey.png` correctly. +- Any component boundary divergence found during debugging has a before/after artifact showing the fix. +- Qwen image control still works. +- Parakeet file and PCM controls still transcribe coherently. +- Smoke tests pass. + +Localization is not acceptance. It is a checkpoint that determines the next fix. + +## Vision gate + +Gemma native vision is accepted only when: +- HF and Cactus use the same image and semantically equivalent prompt. +- The first bad component boundary has been identified and fixed or ruled out. +- Cactus output identifies the monkey image as a monkey/orangutan/proboscis monkey class animal. +- Qwen image control still works. + +## Audio gate + +Gemma audio work starts after vision has a known-good path. A documented vision boundary failure is not enough to accept the mission. + +Audio is accepted only when: +- The paired WAV is verified by Parakeet. +- Gemma audio preprocessing and audio encoder behavior are compared against the Python/HF reference. +- Cactus output shows evidence of using the spoken content. + +## Mixed gate + +Mixed image+audio is accepted only after vision and audio have each passed their gates and the combined path works. + +Mixed image+audio must use the actual paired test assets: +- Image: `cactus-engine/tests/assets/test_monkey.png` +- Audio prompt: `what is in this image` +- Expected answer: a short coherent answer identifying the monkey/orangutan/proboscis monkey class. + +## Regression gate + +After any code fix: + +```bash +source ../cactus/venv/bin/activate +./cactus/build.sh +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s +``` diff --git a/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md b/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md new file mode 100644 index 000000000..9a27aa908 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md @@ -0,0 +1,44 @@ +# Handoff + +Status: local acceptance probes pass; leave uncommitted for user audit. + +Current proven state: +- Gemma4 image preprocessing parity is fixed. +- Gemma4 vision/image-only generation passes. +- Gemma4 audio encoder output is FP16-close to Python after the audio bias FP16 fix. +- Gemma4 audio-only generation is coherent. +- Gemma4 mixed image+audio answers the spoken question about `test_monkey.png` correctly. +- Qwen image, Parakeet file/PCM, policy tests, and smoke tests pass. + +## Key artifacts + +- Image preprocessing: `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` +- Vision encoder: `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` +- Audio encoder after bias fix: `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` +- First-token dynamic merge: `artifacts/E029_first_token_after_dynamic_merge/` +- Gemma audio-only: `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` +- Gemma mixed: `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` +- Gemma image regression: `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` +- Qwen regression: `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` +- Parakeet regression: `artifacts/E034_parakeet_regression_after_dynamic_merge/` +- Audio encoder unchanged after dynamic merge: `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` + +## Latest outputs + +- Gemma image-only: `The animal in the image is a golden monkey.` +- Gemma audio-only: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- Gemma mixed: `This image is of a golden monkey.` +- Qwen image: identifies a proboscis monkey. +- Parakeet file and PCM: `What is in this image?` + +## Tests + +- Build completed with `source ../cactus/venv/bin/activate && ./cactus/build.sh`. +- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q` -> `16 passed`. +- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s` -> `7 passed, 2 deselected`. + +## Review focus + +- Runtime: dynamic Gemma4 audio merge in `cactus-engine/src/model.cpp`. +- Policy: Gemma4 audio bias FP16 handling in `python/cactus/convert/model_adapters/policy.py`. +- Artifact changes: active fresh Gemma audio encoder graph/bias files were adjusted during the investigation. diff --git a/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md b/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md new file mode 100644 index 000000000..da4b6605a --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md @@ -0,0 +1,74 @@ +# Status For Next Agent + +## One-line status + +Gemma4 image-only, audio-only, and mixed image+audio now pass local end-to-end probes after the audio bias FP16 fix and dynamic Gemma4 audio merge. Qwen, Parakeet, policy, and smoke controls also pass. + +## Current working state + +Repo: +- `/Users/noahcylich/Documents/Desert/cactus-v2-chunked-prefill` + +Branch: +- `v2-chunked-prefill` + +Do not commit or push unless the user explicitly asks. The worktree contains unrelated and related tracked changes plus local investigation artifacts. + +Default Gemma artifact: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +HF reference snapshot: +- `/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db` + +Assets: +- Image: `cactus-engine/tests/assets/test_monkey.png` +- Audio WAV: `local_integration_tests/.tmp/what_is_in_this_image.wav` +- Source MP3: `../cactus/what_is_in_this_image.mp3` + +## Fixes applied + +- Gemma4 audio component bias tensors are kept FP16. E023 shows the audio encoder boundary became FP16-close to Python after this. +- Gemma4 prompts containing audio now bypass the stale static full `lm_encoder` merge graph. The runtime dynamically walks the actual token sequence, routes text through `lm_encoder_step`, routes image/audio token rows through `lm_encoder_media_step`, assembles the LM encoder outputs, and feeds `decoder_prefill_chunk`. +- This keeps the existing static path for image-only and non-audio cases. + +## Latest proof artifacts + +- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json`: first 64 audio rows cosine `0.9978210926`, mean abs diff `0.00645508` versus Python. +- `artifacts/E029_first_token_after_dynamic_merge/`: audio first token `Please`, mixed first token `This`; no `` or immediate stop. +- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json`: audio-only response is coherent with the spoken prompt: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json`: mixed response: `This image is of a golden monkey.` +- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json`: image-only response: `The animal in the image is a golden monkey.` +- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json`: Qwen identifies a proboscis monkey. +- `artifacts/E034_parakeet_regression_after_dynamic_merge/`: Parakeet file and PCM both transcribe `What is in this image?` +- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json`: full emitted audio tensor matches E023 exactly, max diff `0`. + +## Latest commands run + +Required build: + +```bash +source ../cactus/venv/bin/activate +./cactus/build.sh +``` + +Policy: + +```bash +PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q +``` + +Result: +- `16 passed` + +Smoke: + +```bash +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s +``` + +Result: +- `7 passed, 2 deselected` + +## Remaining work + +No known local acceptance failure remains from the planned test matrix. Next step is user audit/review of the uncommitted changes. diff --git a/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md b/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md new file mode 100644 index 000000000..643082b7b --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md @@ -0,0 +1,58 @@ +# Continuation Plan + +## Goal + +Keep fresh Gemma4 Cactus native working end to end for image-only, audio-only, and mixed image+audio, while preserving Qwen, Parakeet, and smoke controls. + +The local acceptance plan now passes. This document is for review and any follow-up hardening, not for continuing a known active failure. + +## Current local result + +- Gemma image-only: passes, identifies `test_monkey.png` as a golden monkey. +- Gemma audio-only: passes, responds coherently to the spoken `what is in this image` prompt when no image is supplied. +- Gemma mixed image+audio: passes, answers `This image is of a golden monkey.` +- Qwen image control: passes, identifies a proboscis monkey. +- Parakeet file and PCM controls: pass, both transcribe `What is in this image?` +- Policy tests: pass, `16 passed`. +- Smoke tests: pass, `7 passed, 2 deselected`. + +## Important implementation decisions + +- Keep Gemma4 audio bias tensors FP16. +- For Gemma4 prompts with audio, use the dynamic media merge path rather than the static full `lm_encoder` merge graph. +- Do not carry additional vision behavior from `justin/v2-cpp-fix`; that branch is already in the ancestry and vision already passes. +- Keep non-audio/image-only behavior on the existing static chunk path. + +## Verification artifacts + +- Audio encoder boundary: `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` +- First-token dynamic merge: `artifacts/E029_first_token_after_dynamic_merge/` +- Gemma audio-only: `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` +- Gemma mixed: `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` +- Gemma image regression: `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` +- Qwen regression: `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` +- Parakeet regression: `artifacts/E034_parakeet_regression_after_dynamic_merge/` +- Audio encoder unchanged after dynamic merge: `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` + +## If further testing is requested + +Run the same sequence under the required build rule: + +```bash +source ../cactus/venv/bin/activate +./cactus/build.sh +``` + +Then rerun: + +```bash +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-audio --experiment +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-mixed --experiment +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-image --experiment +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-qwen-image --experiment +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case parakeet-audio --experiment +PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s +``` + +Do not commit or push until the user audits the changes. diff --git a/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py b/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py new file mode 100644 index 000000000..cb12337b1 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py @@ -0,0 +1,785 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[3] +INVESTIGATION_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_WEIGHTS_ROOT = REPO_ROOT.parent / "cactus" / "weights" +ASSET_DIR = REPO_ROOT / "cactus-engine" / "tests" / "assets" +TMP_DIR = REPO_ROOT / "local_integration_tests" / ".tmp" + + +def run(cmd: list[str], *, timeout: int = 300) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["CACTUS_NO_CLOUD_TELE"] = "1" + return subprocess.run( + cmd, + cwd=REPO_ROOT, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def command_record(case: str, command: list[str], completed: subprocess.CompletedProcess[str]) -> dict[str, Any]: + parsed: Any = None + if completed.stdout.strip().startswith("{"): + try: + parsed = json.loads(completed.stdout) + except json.JSONDecodeError: + parsed = None + return { + "case": case, + "command": command, + "returncode": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + "parsed_stdout": parsed, + } + + +def ensure_runner() -> Path: + build_dir = TMP_DIR / "build" + runner = build_dir / "integration_runner" + sources = [ + REPO_ROOT / "local_integration_tests" / "native_runner.cpp", + REPO_ROOT / "local_integration_tests" / "CMakeLists.txt", + REPO_ROOT / "cactus" / "build" / "libcactus.a", + ] + if runner.exists() and all(runner.stat().st_mtime >= path.stat().st_mtime for path in sources if path.exists()): + return runner + cmake = run( + [ + "cmake", + "-S", + str(REPO_ROOT / "local_integration_tests"), + "-B", + str(build_dir), + f"-DCACTUS_REPO_ROOT={REPO_ROOT}", + "-DCMAKE_BUILD_TYPE=Release", + ], + timeout=120, + ) + if cmake.returncode != 0: + raise RuntimeError(cmake.stderr or cmake.stdout) + build = run(["cmake", "--build", str(build_dir), "--target", "integration_runner", "-j", str(os.cpu_count() or 4)], timeout=120) + if build.returncode != 0: + raise RuntimeError(build.stderr or build.stdout) + if not runner.exists(): + raise RuntimeError(f"integration runner was not built: {runner}") + return runner + + +def ensure_question_wav() -> Path: + wav = TMP_DIR / "what_is_in_this_image.wav" + if wav.exists(): + return wav + mp3 = REPO_ROOT.parent / "cactus" / "what_is_in_this_image.mp3" + if not mp3.exists(): + raise RuntimeError(f"missing paired mp3: {mp3}") + wav.parent.mkdir(parents=True, exist_ok=True) + completed = run( + [ + "ffmpeg", + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(mp3), + "-ac", + "1", + "-ar", + "16000", + "-sample_fmt", + "s16", + str(wav), + ], + timeout=120, + ) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + return wav + + +def cactus_complete(model: Path, prompt: str, image: Path | None = None, audio: Path | None = None, max_tokens: int = 60) -> dict[str, Any]: + runner = ensure_runner() + command: list[str] = [ + str(runner), + "complete", + "--model", + str(model), + "--prompt", + prompt, + "--max-tokens", + str(max_tokens), + ] + if image is not None: + command.extend(["--image", str(image)]) + if audio is not None: + command.extend(["--audio", str(audio)]) + return command_record("cactus_complete", command, run(command, timeout=360)) + + +def native_json(command: list[str], *, timeout: int = 360) -> dict[str, Any]: + completed = run(command, timeout=timeout) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + try: + return json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError(f"native command did not return JSON: {command}\n{completed.stdout[:1000]}") from exc + + +def parakeet_transcribe(model: Path, audio: Path, pcm: bool = False) -> dict[str, Any]: + runner = ensure_runner() + command = [str(runner), "transcribe-pcm" if pcm else "transcribe", "--model", str(model), "--audio", str(audio)] + return command_record("parakeet_transcribe_pcm" if pcm else "parakeet_transcribe", command, run(command, timeout=240)) + + +def hf_gemma_image(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: + script = f""" +import json +import torch +from pathlib import Path +from transformers import AutoProcessor, AutoModelForImageTextToText + +model_dir = Path({str(model_dir)!r}) +image_path = Path({str(image)!r}) +prompt = {prompt!r} +processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) +model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='auto') +messages = [{{'role': 'user', 'content': [{{'type': 'image', 'image': str(image_path)}}, {{'type': 'text', 'text': prompt}}]}}] +inputs = processor.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors='pt') +inputs = {{k: v.to(model.device) if hasattr(v, 'to') else v for k, v in inputs.items()}} +with torch.no_grad(): + out = model.generate(**inputs, max_new_tokens=40, do_sample=False) +text = processor.decode(out[0][inputs['input_ids'].shape[-1]:], skip_special_tokens=True) +print(json.dumps({{'response': text, 'input_tokens': int(inputs['input_ids'].shape[-1])}})) +""" + command = [sys.executable, "-c", script] + return command_record("hf_gemma_image", command, run(command, timeout=360)) + + +def hf_gemma_native_image(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: + script = f""" +import json +import torch +from pathlib import Path +from transformers import AutoProcessor, AutoModelForImageTextToText +from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs + +model_dir = Path({str(model_dir)!r}) +image_path = Path({str(image)!r}) +prompt = {prompt!r} +processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) +prepared = prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=(str(image_path),), + audio_file=None, + torch_dtype=torch.float16, + use_gemma4_chat_template=True, +) +model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='auto') +inputs = {{}} +summaries = {{}} +for name, tensor in zip(prepared.names, prepared.tensors): + key = 'image_position_ids' if name == 'pixel_position_ids' else name + inputs[key] = tensor.to(model.device) + summaries[name] = {{ + 'shape': list(tensor.shape), + 'dtype': str(tensor.dtype), + 'min': float(tensor.float().min().item()) if tensor.numel() else 0.0, + 'max': float(tensor.float().max().item()) if tensor.numel() else 0.0, + 'mean': float(tensor.float().mean().item()) if tensor.numel() else 0.0, + }} +with torch.no_grad(): + out = model.generate(**inputs, max_new_tokens=40, do_sample=False) +text = processor.decode(out[0][inputs['input_ids'].shape[-1]:], skip_special_tokens=True) +input_ids = prepared.tensors[prepared.names.index('input_ids')] +print(json.dumps({{ + 'response': text, + 'input_tokens': int(input_ids.shape[-1]), + 'input_shapes': prepared.metadata.get('input_shapes'), + 'processor_prompt_prefix': str(prepared.metadata.get('processor_prompt'))[:240], + 'tensor_summaries': summaries, +}})) +""" + command = [sys.executable, "-c", script] + return command_record("hf_gemma_native_image", command, run(command, timeout=360)) + + +def hf_gemma_e002_inputs(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: + script = f""" +import hashlib +import json +import torch +from pathlib import Path +from transformers import AutoProcessor +from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs + +model_dir = Path({str(model_dir)!r}) +image_path = Path({str(image)!r}) +prompt = {prompt!r} +processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) +prepared = prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=(str(image_path),), + audio_file=None, + torch_dtype=torch.float32, + use_gemma4_chat_template=True, +) +by_name = {{name: tensor.detach().cpu() for name, tensor in zip(prepared.names, prepared.tensors)}} +input_ids = by_name['input_ids'].reshape(-1).to(torch.int64) +pixel_values = by_name['pixel_values'].to(torch.float32).contiguous() +pixel_position_ids = by_name['pixel_position_ids'].reshape(-1).to(torch.int64).contiguous() +tokenizer = getattr(processor, 'tokenizer', processor) +image_token = getattr(processor, 'image_token', None) +image_token_id = tokenizer.convert_tokens_to_ids(image_token) if image_token is not None and hasattr(tokenizer, 'convert_tokens_to_ids') else -1 +ids = input_ids.tolist() +image_positions = [i for i, tok in enumerate(ids) if tok == image_token_id] +processor_prompt = str(prepared.metadata.get('processor_prompt')) +assistant_generation_start = len(ids) +if processor_prompt.endswith('<|turn>model\\n'): + no_generation_prompt = processor_prompt[:-len('<|turn>model\\n')] + no_gen = tokenizer(no_generation_prompt, return_tensors='pt', add_special_tokens=False)['input_ids'].reshape(-1) + assistant_generation_start = int(no_gen.numel()) +valid_positions = int(((by_name['pixel_position_ids'] != -1).any(dim=-1)).sum().item()) +pixel_np = pixel_values.numpy() +pos_np = by_name['pixel_position_ids'].to(torch.int64).contiguous().numpy() +payload = {{ + 'token_count': len(ids), + 'assistant_generation_start': assistant_generation_start, + 'image_token_id': int(image_token_id), + 'image_token_start': int(image_positions[0]) if image_positions else -1, + 'image_token_count': len(image_positions), + 'first_20': ids[:20], + 'last_20': ids[-20:], + 'input_ids': ids, + 'prompt_prefix': processor_prompt[:320], + 'pixel_values_shape': list(pixel_values.shape), + 'pixel_position_ids_shape': list(by_name['pixel_position_ids'].shape), + 'pixel_values_min': float(pixel_values.min().item()), + 'pixel_values_max': float(pixel_values.max().item()), + 'pixel_values_mean': float(pixel_values.mean().item()), + 'pixel_values_sha256': hashlib.sha256(pixel_np.tobytes()).hexdigest(), + 'pixel_position_ids_sha256': hashlib.sha256(pos_np.tobytes()).hexdigest(), + 'valid_position_count': valid_positions, + 'first_pixel_values': pixel_values.reshape(-1)[:32].tolist(), + 'first_position_rows_flat': pixel_position_ids[:32].tolist(), + 'pixel_values': pixel_values.reshape(-1).tolist(), + 'pixel_position_ids': pixel_position_ids.tolist(), +}} +print(json.dumps(payload)) +""" + command = [sys.executable, "-c", script] + completed = run(command, timeout=360) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + return json.loads(completed.stdout) + + +def e002_cactus_tokens(model: Path, image: Path, prompt: str) -> dict[str, Any]: + runner = ensure_runner() + command = [ + str(runner), + "e002-tokens", + "--model", + str(model), + "--prompt", + prompt, + "--image", + str(image), + ] + return native_json(command, timeout=360) + + +def e002_cactus_preprocess(model: Path, image: Path) -> dict[str, Any]: + runner = ensure_runner() + command = [ + str(runner), + "e002-preprocess", + "--model", + str(model), + "--image", + str(image), + ] + return native_json(command, timeout=360) + + +def e003_cactus_preprocess_variants(model: Path, image: Path) -> dict[str, Any]: + runner = ensure_runner() + command = [ + str(runner), + "e003-preprocess-variants", + "--model", + str(model), + "--image", + str(image), + ] + return native_json(command, timeout=360) + + +def e002_cactus_vision(model: Path, image: Path) -> dict[str, Any]: + runner = ensure_runner() + command = [ + str(runner), + "e002-vision", + "--model", + str(model), + "--image", + str(image), + "--prompt", + "What animal is in this image? Reply in one complete short sentence.", + ] + return native_json(command, timeout=360) + + +def hf_gemma_vision_features(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: + script = f""" +import json +import torch +from pathlib import Path +from transformers import AutoProcessor, AutoModelForImageTextToText +from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs + +model_dir = Path({str(model_dir)!r}) +image_path = Path({str(image)!r}) +prompt = {prompt!r} +processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) +prepared = prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=(str(image_path),), + audio_file=None, + torch_dtype=torch.float16, + use_gemma4_chat_template=True, +) +by_name = {{name: tensor for name, tensor in zip(prepared.names, prepared.tensors)}} +model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='cpu') +pixel_values = by_name['pixel_values'].to(dtype=torch.float16) +position_ids = by_name['pixel_position_ids'].to(dtype=torch.long) +with torch.no_grad(): + outputs = model.get_image_features(pixel_values=pixel_values, image_position_ids=position_ids) +candidates = {{}} +for name, tensor in ( + ('last_hidden_state', getattr(outputs, 'last_hidden_state', None)), + ('pooler_output', getattr(outputs, 'pooler_output', None)), +): + if tensor is None: + continue + flat = tensor.detach().float().cpu().reshape(-1) + candidates[name] = {{ + 'shape': list(tensor.shape), + 'dtype': str(tensor.dtype), + 'count': int(flat.numel()), + 'min': float(flat.min().item()) if flat.numel() else 0.0, + 'max': float(flat.max().item()) if flat.numel() else 0.0, + 'mean': float(flat.mean().item()) if flat.numel() else 0.0, + 'l2': float(torch.linalg.vector_norm(flat).item()) if flat.numel() else 0.0, + 'first_values': flat[:32].tolist(), + 'values': flat.tolist(), + }} +print(json.dumps({{'features': candidates}})) +""" + completed = run([sys.executable, "-c", script], timeout=420) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + return json.loads(completed.stdout) + + +def first_mismatch(left: list[Any], right: list[Any]) -> int | None: + for idx, (lval, rval) in enumerate(zip(left, right)): + if lval != rval: + return idx + if len(left) != len(right): + return min(len(left), len(right)) + return None + + +def sha256_float_list(values: list[float]) -> str: + import array + + arr = array.array("f", values) + return hashlib.sha256(arr.tobytes()).hexdigest() + + +def run_e002_boundary(out_dir: Path, weights_root: Path) -> None: + image = ASSET_DIR / "test_monkey.png" + prompt = "What animal is in this image? Reply in one complete short sentence." + fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" + hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") + + hf = hf_gemma_e002_inputs(hf_snapshot, image, prompt) + cactus_tokens = e002_cactus_tokens(fresh_gemma, image, prompt) + write_json(out_dir / "hf_native_inputs.json", hf) + write_json(out_dir / "cactus_token_probe.json", cactus_tokens) + + token_mismatch = first_mismatch(hf["input_ids"], cactus_tokens["input_ids"]) + token_compare = { + "gate": "prompt_and_token_spans", + "status": "pass" if token_mismatch is None else "fail", + "input_ids_match": token_mismatch is None, + "first_mismatch_index": token_mismatch, + "hf": {key: hf[key] for key in ("token_count", "assistant_generation_start", "image_token_id", "image_token_start", "image_token_count", "first_20", "last_20", "prompt_prefix")}, + "cactus": {key: cactus_tokens[key] for key in ("token_count", "assistant_generation_start", "image_token_id", "image_token_start", "image_token_count", "first_20", "last_20", "prompt_prefix")}, + } + if token_mismatch is not None: + lo = max(0, token_mismatch - 8) + hi = token_mismatch + 9 + token_compare["mismatch_window"] = { + "hf": hf["input_ids"][lo:hi], + "cactus": cactus_tokens["input_ids"][lo:hi], + "start": lo, + } + write_json(out_dir / "token_spans.json", token_compare) + if token_mismatch is not None: + return + + cactus_preprocess = e002_cactus_preprocess(fresh_gemma, image) + write_json(out_dir / "cactus_preprocess.json", cactus_preprocess) + hf_values = hf["pixel_values"] + cactus_values = cactus_preprocess["pixel_values"] + hf_positions = hf["pixel_position_ids"] + cactus_positions = cactus_preprocess["pixel_position_ids"] + if len(hf_values) != len(cactus_values): + max_abs_diff = None + mean_abs_diff = None + first_pixel_mismatch = min(len(hf_values), len(cactus_values)) + else: + diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, cactus_values)] + max_abs_diff = max(diffs) if diffs else 0.0 + mean_abs_diff = sum(diffs) / len(diffs) if diffs else 0.0 + first_pixel_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-5), None) + position_mismatch = first_mismatch(hf_positions, cactus_positions) + preprocess_pass = ( + hf["pixel_values_shape"] == cactus_preprocess["pixel_values_shape"] + and hf["pixel_position_ids_shape"] == cactus_preprocess["pixel_position_ids_shape"] + and position_mismatch is None + and max_abs_diff is not None + and max_abs_diff <= 1e-5 + ) + compare = { + "gate": "image_preprocessing_tensors", + "status": "pass" if preprocess_pass else "fail", + "pixel_values_shape_match": hf["pixel_values_shape"] == cactus_preprocess["pixel_values_shape"], + "pixel_position_ids_shape_match": hf["pixel_position_ids_shape"] == cactus_preprocess["pixel_position_ids_shape"], + "pixel_position_ids_match": position_mismatch is None, + "first_position_mismatch_index": position_mismatch, + "pixel_values_max_abs_diff": max_abs_diff, + "pixel_values_mean_abs_diff": mean_abs_diff, + "first_pixel_value_mismatch_index_at_1e-5": first_pixel_mismatch, + "hf_summary": {key: hf[key] for key in ("pixel_values_shape", "pixel_position_ids_shape", "pixel_values_min", "pixel_values_max", "pixel_values_mean", "pixel_values_sha256", "pixel_position_ids_sha256", "valid_position_count", "first_pixel_values", "first_position_rows_flat")}, + "cactus_summary": {key: cactus_preprocess[key] for key in ("pixel_values_shape", "pixel_position_ids_shape", "pixel_values_min", "pixel_values_max", "pixel_values_mean", "pixel_values_hash", "pixel_position_ids_hash", "valid_position_count", "first_pixel_values", "first_position_rows_flat")}, + } + if first_pixel_mismatch is not None: + lo = max(0, first_pixel_mismatch - 8) + hi = first_pixel_mismatch + 9 + compare["pixel_mismatch_window"] = { + "start": lo, + "hf": hf_values[lo:hi], + "cactus": cactus_values[lo:hi], + } + if position_mismatch is not None: + lo = max(0, position_mismatch - 8) + hi = position_mismatch + 9 + compare["position_mismatch_window"] = { + "start": lo, + "hf": hf_positions[lo:hi], + "cactus": cactus_positions[lo:hi], + } + write_json(out_dir / "image_preprocess_compare.json", compare) + + +def run_e003_preprocess_isolation(out_dir: Path, weights_root: Path) -> None: + image = ASSET_DIR / "test_monkey.png" + prompt = "What animal is in this image? Reply in one complete short sentence." + fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" + hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") + + hf = hf_gemma_e002_inputs(hf_snapshot, image, prompt) + variants = e003_cactus_preprocess_variants(fresh_gemma, image) + write_json(out_dir / "hf_native_inputs.json", hf) + write_json(out_dir / "cactus_resize_variants.json", variants) + + hf_values = hf["pixel_values"] + comparisons: dict[str, Any] = {} + best_name: str | None = None + best_max: float | None = None + for name, variant in variants["variants"].items(): + values = variant["pixel_values"] + if len(values) != len(hf_values): + comparison = { + "status": "fail", + "length_match": False, + "hf_length": len(hf_values), + "variant_length": len(values), + } + else: + diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, values)] + max_abs = max(diffs) if diffs else 0.0 + mean_abs = sum(diffs) / len(diffs) if diffs else 0.0 + first_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-5), None) + comparison = { + "status": "pass" if max_abs <= 1e-5 else "fail", + "length_match": True, + "max_abs_diff": max_abs, + "mean_abs_diff": mean_abs, + "first_mismatch_index_at_1e-5": first_mismatch, + "variant_summary": { + key: variant[key] + for key in ( + "resized_min", + "resized_max", + "first_resized_values", + "pixel_values_min", + "pixel_values_max", + "pixel_values_mean", + "first_pixel_values", + ) + }, + } + if first_mismatch is not None: + lo = max(0, first_mismatch - 8) + hi = first_mismatch + 9 + comparison["mismatch_window"] = { + "start": lo, + "hf": hf_values[lo:hi], + "variant": values[lo:hi], + } + if best_max is None or max_abs < best_max: + best_name = name + best_max = max_abs + comparisons[name] = comparison + + write_json( + out_dir / "preprocess_operation_compare.json", + { + "gate": "e003_preprocess_operation_isolation", + "source_width": variants["source_width"], + "source_height": variants["source_height"], + "target_width": variants["target_width"], + "target_height": variants["target_height"], + "patch_size": variants["patch_size"], + "pooling_kernel_size": variants["pooling_kernel_size"], + "max_patches": variants["max_patches"], + "patch_dim": variants["patch_dim"], + "rescale_factor": variants["rescale_factor"], + "hf_summary": { + key: hf[key] + for key in ( + "pixel_values_shape", + "pixel_values_min", + "pixel_values_max", + "pixel_values_mean", + "first_pixel_values", + ) + }, + "best_variant": best_name, + "best_variant_max_abs_diff": best_max, + "comparisons": comparisons, + }, + ) + + +def compare_float_vectors(hf_values: list[float], cactus_values: list[float]) -> dict[str, Any]: + if len(hf_values) != len(cactus_values): + return { + "status": "fail", + "length_match": False, + "hf_length": len(hf_values), + "cactus_length": len(cactus_values), + } + cactus_nonfinite = sum(1 for value in cactus_values if value is None) + hf_nonfinite = sum(1 for value in hf_values if value is None) + if cactus_nonfinite or hf_nonfinite: + return { + "status": "fail", + "length_match": True, + "hf_nonfinite_count": hf_nonfinite, + "cactus_nonfinite_count": cactus_nonfinite, + "first_nonfinite_index": next( + (i for i, (a, b) in enumerate(zip(hf_values, cactus_values)) if a is None or b is None), + None, + ), + } + diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, cactus_values)] + dot = sum(float(a) * float(b) for a, b in zip(hf_values, cactus_values)) + hf_norm = sum(float(a) * float(a) for a in hf_values) ** 0.5 + cactus_norm = sum(float(b) * float(b) for b in cactus_values) ** 0.5 + max_abs = max(diffs) if diffs else 0.0 + mean_abs = sum(diffs) / len(diffs) if diffs else 0.0 + first_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-3), None) + result: dict[str, Any] = { + "status": "pass" if max_abs <= 1e-2 and mean_abs <= 1e-3 else "fail", + "length_match": True, + "max_abs_diff": max_abs, + "mean_abs_diff": mean_abs, + "cosine_similarity": dot / (hf_norm * cactus_norm) if hf_norm and cactus_norm else 0.0, + "first_mismatch_index_at_1e-3": first_mismatch, + } + if first_mismatch is not None: + lo = max(0, first_mismatch - 8) + hi = first_mismatch + 9 + result["mismatch_window"] = { + "start": lo, + "hf": hf_values[lo:hi], + "cactus": cactus_values[lo:hi], + } + return result + + +def run_e002_vision_boundary(out_dir: Path, weights_root: Path) -> None: + image = ASSET_DIR / "test_monkey.png" + prompt = "What animal is in this image? Reply in one complete short sentence." + fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" + hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") + + hf = hf_gemma_vision_features(hf_snapshot, image, prompt) + cactus = e002_cactus_vision(fresh_gemma, image) + write_json(out_dir / "hf_vision_features.json", hf) + write_json(out_dir / "cactus_vision_features.json", cactus) + + comparisons: dict[str, Any] = {} + for cactus_name, cactus_feature in cactus["features"].items(): + comparisons[cactus_name] = {} + for hf_name, hf_feature in hf["features"].items(): + comparison = compare_float_vectors(hf_feature["values"], cactus_feature["values"]) + comparison["hf_shape"] = hf_feature["shape"] + comparison["cactus_shape"] = cactus_feature["shape"] + comparison["hf_summary"] = {key: hf_feature[key] for key in ("dtype", "count", "min", "max", "mean", "l2", "first_values")} + comparison["cactus_summary"] = {key: cactus_feature[key] for key in ("precision", "count", "finite_count", "nonfinite_count", "min", "max", "mean", "l2", "first_values")} + comparisons[cactus_name][hf_name] = comparison + + write_json( + out_dir / "vision_encoder_compare.json", + { + "gate": "vision_encoder_output", + "comparisons": comparisons, + }, + ) + + +def manifest_summary(path: Path) -> dict[str, Any]: + manifest = path / "components" / "manifest.json" + config = path / "config.txt" + result: dict[str, Any] = {"path": str(path), "manifest_exists": manifest.exists(), "config_exists": config.exists()} + if manifest.exists(): + data = json.loads(manifest.read_text()) + result.update( + { + "family": data.get("family"), + "task": data.get("task"), + "model_source": data.get("model_source"), + "component_order": data.get("component_order"), + "components": [ + { + "component": component.get("component"), + "logical_inputs": component.get("logical_inputs"), + "logical_outputs": component.get("logical_outputs"), + } + for component in data.get("components", []) + ], + } + ) + return result + + +def run_case(case: str, out_dir: Path, weights_root: Path) -> None: + image = ASSET_DIR / "test_monkey.png" + prompt = "What animal is in this image? Reply in one complete short sentence." + fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" + qwen = weights_root / "qwen3-vl-2b-instruct-reconvert" + parakeet = weights_root / "parakeet-tdt-0.6b-v3-transpiled" + hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") + + if case == "inventory": + write_json(out_dir / "inventory.json", {"gemma_fresh": manifest_summary(fresh_gemma)}) + return + if case == "hf-gemma-image": + write_json(out_dir / "hf_gemma_image.json", hf_gemma_image(hf_snapshot, image, prompt)) + return + if case == "hf-gemma-native-image": + write_json(out_dir / "hf_gemma_native_image.json", hf_gemma_native_image(hf_snapshot, image, prompt)) + return + if case == "cactus-gemma-image": + write_json(out_dir / "cactus_gemma_image.json", cactus_complete(fresh_gemma, prompt, image=image)) + return + if case == "cactus-gemma-audio": + wav = ensure_question_wav() + audio_prompt = "Listen to the audio and answer briefly." + write_json(out_dir / "cactus_gemma_audio.json", cactus_complete(fresh_gemma, audio_prompt, audio=wav)) + return + if case == "cactus-gemma-mixed": + wav = ensure_question_wav() + mixed_prompt = "Answer the spoken question using the provided image. Reply in one short sentence." + write_json(out_dir / "cactus_gemma_mixed.json", cactus_complete(fresh_gemma, mixed_prompt, image=image, audio=wav)) + return + if case == "cactus-qwen-image": + write_json(out_dir / "cactus_qwen_image.json", cactus_complete(qwen, prompt, image=image, max_tokens=40)) + return + if case == "parakeet-audio": + wav = ensure_question_wav() + write_json(out_dir / "parakeet_audio.json", parakeet_transcribe(parakeet, wav)) + write_json(out_dir / "parakeet_audio_pcm.json", parakeet_transcribe(parakeet, wav, pcm=True)) + return + if case == "all-baseline": + for subcase in ("inventory", "hf-gemma-image", "hf-gemma-native-image", "cactus-gemma-image", "cactus-qwen-image", "parakeet-audio"): + run_case(subcase, out_dir, weights_root) + return + if case == "e002-boundary": + run_e002_boundary(out_dir, weights_root) + return + if case == "e003-preprocess-isolation": + run_e003_preprocess_isolation(out_dir, weights_root) + return + if case == "e002-vision-boundary": + run_e002_vision_boundary(out_dir, weights_root) + return + raise ValueError(f"unknown case: {case}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--case", required=True) + parser.add_argument("--experiment", required=True) + parser.add_argument("--weights-root", default=str(DEFAULT_WEIGHTS_ROOT)) + args = parser.parse_args() + + out_dir = INVESTIGATION_ROOT / "artifacts" / args.experiment + out_dir.mkdir(parents=True, exist_ok=True) + write_json( + out_dir / "run_metadata.json", + { + "case": args.case, + "experiment": args.experiment, + "repo_root": str(REPO_ROOT), + "weights_root": str(Path(args.weights_root).resolve()), + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + }, + ) + run_case(args.case, out_dir, Path(args.weights_root).resolve()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 14fc7b1b8..04dec765a 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -297,6 +297,12 @@ def create_parser(): help='Allow HF remote code during the transpile phase') convert_parser.add_argument('--local-files-only', action='store_true', help='Require HF model/processor files to already be local during transpile') + convert_parser.add_argument('--npu', action='store_true', + help='Also emit a CoreML .mlpackage for NPU (Apple Neural Engine) prefill') + convert_parser.add_argument('--npu-chunk-size', type=int, default=256, + help='Prefill chunk size for the emitted .mlpackage (default: 256)') + convert_parser.add_argument('--npu-quantize', type=int, choices=[0, 4, 8], default=4, + help='Quantize .mlpackage weights (0=off, 4=int4 default, 8=int8). Smaller weights = faster on ANE + fits saved .mlpackage in less disk.') return parser diff --git a/python/cactus/cli/compile.py b/python/cactus/cli/compile.py index 9681600ab..423f8ab2b 100644 --- a/python/cactus/cli/compile.py +++ b/python/cactus/cli/compile.py @@ -75,6 +75,14 @@ def cmd_build(args): if result.returncode != 0: print_color(RED, "Failed to build cactus library") return 1 + for _ in range(50): + if lib_path.exists(): + break + import time + time.sleep(0.1) + if not lib_path.exists(): + print_color(RED, f"Build did not produce {lib_path}") + return 1 tests_dir = PROJECT_ROOT / "cactus-engine" / "tests" build_dir = tests_dir / "build" build_dir.mkdir(parents=True, exist_ok=True) diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index cb1d20c76..456cd98ce 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -244,6 +244,12 @@ def cmd_convert(args): extra_args.append("--trust-remote-code") if getattr(args, "local_files_only", False): extra_args.append("--local-files-only") + if getattr(args, "npu", False): + extra_args.append("--npu") + extra_args.extend(["--npu-chunk-size", str(getattr(args, "npu_chunk_size", 256))]) + npu_q = getattr(args, "npu_quantize", None) + if npu_q is not None: + extra_args.extend(["--npu-quantize", str(int(npu_q))]) transpile_args = argparse.Namespace( model_id=model_id, diff --git a/python/cactus/convert/cactus_adapters/tensor_io.py b/python/cactus/convert/cactus_adapters/tensor_io.py index ad36c8c38..3ce614cc7 100644 --- a/python/cactus/convert/cactus_adapters/tensor_io.py +++ b/python/cactus/convert/cactus_adapters/tensor_io.py @@ -281,8 +281,7 @@ def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=Fal data = data * GEMMA4_WEIGHT_SCALE elif 'router_scale' in filename: data = data / GEMMA4_WEIGHT_SCALE - elif filename in ('token_embeddings.weights', 'output_weight.weights', - 'embed_vision_proj.weights', 'embed_vision_embedding.weights'): + elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights'): data = data / GEMMA4_WEIGHT_SCALE elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE diff --git a/python/cactus/convert/model_adapters/adapters.py b/python/cactus/convert/model_adapters/adapters.py index bf51ec941..dbb10eb45 100644 --- a/python/cactus/convert/model_adapters/adapters.py +++ b/python/cactus/convert/model_adapters/adapters.py @@ -162,7 +162,7 @@ def policy(self, match: NameMatch, shape: tuple[int, ...], requested_bits: int) policy = super().policy(match, shape, requested_bits) name = match.source_name if name == "model.embed_vision.embedding_projection.weight": - return replace(policy, use_gptq=False, fallback_reason=policy.fallback_reason or "vision embedding projection has no stable Hessian target") + return policy if ".self_attn." in name and (name.endswith(".k_proj.weight") or name.endswith(".v_proj.weight")): parts = name.split(".") try: diff --git a/python/cactus/convert/model_adapters/naming.py b/python/cactus/convert/model_adapters/naming.py index e74c26236..eaeaa0f44 100644 --- a/python/cactus/convert/model_adapters/naming.py +++ b/python/cactus/convert/model_adapters/naming.py @@ -23,7 +23,6 @@ class NameMatch: GEMMA4_DIV_BASENAMES = { "token_embeddings", "output_weight", - "embed_vision_proj", "embed_vision_embedding", } diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index 118683495..f8e64fb05 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -24,6 +24,8 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("ignored", "none", None, component, False, "none", "no output filename") if "norm" in out.lower(): return TensorPolicy("fallback", "FP16", None, component, False, "none", "norm tensor") + if family == "gemma4" and name == "model.embed_vision.embedding_projection.weight": + return TensorPolicy("fallback", "FP16", None, component, False, "none", "vision embedding projection scale-sensitive") if family in {"parakeet", "parakeet_tdt"} and out.endswith(".bias") and ( "conv_" in out or out.startswith("subsampling_") or out.startswith("ctc_head_") ): @@ -32,18 +34,20 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("fallback", "FP16", None, component, False, "none", "lstm recurrent tensor") if family == "parakeet_tdt" and out.startswith("tdt_"): return TensorPolicy("fallback", "FP16", None, component, False, "none", "tdt decoder tensor") + if family in {"parakeet", "parakeet_tdt"} and "conv_pointwise" in out and len(shape) == 3 and shape[2] == 1: + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "pointwise conv tensor") + if "conv_depthwise.weights" in out and len(shape) == 3 and shape[1] == 1: + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "depthwise conv tensor") if family in {"parakeet", "parakeet_tdt"} and out.startswith("layer_") and ( "conv_pointwise" in out or "conv_depthwise" in out ): return TensorPolicy("fallback", "FP16", None, component, False, "none", "conformer conv tensor") if family == "whisper" and out.startswith("encoder.layer_"): return TensorPolicy("fallback", "FP16", None, component, False, "none", "whisper encoder tensor") + if family == "gemma4" and component == "audio" and (name.endswith(".bias") or out.endswith(".bias") or ".bias." in out): + return TensorPolicy("fallback", "FP16", None, component, False, "none", "audio bias tensor") if name.endswith(".bias") or out.endswith(".bias") or ".bias." in out: return TensorPolicy("fallback", "FP16", None, component, False, "none", "bias tensor") - if family in {"parakeet", "parakeet_tdt"} and "conv_pointwise" in out and len(shape) == 3 and shape[2] == 1: - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "pointwise conv tensor") - if "conv_depthwise.weights" in out and len(shape) == 3 and shape[1] == 1: - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "depthwise conv tensor") if len(shape) != 2: return TensorPolicy("fallback", "FP16", None, component, False, "none", "non-2d tensor") if "position_embedding" in out.lower() or "pos_embed" in out.lower() or "embed_positions" in out.lower(): diff --git a/python/cactus/convert/tests/test_policy.py b/python/cactus/convert/tests/test_policy.py index 441b94d2c..3a39f9fe8 100644 --- a/python/cactus/convert/tests/test_policy.py +++ b/python/cactus/convert/tests/test_policy.py @@ -38,7 +38,7 @@ def test_gemma4_adapter_disables_gptq_for_unhookable_tensors(): vision_proj = cactus_name_for_tensor("model.embed_vision.embedding_projection.weight", "gemma4", 35) vision_policy = adapter.policy(vision_proj, (1536, 1152), 4) - assert vision_policy.precision == "CQ4" + assert vision_policy.precision == "FP16" assert not vision_policy.use_gptq vision_tower = cactus_name_for_tensor("model.vision_tower.encoder.layers.0.mlp.down_proj.linear.weight", "gemma4", 35) @@ -56,11 +56,11 @@ def test_policy_audio_no_gptq(): assert not p.use_gptq -def test_policy_bias_int8(): +def test_policy_gemma4_audio_bias_fp16(): match = cactus_name_for_tensor("model.audio_tower.output_proj.bias", "gemma4", 1) p = policy_for_tensor(match, (1536,), 4, "gemma4") - assert p.precision == "INT8" - assert p.bits == 8 + assert p.precision == "FP16" + assert p.bits is None def test_policy_position_embedding_fp16(): diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index d3daf70f3..a1b6e8609 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -295,6 +295,7 @@ def _write_component_bundle( transpiled_component_graphs: dict[str, TranspiledGraph] | None = None, component_io_signatures: dict[str, dict[str, tuple[str, ...]]] | None = None, graph_filename: str = "graph.cactus", + npu_prefill_mlpackage: str | None = None, ) -> Path: bundle_dir = artifact_dir / "components" component_order = [ @@ -303,6 +304,7 @@ def _write_component_bundle( "audio_encoder", "vision_encoder", "lm_encoder", + "lm_encoder_text_chunk", "decoder_prefill_chunk", "decoder", "lm_encoder_step", @@ -397,18 +399,18 @@ def _write_component_bundle( ) manifest_path = bundle_dir / "manifest.json" - _write_json( - manifest_path, - { - "model_id": model_id, - "model_source": model_source, - "task": task, - "family": family, - "component_order": component_order, - "inputs": _serialize_json_compatible(inputs_metadata), - "components": manifest_components, - }, - ) + manifest_payload: dict[str, object] = { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component_order": component_order, + "inputs": _serialize_json_compatible(inputs_metadata), + "components": manifest_components, + } + if npu_prefill_mlpackage: + manifest_payload["npu_prefill"] = npu_prefill_mlpackage + _write_json(manifest_path, manifest_payload) return manifest_path @@ -665,6 +667,15 @@ def _run_component_pipeline_transpile( f"saved_optimized_component_ir_{component}=" f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" ) + from .npu import run_prefill_pipeline + npu_prefill_filename = run_prefill_pipeline( + model, + artifact_dir, + chunk_size=int(getattr(args, "npu_chunk_size", 256)), + enabled=bool(getattr(args, "npu", False)), + quantize_bits=getattr(args, "npu_quantize", None), + ) + component_manifest_path = _write_component_bundle( artifact_dir=artifact_dir, model_id=args.model_id, @@ -677,6 +688,7 @@ def _run_component_pipeline_transpile( transpiled_component_graphs=transpiled_component_graphs, component_io_signatures=component_io_signatures, graph_filename=args.graph_filename, + npu_prefill_mlpackage=npu_prefill_filename, ) print(f"saved_component_bundle_manifest={component_manifest_path}") for component in transpiled_component_graphs: @@ -2471,6 +2483,14 @@ def _load_transformers_bundle( ): config = _load_config_json(model_id) config_model_type = str(config.get("model_type", "") or "").lower() + if not config_model_type: + normalized_model_id = model_id.lower().replace("_", "-") + if "qwen3-vl" in normalized_model_id: + config_model_type = "qwen3_vl" + elif "lfm2.5-vl" in normalized_model_id or "lfm2-vl" in normalized_model_id: + config_model_type = "lfm2_vl" + if config_model_type: + config = {"model_type": config_model_type} external_transformers_site_packages = _ensure_transformers_supports_model_type(config_model_type) patch_note = _patch_transformers_torchvision_probe() if patch_note: @@ -2524,7 +2544,7 @@ def _load_transformers_bundle( f"Could not load tokenizer for {model_id}:\n" + "\n".join(tokenizer_errors) ) - if config_model_type in {"lfm2_vl", "qwen3_5"}: + if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}: model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -2535,7 +2555,7 @@ def _load_transformers_bundle( tie_note = _tie_lfm2_vl_lm_head_if_needed(model) if tie_note: print(f"note={tie_note}") - elif config_model_type == "qwen3_5" and isinstance(config.get("text_config"), dict): + elif config_model_type in {"qwen3_5", "qwen3_vl"} and isinstance(config.get("text_config"), dict): model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -2584,7 +2604,7 @@ def _load_transformers_bundle( + processor_config_hint ) - if config_model_type in {"lfm2_vl", "qwen3_5"}: + if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}: model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -2918,6 +2938,24 @@ def main() -> int: "(for example: vision_encoder,audio_encoder,lm_encoder,decoder)." ), ) + parser.add_argument( + "--npu", + action="store_true", + help="Also emit a CoreML .mlpackage for Apple Neural Engine prefill.", + ) + parser.add_argument( + "--npu-chunk-size", + type=int, + default=256, + help="Prefill chunk size baked into the emitted .mlpackage (default: 256).", + ) + parser.add_argument( + "--npu-quantize", + type=int, + default=4, + choices=[0, 4, 8], + help="Post-conversion weight quantization for the .mlpackage (0=off, 4=int4 default, 8=int8). Smaller weights = faster on ANE and smaller .mlpackage on disk.", + ) parser.add_argument("--no-fuse-gated-deltanet", action="store_true") parser.add_argument("--no-fuse-rms-norm", action="store_true") parser.add_argument("--no-fuse-rope", action="store_true") @@ -3017,7 +3055,7 @@ def main() -> int: system_prompt=args.system_prompt, enable_thinking_if_supported=args.enable_thinking, ) - elif config_model_type == "qwen3_5": + elif config_model_type in {"qwen3_5", "qwen3_vl"}: prepared = _prepare_qwen3_5_multimodal_inputs( processor_or_tokenizer, prompt=args.prompt, @@ -3310,6 +3348,15 @@ def main() -> int: f"saved_optimized_component_ir_{component}=" f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" ) + from .npu import run_prefill_pipeline + npu_prefill_filename = run_prefill_pipeline( + model, + artifact_dir, + chunk_size=int(getattr(args, "npu_chunk_size", 256)), + enabled=bool(getattr(args, "npu", False)), + quantize_bits=getattr(args, "npu_quantize", None), + ) + component_manifest_path = _write_component_bundle( artifact_dir=artifact_dir, model_id=args.model_id, @@ -3321,6 +3368,7 @@ def main() -> int: optimized_component_graphs=optimized_component_graphs, transpiled_component_graphs=transpiled_component_graphs, graph_filename=args.graph_filename, + npu_prefill_mlpackage=npu_prefill_filename, ) print(f"saved_component_bundle_manifest={component_manifest_path}") for component in optimized_component_graphs: diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index 4925e8e47..5ec2e48c0 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -23,6 +23,16 @@ _DYNAMIC_KV_POSITION_OFFSET = (1 << 64) - 1 +_DYNAMIC_KV_QUERY_POSITION_OFFSET = (1 << 64) - 2 +_KV_CACHE_TRANSPARENT_OPS = { + "expand", + "precision_cast", + "reshape", + "slice", + "transpose", + "unsqueeze", + "view", +} @dataclass @@ -254,6 +264,13 @@ def _lower_attention_with_internal_kv_cache( sink_size = int(ir.meta.get("cache_sink_size", 4) or 4) g.kv_cache_append(key, k_cache, window_size=window_size, sink_size=sink_size) g.kv_cache_append(value, v_cache, window_size=window_size, sink_size=sink_size) + kv_input_cache_states: dict[tuple[str, str], tuple[Tensor, Tensor, int, int]] = env.setdefault( + "__internal_kv_cache_by_kv_inputs", + {}, + ) # type: ignore[assignment] + cache_entry = (k_cache, v_cache, window_size, int(value_shape[3])) + kv_input_cache_states[(node.inputs[1], node.inputs[2])] = cache_entry + kv_input_cache_states[_canonical_kv_cache_input_key(ir, node)] = cache_entry out = g.attention_cached( query, key, @@ -270,6 +287,66 @@ def _lower_attention_with_internal_kv_cache( return [g.permute(out, (0, 2, 1, 3))] +def _lower_attention_with_shared_internal_kv_cache( + g: Graph, + ir: IRGraph, + env: dict[str, Any], + node: IRNode, + *, + query: Tensor, + key: Tensor, + value: Tensor, + output_layout: str, +) -> list[Tensor] | None: + kv_input_cache_states: dict[tuple[str, str], tuple[Tensor, Tensor, int, int]] = env.get( + "__internal_kv_cache_by_kv_inputs", + {}, + ) + cache_state = kv_input_cache_states.get((node.inputs[1], node.inputs[2])) + if cache_state is None: + cache_state = kv_input_cache_states.get(_canonical_kv_cache_input_key(ir, node)) + if cache_state is None: + return None + k_cache, v_cache, window_size, v_head_dim = cache_state + out = g.attention_cached( + query, + key, + value, + k_cache, + v_cache, + scale=_resolve_attention_scale(node, query), + position_offset=_DYNAMIC_KV_QUERY_POSITION_OFFSET, + window_size=window_size, + v_head_dim=v_head_dim, + ) + if output_layout == "bthd": + return [out] + return [g.permute(out, (0, 2, 1, 3))] + + +def _canonical_kv_cache_input_key(ir: IRGraph, node: IRNode) -> tuple[str, str]: + return ( + _canonical_kv_cache_value_id(ir, node.inputs[1]), + _canonical_kv_cache_value_id(ir, node.inputs[2]), + ) + + +def _canonical_kv_cache_value_id(ir: IRGraph, value_id: str) -> str: + current = value_id + seen: set[str] = set() + while current not in seen: + seen.add(current) + value = ir.values.get(current) + producer_id = None if value is None else value.producer + if producer_id is None: + break + producer = ir.nodes.get(producer_id) + if producer is None or producer.op not in _KV_CACHE_TRANSPARENT_OPS or len(producer.inputs) != 1: + break + current = producer.inputs[0] + return current + + def _should_lower_conv1d_with_internal_conv_cache(ir: IRGraph, node: IRNode, x: Tensor, weight: Tensor) -> bool: if not bool(ir.meta.get("use_internal_conv_cache", False)): return False @@ -1090,6 +1167,18 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> if mask_tensor is not None: mask_tensor = _ensure_fp16_tensor(g, mask_tensor) mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + shared_cached = _lower_attention_with_shared_internal_kv_cache( + g, + ir, + env, + node, + query=query, + key=key, + value=value, + output_layout=output_layout, + ) + if shared_cached is not None: + return shared_cached if _should_lower_attention_with_internal_kv_cache(ir, node): return _lower_attention_with_internal_kv_cache( g, @@ -2672,9 +2761,7 @@ def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: def _ensure_scalar_math_tensor(g: Graph, tensor: Tensor) -> Tensor: - if tensor.dtype == Graph.FP16: - return tensor - return g.precision_cast(tensor, Graph.FP16) + return tensor def _ensure_tensor_dtype(g: Graph, tensor: Tensor, dtype: int) -> Tensor: diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 62be8a529..a31973b0e 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from dataclasses import dataclass import inspect +import math import struct import os from pathlib import Path @@ -18,6 +19,7 @@ _GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE = 1.0 / 64.0 +_UNSET = object() try: from transformers.models.gemma4.modeling_gemma4 import create_bidirectional_mask as _GEMMA4_CREATE_BIDIRECTIONAL_MASK # type: ignore except Exception: @@ -330,6 +332,10 @@ def _torch_is_compiling() -> bool: def _gemma4_cpu_safe_text_mlp_enabled(layer: torch.nn.Module, hidden_states: torch.Tensor) -> bool: if hidden_states.device.type != "cpu" or hidden_states.dtype != torch.float16: return False + return _gemma4_text_mlp_manual_available(layer) + + +def _gemma4_text_mlp_manual_available(layer: torch.nn.Module) -> bool: if bool(getattr(layer, "enable_moe_block", False)): return False mlp = getattr(layer, "mlp", None) @@ -417,8 +423,12 @@ def _gemma4_text_attention_forward( value_states = attn.v_norm(value_states) value_states = value_states.transpose(1, 2) - if bool(getattr(attn, "store_full_length_kv", False)): - shared_kv_states[int(attn.layer_idx)] = (key_states, value_states) + layer_idx = getattr(attn, "layer_idx", None) + if layer_idx is not None and ( + bool(getattr(attn, "store_full_length_kv", False)) + or not bool(getattr(attn, "is_kv_shared_layer", False)) + ): + shared_kv_states[int(layer_idx)] = (key_states, value_states) attention_interface = eager_attention_forward if getattr(attn.config, "_attn_implementation", "eager") != "eager": @@ -450,7 +460,8 @@ def _gemma4_text_decoder_layer_forward( use_cache: bool = False, shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, ) -> torch.Tensor: - if not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): + manual_attention_required = shared_kv_states is not None + if not manual_attention_required and not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): return layer( hidden_states, position_embeddings=position_embeddings, @@ -460,6 +471,8 @@ def _gemma4_text_decoder_layer_forward( past_key_values=past_key_values, use_cache=use_cache, ) + if not _gemma4_text_mlp_manual_available(layer): + raise RuntimeError("Gemma4 KV sharing requires a supported text MLP structure") residual = hidden_states @@ -506,6 +519,10 @@ def _gemma4_text_backbone_forward( per_layer_inputs: torch.Tensor | None, causal_mask_mapping: dict[str, torch.Tensor], position_ids: torch.LongTensor, + layer_start: int = 0, + layer_end: int | None = None, + apply_norm: bool = True, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None | object = _UNSET, ) -> torch.Tensor: hidden_states = inputs_embeds layer_types = tuple(dict.fromkeys(getattr(backbone.config, "layer_types", ()))) @@ -513,10 +530,14 @@ def _gemma4_text_backbone_forward( layer_type: backbone.rotary_emb(hidden_states, position_ids, layer_type) for layer_type in layer_types } - shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + if shared_kv_states is _UNSET: + num_shared_layers = int(getattr(backbone.config, "num_kv_shared_layers", 0) or 0) + shared_kv_states = {} if num_shared_layers > 0 else None config_layer_types = tuple(getattr(backbone.config, "layer_types", ())) - for layer_index, decoder_layer in enumerate(backbone.layers[: backbone.config.num_hidden_layers]): + end = backbone.config.num_hidden_layers if layer_end is None else min(int(layer_end), int(backbone.config.num_hidden_layers)) + for layer_index in range(max(0, int(layer_start)), end): + decoder_layer = backbone.layers[layer_index] layer_per_input = None if per_layer_inputs is not None: layer_per_input = per_layer_inputs[:, :, decoder_layer.layer_idx, :] @@ -537,7 +558,7 @@ def _gemma4_text_backbone_forward( shared_kv_states=shared_kv_states, ) - return backbone.norm(hidden_states) + return backbone.norm(hidden_states) if apply_norm else hidden_states def _gemma4_strip_audio_padding(audio_output: object) -> torch.Tensor: @@ -712,7 +733,15 @@ def _gemma4_pool_vision_hidden_native_like( else: valid_bins = torch.logical_not((weights == 0).all(dim=0)) pooled_batches.append(pooled_full[valid_bins]) - return torch.cat(pooled_batches, dim=0) + pooled_hidden = pooled_batches[0] if len(pooled_batches) == 1 else torch.cat(pooled_batches, dim=0) + hidden_size = int(_module_or_config_attr(vision_tower, "hidden_size", pooled_hidden.shape[-1]) or pooled_hidden.shape[-1]) + pooled_hidden = pooled_hidden * math.sqrt(float(hidden_size)) + if bool(_module_or_config_attr(vision_tower, "standardize", False)): + std_bias = getattr(vision_tower, "std_bias", None) + std_scale = getattr(vision_tower, "std_scale", None) + if std_bias is not None and std_scale is not None: + pooled_hidden = (pooled_hidden - std_bias) * std_scale + return pooled_hidden def _gemma4_vision_encoder_hidden_states( @@ -837,6 +866,10 @@ def _gemma4_compute_native_like_image_features( projection = getattr(embed_vision, "embedding_projection", None) if not isinstance(projection, torch.nn.Linear): raise TypeError("Gemma4 vision embedder is missing embedding_projection") + pre_projection_norm = getattr(embed_vision, "embedding_pre_projection_norm", None) + if isinstance(pre_projection_norm, torch.nn.Module): + pooled_hidden = pre_projection_norm(pooled_hidden) + pooled_hidden = pooled_hidden.to(dtype=projection.weight.dtype) projected = F.linear( pooled_hidden, projection.weight, @@ -848,7 +881,7 @@ def _gemma4_compute_native_like_image_features( post_proj_norm_weight, eps=_gemma4_layer_norm_eps(multimodal_backbone), ) - return _gemma4_rms_norm_no_scale(projected, eps=float(getattr(embed_vision, "eps", 1e-6))) + return projected def _gemma4_compute_native_like_audio_features( @@ -1688,6 +1721,10 @@ def get_transpile_metadata(self): } +class Lfm2VlLMEncoderTextChunkAdapter(Lfm2VlLMEncoderStepAdapter): + pass + + class Lfm2VlDecoderAdapter(torch.nn.Module): def __init__( self, @@ -3040,6 +3077,75 @@ class Gemma4DecoderStepAdapter(Gemma4DecoderAdapter): pass +class Gemma4DecoderPrefillChunkAdapter(Gemma4DecoderAdapter): + def forward( + self, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor, + position_ids: torch.LongTensor, + ) -> torch.Tensor: + normalized_per_layer_inputs = per_layer_inputs + if normalized_per_layer_inputs.numel() == 0: + normalized_per_layer_inputs = None + attention_mask = torch.ones( + position_ids.shape, + dtype=torch.long, + device=position_ids.device, + ) + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + num_layers = int(getattr(self.backbone.config, "num_hidden_layers", len(self.backbone.layers))) + num_shared_layers = int(getattr(self.backbone.config, "num_kv_shared_layers", 0) or 0) + first_shared_layer = max(0, num_layers - num_shared_layers) + if first_shared_layer <= 0 or first_shared_layer >= num_layers or int(inputs_embeds.shape[1]) <= 1: + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + ) + else: + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + layer_end=first_shared_layer, + apply_norm=False, + shared_kv_states=shared_kv_states, + ) + tail_masks = { + key: value[:, :, -1:, :] if value.ndim == 4 else value + for key, value in causal_mask_mapping.items() + } + tail_per_layer_inputs = ( + None + if normalized_per_layer_inputs is None + else normalized_per_layer_inputs[:, -1:, :, :] + ) + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=hidden_states[:, -1:, :], + per_layer_inputs=tail_per_layer_inputs, + causal_mask_mapping=tail_masks, + position_ids=position_ids[:, -1:], + layer_start=first_shared_layer, + apply_norm=True, + shared_kv_states=shared_kv_states, + ) + logits = self.model.lm_head(hidden_states[:, -1:, :]) + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def _build_gemma4_multimodal_component_specs( model: torch.nn.Module, *, @@ -3076,32 +3182,27 @@ def _require(component: str) -> None: _require("lm_encoder_step") _require("lm_encoder_media_step") _require("decoder_step") - if "decoder_prefill_chunk" in requested_set: + if "lm_encoder" in requested_set: _require("vision_encoder") _require("audio_encoder") _require("lm_encoder") + if "decoder_prefill_chunk" in requested_set: _require("decoder_prefill_chunk") - elif "lm_encoder" in requested_set: + if "vision_encoder" in requested_set: _require("vision_encoder") + if "audio_encoder" in requested_set: _require("audio_encoder") - _require("lm_encoder") - else: - if "vision_encoder" in requested_set: - _require("vision_encoder") - if "audio_encoder" in requested_set: - _require("audio_encoder") - if "lm_encoder_step" in requested_set: - _require("lm_encoder_step") - if "lm_encoder_text_chunk" in requested_set: - _require("lm_encoder_text_chunk") - if "lm_encoder_media_step" in requested_set: - _require("lm_encoder_media_step") - if "lm_encoder_media_chunk" in requested_set: - _require("lm_encoder_media_chunk") - if "decoder_step" in requested_set: - _require("lm_encoder_step") - _require("lm_encoder_media_step") - _require("decoder_step") + if "lm_encoder_step" in requested_set: + _require("lm_encoder_step") + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "lm_encoder_media_step" in requested_set: + _require("lm_encoder_media_step") + if "lm_encoder_media_chunk" in requested_set: + _require("lm_encoder_media_chunk") + if "decoder_step" in requested_set: + _require("lm_encoder_step") + _require("decoder_step") vision_tower = getattr(getattr(model, "model", model), "vision_tower", None) pooling_kernel_size = int(_module_or_config_attr(vision_tower, "pooling_kernel_size", 3) or 3) @@ -3125,6 +3226,7 @@ def _require(component: str) -> None: image_features: torch.Tensor | None = None audio_features: torch.Tensor | None = None decoder_inputs: tuple[torch.Tensor, ...] | None = None + prefill_decoder_inputs: tuple[torch.Tensor, ...] | None = None decoder_step_inputs: tuple[torch.Tensor, ...] | None = None step_input_ids: torch.Tensor | None = None step_position_ids: torch.Tensor | None = None @@ -3135,16 +3237,14 @@ def _require(component: str) -> None: if "vision_encoder" in expanded_components and ( "lm_encoder" in expanded_components or "decoder" in expanded_components - or "decoder_prefill_chunk" in expanded_components ): image_features = vision_encoder(pixel_values, pixel_position_ids) if "audio_encoder" in expanded_components and ( "lm_encoder" in expanded_components or "decoder" in expanded_components - or "decoder_prefill_chunk" in expanded_components ): audio_features = audio_encoder(input_features, input_features_mask) - if "lm_encoder" in expanded_components or "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + if "lm_encoder" in expanded_components or "decoder" in expanded_components: if image_features is None: image_features = vision_encoder(pixel_values, pixel_position_ids) if audio_features is None: @@ -3160,17 +3260,18 @@ def _require(component: str) -> None: weights_dir=weights_dir, native_merge_plan=native_merge_plan, ).eval() - encoder_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_ENCODER_CHUNK", "32") or "32")) + encoder_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_ENCODER_CHUNK", "128") or "128")) encoder_chunk_size = min(encoder_chunk_size, int(input_ids.shape[1])) lm_encoder_step = Gemma4LMEncoderStepAdapter(model, weights_dir=weights_dir).eval() lm_encoder_text_chunk = Gemma4LMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() lm_encoder_media_step = Gemma4LMEncoderMediaStepAdapter(model, weights_dir=weights_dir).eval() lm_encoder_media_chunk = Gemma4LMEncoderMediaChunkAdapter(model, weights_dir=weights_dir).eval() decoder = Gemma4DecoderAdapter(model, weights_dir=weights_dir).eval() + decoder_prefill = Gemma4DecoderPrefillChunkAdapter(model, weights_dir=weights_dir).eval() decoder_step = Gemma4DecoderStepAdapter(model, weights_dir=weights_dir).eval() with torch.no_grad(): - if "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + if "decoder" in expanded_components: decoder_inputs = lm_encoder( input_ids, attention_mask, @@ -3209,6 +3310,8 @@ def _require(component: str) -> None: dtype=torch.long, device=input_ids.device, ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + if "decoder_prefill_chunk" in expanded_components and decoder_inputs is None: + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) if media_step_embeds is not None: hidden_size = int(media_step_embeds.shape[-1]) media_chunk_embeds = torch.zeros( @@ -3268,7 +3371,7 @@ def _require(component: str) -> None: metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) cache_seq_len = max(1024, int(input_ids.shape[1]) + 256) - prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "64") or "64")) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "128") or "128")) prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) if "decoder" in expanded_components: if decoder_inputs is None: @@ -3283,17 +3386,21 @@ def _require(component: str) -> None: metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) if "decoder_prefill_chunk" in expanded_components: - if decoder_inputs is None: + if decoder_inputs is not None: + chunk_inputs = tuple( + value[:, :prefill_chunk_size, ...].contiguous() + if value.ndim >= 2 + else value + for value in decoder_inputs + ) + elif prefill_decoder_inputs is not None: + prefill_chunk_size = int(prefill_decoder_inputs[0].shape[1]) + chunk_inputs = tuple(prefill_decoder_inputs) + else: raise RuntimeError("Gemma4 decoder_prefill_chunk spec requires precomputed decoder inputs") - chunk_inputs = tuple( - value[:, :prefill_chunk_size, ...].contiguous() - if value.ndim >= 2 - else value - for value in decoder_inputs - ) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", - module=decoder, + module=decoder_prefill, example_inputs=chunk_inputs, input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, output_keys=("logits",), @@ -3422,11 +3529,15 @@ def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[tor position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=linear_attn_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3443,7 +3554,7 @@ def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[tor return self.model.lm_head(hidden_states), checkpoints def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3488,11 +3599,15 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch. hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=linear_attn_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3502,7 +3617,7 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch. return self.model.lm_head(hidden_states[:, -1:, :]) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3534,6 +3649,26 @@ def _resolve_qwen35_multimodal_backbone(model: torch.nn.Module) -> torch.nn.Modu raise AttributeError(f"{type(model).__name__} does not expose a Qwen3.5 multimodal backbone") +def _qwen35_layer_types(config) -> tuple[str, ...]: + layer_types = tuple(str(value) for value in (getattr(config, "layer_types", ()) or ())) + if layer_types: + return layer_types + num_layers = int(getattr(config, "num_hidden_layers", 0) or 0) + return tuple("full_attention" for _ in range(num_layers)) + + +def _qwen35_attention_mask_for_layer( + config, + layer_index: int, + *, + causal_mask, + linear_attention_mask, +): + layer_types = _qwen35_layer_types(config) + layer_type = layer_types[layer_index] if layer_index < len(layer_types) else "full_attention" + return linear_attention_mask if layer_type == "linear_attention" else causal_mask + + class Qwen35VisionEncoderAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module, *, image_grid_thw: torch.Tensor): super().__init__() @@ -3677,11 +3812,15 @@ def forward( hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = attention_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=attention_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3690,7 +3829,7 @@ def forward( return self.model.lm_head(hidden_states) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3732,6 +3871,10 @@ def get_transpile_metadata(self): } +class Qwen35LMEncoderTextChunkAdapter(Qwen35LMEncoderStepAdapter): + pass + + class Qwen35EmbedsCausalLMStepAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module): super().__init__() @@ -3762,11 +3905,15 @@ def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> to hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = None if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=None, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3775,7 +3922,7 @@ def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> to return self.model.lm_head(hidden_states[:, -1:, :]) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3790,6 +3937,10 @@ def get_transpile_metadata(self): } +class Qwen35EmbedsCausalLMPrefillChunkAdapter(Qwen35EmbedsCausalLMStepAdapter): + pass + + class Qwen3CausalLMLogitsAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): super().__init__() @@ -4011,6 +4162,8 @@ def _build_qwen3_5_multimodal_component_specs( "vision_encoder", "lm_encoder", "decoder", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", "lm_encoder_step", "decoder_media_step", "decoder_step", @@ -4026,9 +4179,16 @@ def _require(component: str) -> None: _require("vision_encoder") _require("lm_encoder") _require("decoder") + _require("lm_encoder_text_chunk") + _require("decoder_prefill_chunk") if "lm_encoder" in requested_set: _require("vision_encoder") _require("lm_encoder") + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "decoder_prefill_chunk" in requested_set: + _require("lm_encoder_text_chunk") + _require("decoder_prefill_chunk") if "vision_encoder" in requested_set: _require("vision_encoder") if "decoder_step" in requested_set: @@ -4042,11 +4202,14 @@ def _require(component: str) -> None: lm_encoder = Qwen35LMEncoderAdapter(model, input_ids=input_ids).eval() decoder = Qwen35EmbedsCausalLMLogitsAdapter(model).eval() lm_encoder_step = Qwen35LMEncoderStepAdapter(model).eval() + lm_encoder_text_chunk = Qwen35LMEncoderTextChunkAdapter(model).eval() decoder_media_step = Qwen35EmbedsCausalLMStepAdapter(model).eval() + decoder_prefill_chunk = Qwen35EmbedsCausalLMPrefillChunkAdapter(model).eval() decoder_step = Qwen35CausalLMStepAdapter(model, pad_token_id=_resolve_model_pad_token_id(model)).eval() image_features: torch.Tensor | None = None decoder_inputs: tuple[torch.Tensor, ...] | None = None + prefill_decoder_inputs: tuple[torch.Tensor, ...] | None = None with torch.no_grad(): if "vision_encoder" in expanded_components and ("lm_encoder" in expanded_components or "decoder" in expanded_components): image_features = vision_encoder(pixel_values) @@ -4054,6 +4217,16 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values) decoder_inputs = lm_encoder(input_ids, attention_mask, position_ids, image_features) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_QWEN_PREFILL_CHUNK", "128") or "128")) + prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) + chunk_input_ids = input_ids[:, :prefill_chunk_size].contiguous() + chunk_position_ids = torch.arange( + prefill_chunk_size, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + if "decoder_prefill_chunk" in expanded_components: + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) common_graph_meta = { "weights_dir": weights_dir, @@ -4107,6 +4280,38 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, )) + if "lm_encoder_text_chunk" in expanded_components: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "position_ids"), + graph_meta={**common_graph_meta, "component": "lm_encoder_text_chunk"}, + metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, + )) + if "decoder_prefill_chunk" in expanded_components: + if prefill_decoder_inputs is None: + raise RuntimeError("Qwen3.5 decoder_prefill_chunk spec requires precomputed text chunk decoder inputs") + max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + specs.append(ComponentModuleSpec( + component="decoder_prefill_chunk", + module=decoder_prefill_chunk, + example_inputs=prefill_decoder_inputs, + input_keys=("inputs_embeds", "position_ids"), + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_prefill_chunk", + "use_internal_kv_cache": True, + "use_internal_conv_cache": True, + "use_internal_gated_deltanet_cache": True, + "max_cache_seq_len": max_cache_seq_len, + "cache_sink_size": 4, + "prefill_chunk_size": prefill_chunk_size, + }, + metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, + )) if "decoder_media_step" in expanded_components: if decoder_inputs is None: raise RuntimeError("Qwen3.5 decoder_media_step spec requires precomputed decoder inputs") @@ -4388,6 +4593,8 @@ def _family_key(model: torch.nn.Module) -> str: return "gemma" if module_name.startswith("transformers.models.qwen3_5."): return "qwen3_5" + if module_name.startswith("transformers.models.qwen3_vl."): + return "qwen3_5" if module_name.startswith("transformers.models.qwen3."): return "qwen3" if module_name.startswith("transformers.models.lfm2_vl."): @@ -4567,7 +4774,7 @@ def _build_lfm2_vl_multimodal_component_specs( dtype=attention_mask.dtype, device=attention_mask.device, ) - prefill_chunk = max(2, int(os.environ.get("CACTUS_LFM2_PREFILL_CHUNK", "64") or "64")) + prefill_chunk = max(2, int(os.environ.get("CACTUS_LFM2_PREFILL_CHUNK", "128") or "128")) prefill_chunk = min(prefill_chunk, int(input_ids.shape[1])) requested_components = tuple(components or ("vision_encoder", "lm_encoder", "text_lm_encoder", "decoder")) @@ -4588,28 +4795,28 @@ def _require(component: str) -> None: _require("decoder") _require("text_decoder") _require("lm_encoder_step") + _require("lm_encoder_text_chunk") _require("decoder_prefill_chunk") _require("decoder_step") - elif "lm_encoder" in requested_set: + if "lm_encoder" in requested_set: _require("vision_encoder") _require("lm_encoder") - elif "lm_encoder_step" in requested_set: + if "lm_encoder_step" in requested_set: _require("lm_encoder_step") - if "decoder_step" in requested_set: - _require("decoder_step") - elif "decoder_step" in requested_set: + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "decoder_step" in requested_set: _require("lm_encoder_step") _require("decoder_step") - elif "decoder_prefill_chunk" in requested_set: + if "decoder_prefill_chunk" in requested_set: + _require("lm_encoder_text_chunk") _require("decoder_prefill_chunk") - elif "text_lm_encoder" in requested_set: + if "text_lm_encoder" in requested_set: _require("text_lm_encoder") - if "text_decoder" in requested_set: - _require("text_decoder") - elif "text_decoder" in requested_set: + if "text_decoder" in requested_set: _require("text_lm_encoder") _require("text_decoder") - elif "vision_encoder" in requested_set: + if "vision_encoder" in requested_set: _require("vision_encoder") vision_encoder = Lfm2VlVisionEncoderAdapter( @@ -4621,6 +4828,7 @@ def _require(component: str) -> None: lm_encoder = Lfm2VlLMEncoderAdapter(model, input_ids=input_ids, weights_dir=weights_dir).eval() text_lm_encoder = Lfm2VlTextLMEncoderAdapter(model, weights_dir=weights_dir).eval() lm_encoder_step = Lfm2VlLMEncoderStepAdapter(model, weights_dir=weights_dir).eval() + lm_encoder_text_chunk = Lfm2VlLMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() decoder = Lfm2VlDecoderAdapter(model, weights_dir=weights_dir, last_token_only=False).eval() decoder_last_token = Lfm2VlDecoderAdapter(model, weights_dir=weights_dir, last_token_only=True).eval() @@ -4637,6 +4845,13 @@ def _require(component: str) -> None: if "text_lm_encoder" in expanded_components: text_decoder_inputs = text_lm_encoder(text_input_ids, text_attention_mask) + chunk_input_ids = input_ids[:, :prefill_chunk].contiguous() + chunk_position_ids = torch.arange( + prefill_chunk, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + common_graph_meta = { "weights_dir": weights_dir, "task": "multimodal_causal_lm_logits", @@ -4715,6 +4930,20 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, )) + if "lm_encoder_text_chunk" in expanded_components: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "attention_mask", "position_ids"), + graph_meta={ + **common_graph_meta, + "component": "lm_encoder_text_chunk", + "encoder_chunk_size": prefill_chunk, + }, + metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, + )) if "decoder_prefill_chunk" in expanded_components: if decoder_inputs is None: if image_features is None: diff --git a/python/cactus/transpile/model_profiles.py b/python/cactus/transpile/model_profiles.py index 7e84e4c69..6a07d9c7c 100644 --- a/python/cactus/transpile/model_profiles.py +++ b/python/cactus/transpile/model_profiles.py @@ -166,7 +166,7 @@ class ModelProfile: QWEN_PROFILE = ModelProfile( family="qwen", - model_types=("qwen", "qwen2", "qwen3", "qwen3_5", "qwen3.5"), + model_types=("qwen", "qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"), multimodal_context_tokens=512, input_combinations=((), ("image",)), model_id_aliases=( @@ -177,16 +177,25 @@ class ModelProfile: ("qwen3", "Qwen/Qwen3-1.7B"), ), model_id_markers=("qwen",), - family_aliases=("qwen2", "qwen3", "qwen3_5", "qwen3.5"), + family_aliases=("qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"), stop_tokens=("<|im_end|>",), avoid_native_loader=True, - cached_step_components=("decoder_step",), + cached_step_components=("decoder_media_step",), cached_step_skip_components=("decoder",), - fp16_kv_cache_components=("decoder_step",), + fp16_kv_cache_components=("decoder_prefill_chunk", "decoder_media_step", "decoder_step"), prompt_style="qwen_chat", multimodal_preprocessor="qwen3_5", default_task="multimodal_causal_lm_logits", - default_components=("vision_encoder", "lm_encoder", "decoder", "lm_encoder_step", "decoder_media_step", "decoder_step"), + default_components=( + "vision_encoder", + "lm_encoder", + "decoder", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + "lm_encoder_step", + "decoder_media_step", + "decoder_step", + ), needs_image=True, force_component_pipeline=True, ) diff --git a/python/cactus/transpile/runtime_compat.py b/python/cactus/transpile/runtime_compat.py index 677b15cb3..9c17c722e 100644 --- a/python/cactus/transpile/runtime_compat.py +++ b/python/cactus/transpile/runtime_compat.py @@ -233,31 +233,31 @@ def permute(self, x, permutation, backend=Graph.CPU): return orig_permute(self, _ensure_scalar_tensor(self, x), permutation, backend=backend) def scalar_add(self, x, value): - return orig_scalar_add(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_add(self, self._ensure_tensor(x), value) def scalar_subtract(self, x, value): - return orig_scalar_subtract(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_subtract(self, self._ensure_tensor(x), value) def scalar_multiply(self, x, value): - return orig_scalar_multiply(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_multiply(self, self._ensure_tensor(x), value) def scalar_divide(self, x, value): - return orig_scalar_divide(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_divide(self, self._ensure_tensor(x), value) def scalar_exp(self, x): - return orig_scalar_exp(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_exp(self, self._ensure_tensor(x)) def scalar_sqrt(self, x): - return orig_scalar_sqrt(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_sqrt(self, self._ensure_tensor(x)) def scalar_cos(self, x): - return orig_scalar_cos(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_cos(self, self._ensure_tensor(x)) def scalar_sin(self, x): - return orig_scalar_sin(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_sin(self, self._ensure_tensor(x)) def scalar_log(self, x): - return orig_scalar_log(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_log(self, self._ensure_tensor(x)) def clamp(self, x, lo, hi): if orig_clamp is None: @@ -284,10 +284,10 @@ def cat(self, tensors, axis=0): return orig_cat(self, legalized, axis=axis) def abs(self, x): - return orig_abs(self, _ensure_scalar_tensor(self, x)) + return orig_abs(self, self._ensure_tensor(x)) def pow(self, x, exponent): - return orig_pow(self, _ensure_scalar_tensor(self, x), exponent) + return orig_pow(self, self._ensure_tensor(x), exponent) def expand(self, x, shape): x = self._ensure_tensor(x) diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py index 676c2eeb2..6061ddbc9 100644 --- a/python/tests/test_cli_transpile_defaults.py +++ b/python/tests/test_cli_transpile_defaults.py @@ -30,7 +30,7 @@ def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[s "--task", "multimodal_causal_lm_logits", "--max-new-tokens", - "32", + "512", "--component-pipeline", "on", "--prompt", @@ -198,14 +198,24 @@ def _fake_cmd_transpile(transpile_args): assert "--audio-file" in extra_args -def test_cmd_convert_infers_text_tasks_for_qwen_and_lfm(monkeypatch, tmp_path: Path) -> None: +def test_cmd_convert_infers_profile_defaults_for_qwen_and_lfm(monkeypatch, tmp_path: Path) -> None: parser = cli.create_parser() import cactus.convert.cli as cq_cli - for alias, model_type, arch in ( - ("qwen", "qwen3", "Qwen3ForCausalLM"), - ("lfm", "lfm2_vl", "Lfm2VlForConditionalGeneration"), + for alias, model_type, arch, expected_components in ( + ( + "qwen", + "qwen3", + "Qwen3ForCausalLM", + "vision_encoder,lm_encoder,decoder,lm_encoder_text_chunk,decoder_prefill_chunk,lm_encoder_step,decoder_media_step,decoder_step", + ), + ( + "lfm", + "lfm2_vl", + "Lfm2VlForConditionalGeneration", + "vision_encoder,lm_encoder,decoder", + ), ): output_dir = tmp_path / alias args = parser.parse_args(["convert", alias, str(output_dir)]) @@ -230,7 +240,10 @@ def _fake_cmd_transpile(transpile_args): assert rc == 0 extra_args = transpile_calls[0].extra_args - assert extra_args[extra_args.index("--task") + 1] == "causal_lm_logits" + assert extra_args[extra_args.index("--task") + 1] == "multimodal_causal_lm_logits" + assert extra_args[extra_args.index("--component-pipeline") + 1] == "on" + assert extra_args[extra_args.index("--components") + 1] == expected_components + assert "--image-file" in extra_args assert "--audio-file" not in extra_args From 989a64325171e206a7a8c813e3e1b08a1c1cfa01 Mon Sep 17 00:00:00 2001 From: justinl66 Date: Thu, 21 May 2026 20:29:37 -0700 Subject: [PATCH 3/8] parakeet npu working + gemma progress --- cactus-engine/src/engine.h | 19 +- cactus-engine/src/model.cpp | 95 ++++- cactus-engine/src/model_npu.cpp | 344 ++++++++++++++++ cactus-engine/src/npu_ane.h | 14 +- cactus-engine/src/npu_ane.mm | 151 ++++--- local_integration_tests/native_runner.cpp | 4 + python/cactus/transpile/hf_model.py | 19 +- python/cactus/transpile/npu/README.md | 110 ++++++ python/cactus/transpile/npu/__init__.py | 12 + python/cactus/transpile/npu/audio.py | 149 +++++++ .../transpile/npu/coremltools_patches.py | 373 ++++++++++++++++++ python/cactus/transpile/npu/pipeline.py | 150 +++++++ python/cactus/transpile/npu/prefill.py | 213 ++++++++++ python/cactus/transpile/npu/vision.py | 149 +++++++ 14 files changed, 1731 insertions(+), 71 deletions(-) create mode 100644 cactus-engine/src/model_npu.cpp create mode 100644 python/cactus/transpile/npu/README.md create mode 100644 python/cactus/transpile/npu/__init__.py create mode 100644 python/cactus/transpile/npu/audio.py create mode 100644 python/cactus/transpile/npu/coremltools_patches.py create mode 100644 python/cactus/transpile/npu/pipeline.py create mode 100644 python/cactus/transpile/npu/prefill.py create mode 100644 python/cactus/transpile/npu/vision.py diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 2e7e769d5..f37e961a9 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -70,10 +70,9 @@ class NPUPrefill { virtual int get_num_layers() const = 0; virtual int get_num_kv_heads() const = 0; virtual int get_head_dim() const = 0; - virtual NPUPrefillDirectResult prefill_chunk_direct( - const std::vector<__fp16>& embeddings, - int position_offset = 0, - const std::string& input_name = "x") = 0; + virtual NPUPrefillDirectResult prefill_chunk_tokens( + const std::vector& input_ids, + const std::vector& position_ids) = 0; }; std::unique_ptr create_prefill(); @@ -634,6 +633,12 @@ class Model { bool has_npu_prefill() const { return npu_prefill_ != nullptr; } size_t get_prefill_chunk_size() const { return 128; } + bool load_npu_audio_encoder(const std::string& model_path); + bool has_npu_audio_encoder() const { return npu_audio_encoder_ != nullptr; } + + bool load_npu_vision_encoder(const std::string& model_path); + bool has_npu_vision_encoder() const { return npu_vision_encoder_ != nullptr; } + void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} @@ -729,11 +734,17 @@ class Model { std::string family_; std::string npu_prefill_mlpackage_; + std::string npu_audio_encoder_mlpackage_; + std::string npu_vision_encoder_mlpackage_; std::unique_ptr npu_prefill_; + std::unique_ptr npu_audio_encoder_; + std::unique_ptr npu_vision_encoder_; bool prefill_via_npu(const std::vector& tokens); void write_npu_kv_to_cache(const npu::NPUPrefillDirectResult& result, size_t prefill_len); + bool audio_encode_via_npu(const std::vector& audio_features); + bool vision_encode_via_npu(const std::vector& pixel_values); std::map> media_features_; std::map> media_feature_shapes_; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index e31440a09..bc85b0c9b 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -315,6 +315,18 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_WARN("model", "NPU prefill load failed for " << full_path << "; falling back to CPU"); } } + if (!npu_audio_encoder_mlpackage_.empty()) { + std::string full_path = bundle_dir + "/" + npu_audio_encoder_mlpackage_; + if (!load_npu_audio_encoder(full_path)) { + CACTUS_LOG_WARN("model", "NPU audio encoder load failed for " << full_path << "; falling back to CPU"); + } + } + if (!npu_vision_encoder_mlpackage_.empty()) { + std::string full_path = bundle_dir + "/" + npu_vision_encoder_mlpackage_; + if (!load_npu_vision_encoder(full_path)) { + CACTUS_LOG_WARN("model", "NPU vision encoder load failed for " << full_path << "; falling back to CPU"); + } + } initialized_ = true; return true; @@ -336,6 +348,12 @@ bool Model::load_manifest() { if (obj.count("npu_prefill") && obj.at("npu_prefill").is()) { npu_prefill_mlpackage_ = obj.at("npu_prefill").get(); } + if (obj.count("npu_audio_encoder") && obj.at("npu_audio_encoder").is()) { + npu_audio_encoder_mlpackage_ = obj.at("npu_audio_encoder").get(); + } + if (obj.count("npu_vision_encoder") && obj.at("npu_vision_encoder").is()) { + npu_vision_encoder_mlpackage_ = obj.at("npu_vision_encoder").get(); + } if (!obj.count("components")) return false; for (const auto& cv : obj.at("components").get()) { const auto& c = cv.get(); @@ -942,10 +960,13 @@ void Model::run_media_step(size_t position, const uint8_t* feature_row, size_t f void Model::run_vision_encoder(const std::string& image_path) { if (!vision_encoder_) return; + Gemma4ImagePreprocessed prep = preprocess_gemma4_image(image_path, config_); + if (has_npu_vision_encoder() && vision_encode_via_npu(prep.pixel_values)) { + return; + } if (!load_component_graph(*vision_encoder_)) { throw std::runtime_error("failed to load vision_encoder"); } - Gemma4ImagePreprocessed prep = preprocess_gemma4_image(image_path, config_); write_bytes_input(*vision_encoder_, "pixel_values", prep.pixel_values.data(), prep.pixel_values.size() * sizeof(float)); write_bytes_input(*vision_encoder_, "pixel_position_ids", prep.pixel_position_ids.data(), @@ -989,6 +1010,9 @@ void Model::run_audio_encoder_messages(const std::vector>& au void Model::run_audio_encoder(const std::vector& audio_features) { if (!audio_encoder_) return; + if (has_npu_audio_encoder() && audio_encode_via_npu(audio_features)) { + return; + } const std::vector candidate_input_names = {"input_features", "audio_features"}; int feature_idx = -1; for (const auto& name : candidate_input_names) { @@ -1916,7 +1940,60 @@ std::vector Model::transcribe_parakeet_tdt(const std::vector& a write_typed_buffer(feat_buf, feat_desc.precision, transposed.data(), transposed.size() * sizeof(float), Precision::FP32); - audio_enc->graph->execute(); + // Try NPU audio encoder first; fall back to CPU graph on failure. + // The NPU mlpackage has a fixed input window (e.g. 500 frames -> 63 hidden frames), + // so we chunk the input and concatenate chunk outputs into a buffer sized to the + // CPU graph's expected T (subsampling preserves the chunk:hidden ratio). + std::vector<__fp16> npu_hidden_storage; + bool used_npu = false; + size_t npu_hidden_T = 0; + if (has_npu_audio_encoder()) { + const std::vector in_shape = npu_audio_encoder_->get_input_shape(); + const std::vector out_shape = npu_audio_encoder_->get_output_shape(); + if (in_shape.size() >= 3 && out_shape.size() >= 3 && + in_shape[1] > 0 && in_shape[2] > 0 && out_shape[1] > 0 && out_shape[2] > 0 && + static_cast(in_shape[2]) == expected_mels) { + const size_t window_frames = static_cast(in_shape[1]); + const size_t window_hidden = static_cast(out_shape[1]); + const size_t hidden_dim_npu = static_cast(out_shape[2]); + const size_t chunk_input_elems = window_frames * expected_mels; + const size_t chunk_output_elems = window_hidden * hidden_dim_npu; + const size_t num_chunks = (copy_frames + window_frames - 1) / window_frames; + const size_t total_hidden_T = num_chunks * window_hidden; + npu_hidden_storage.assign(total_hidden_T * hidden_dim_npu, __fp16(0)); + std::vector<__fp16> input_fp16(chunk_input_elems); + bool all_ok = num_chunks > 0; + for (size_t c = 0; c < num_chunks && all_ok; ++c) { + const size_t frame_start = c * window_frames; + const size_t frame_end = std::min(frame_start + window_frames, copy_frames); + std::fill(input_fp16.begin(), input_fp16.end(), __fp16(0)); + for (size_t t = frame_start; t < frame_end; ++t) { + const size_t local = (t - frame_start) * expected_mels; + const size_t src = t * expected_mels; + for (size_t m = 0; m < expected_mels; ++m) { + input_fp16[local + m] = static_cast<__fp16>(transposed[src + m]); + } + } + __fp16* out_ptr = npu_hidden_storage.data() + c * chunk_output_elems; + size_t written = npu_audio_encoder_->encode( + input_fp16.data(), out_ptr, in_shape, "x", "encoded"); + if (written == 0) { all_ok = false; break; } + } + if (all_ok) { + used_npu = true; + const size_t valid_input = copy_frames; + npu_hidden_T = (valid_input * window_hidden + window_frames - 1) / window_frames; + if (npu_hidden_T > total_hidden_T) npu_hidden_T = total_hidden_T; + CACTUS_LOG_INFO("model", "Parakeet audio encoder ran on NPU (" + << num_chunks << " chunks, " << npu_hidden_T << " valid hidden frames)"); + } else { + CACTUS_LOG_WARN("model", "NPU audio encoder chunk failed; falling back to CPU graph"); + } + } + } + if (!used_npu) { + audio_enc->graph->execute(); + } int hidden_idx = output_index(*audio_enc, "encoder_hidden_states"); if (hidden_idx < 0) { @@ -1925,14 +2002,20 @@ std::vector Model::transcribe_parakeet_tdt(const std::vector& a } size_t hidden_node = static_cast(audio_enc->output_node_ids[hidden_idx]); const auto& hidden_desc = audio_enc->graph->get_output_buffer(hidden_node); - const uint8_t* hidden_ptr = static_cast(audio_enc->graph->get_output(hidden_node)); + const uint8_t* hidden_ptr; + if (used_npu) { + hidden_ptr = reinterpret_cast(npu_hidden_storage.data()); + } else { + hidden_ptr = static_cast(audio_enc->graph->get_output(hidden_node)); + } if (hidden_desc.shape.size() < 3 || hidden_ptr == nullptr) { CACTUS_LOG_ERROR("model", "encoder_hidden_states must be 3D [B, T, D]"); return emitted; } - const size_t T = hidden_desc.shape[1]; + const size_t T = used_npu ? npu_hidden_T : hidden_desc.shape[1]; const size_t D = hidden_desc.shape[2]; - const size_t hidden_elem = PrecisionTraits::size_of(hidden_desc.precision); + const Precision hidden_precision = used_npu ? Precision::FP16 : hidden_desc.precision; + const size_t hidden_elem = PrecisionTraits::size_of(hidden_precision); const size_t frame_bytes = D * hidden_elem; auto zero_state = [&](const std::string& name) { @@ -1998,7 +2081,7 @@ std::vector Model::transcribe_parakeet_tdt(const std::vector& a while (time_index < T) { const uint8_t* frame_ptr = hidden_ptr + time_index * frame_bytes; - write_typed_buffer(ef_buf, ef_desc.precision, frame_ptr, frame_bytes, hidden_desc.precision); + write_typed_buffer(ef_buf, ef_desc.precision, frame_ptr, frame_bytes, hidden_precision); size_t symbols_added = 0; bool advanced = false; diff --git a/cactus-engine/src/model_npu.cpp b/cactus-engine/src/model_npu.cpp new file mode 100644 index 000000000..a734a03fc --- /dev/null +++ b/cactus-engine/src/model_npu.cpp @@ -0,0 +1,344 @@ +#include "engine.h" +#include "cactus_graph.h" +#include "cactus_kernels.h" + +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +namespace { + +struct NPUCacheHeader { + uint64_t current_seq_len; + uint64_t max_seq_len; + uint64_t num_kv_heads; + uint64_t head_dim; + uint64_t sink_size; + uint64_t reserved[3]; +}; +constexpr size_t kNPUCacheHeaderBytes = 64; +static_assert(sizeof(NPUCacheHeader) == kNPUCacheHeaderBytes, "NPUCacheHeader layout mismatch"); + +bool is_gemma_family(const std::string& family) { + return family.find("gemma") != std::string::npos; +} + +} // namespace + +bool Model::load_npu_prefill(const std::string& model_path) { + auto prefill = npu::create_prefill(); + if (!prefill) return false; + if (!prefill->load(model_path)) return false; + if (!prefill->is_available()) return false; + npu_prefill_ = std::move(prefill); + CACTUS_LOG_INFO("model", "NPU prefill loaded: chunk=" << npu_prefill_->get_chunk_size() + << " hidden=" << npu_prefill_->get_hidden_dim() + << " layers=" << npu_prefill_->get_num_layers() + << " kv_heads=" << npu_prefill_->get_num_kv_heads() + << " head_dim=" << npu_prefill_->get_head_dim()); + return true; +} + +bool Model::load_npu_audio_encoder(const std::string& model_path) { + auto encoder = npu::create_encoder(); + if (!encoder) return false; + if (!encoder->load(model_path)) return false; + if (!encoder->is_available()) return false; + npu_audio_encoder_ = std::move(encoder); + CACTUS_LOG_INFO("model", "NPU audio encoder loaded from: " << model_path); + return true; +} + +bool Model::load_npu_vision_encoder(const std::string& model_path) { + auto encoder = npu::create_encoder(); + if (!encoder) return false; + if (!encoder->load(model_path)) return false; + if (!encoder->is_available()) return false; + npu_vision_encoder_ = std::move(encoder); + CACTUS_LOG_INFO("model", "NPU vision encoder loaded from: " << model_path); + return true; +} + +bool Model::audio_encode_via_npu(const std::vector& audio_features) { + if (!npu_audio_encoder_ || !npu_audio_encoder_->is_available() || !audio_encoder_) { + return false; + } + const std::vector input_shape = npu_audio_encoder_->get_input_shape(); + if (input_shape.empty()) return false; + + size_t expected_elems = 1; + for (int d : input_shape) { + if (d <= 0) return false; + expected_elems *= static_cast(d); + } + if (audio_features.size() > expected_elems) return false; + + std::vector<__fp16> input_fp16(expected_elems, __fp16(0)); + for (size_t i = 0; i < audio_features.size(); ++i) { + input_fp16[i] = static_cast<__fp16>(audio_features[i]); + } + + const std::vector output_shape = npu_audio_encoder_->get_output_shape(); + size_t output_elems = 1; + for (int d : output_shape) { + if (d <= 0) { output_elems = 0; break; } + output_elems *= static_cast(d); + } + if (output_elems == 0) { + output_elems = npu_audio_encoder_->get_output_buffer_size(); + } + std::vector<__fp16> output_fp16(output_elems, __fp16(0)); + + size_t written = npu_audio_encoder_->encode( + input_fp16.data(), output_fp16.data(), input_shape, "x", "encoded"); + if (written == 0) return false; + + // Copy NPU output into the CPU audio_encoder's primary output buffer so + // downstream code (dynamic walker / media_features_) sees the same path. + for (size_t i = 0; i < audio_encoder_->output_node_ids.size() + && i < audio_encoder_->logical_outputs.size(); ++i) { + const std::string& name = audio_encoder_->logical_outputs[i]; + size_t node_id = static_cast(audio_encoder_->output_node_ids[i]); + const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); + const size_t copy_bytes = std::min(desc.byte_size, written * sizeof(__fp16)); + auto& slot = media_features_[name]; + const size_t prev = slot.size(); + slot.resize(prev + copy_bytes); + if (desc.precision == Precision::FP16) { + std::memcpy(slot.data() + prev, output_fp16.data(), copy_bytes); + } else if (desc.precision == Precision::FP32) { + const size_t n = copy_bytes / sizeof(__fp16); + float* dst = reinterpret_cast(slot.data() + prev); + for (size_t k = 0; k < n; ++k) dst[k] = static_cast(output_fp16[k]); + } else { + std::memcpy(slot.data() + prev, output_fp16.data(), copy_bytes); + } + auto shape_it = media_feature_shapes_.find(name); + if (shape_it == media_feature_shapes_.end() || shape_it->second.empty()) { + std::vector shape; + for (int d : output_shape) shape.push_back(static_cast(d)); + media_feature_shapes_[name] = std::move(shape); + } + media_feature_precisions_[name] = desc.precision; + break; // audio encoder only has one logical output + } + return true; +} + +bool Model::vision_encode_via_npu(const std::vector& pixel_values) { + if (!npu_vision_encoder_ || !npu_vision_encoder_->is_available() || !vision_encoder_) { + return false; + } + const std::vector input_shape = npu_vision_encoder_->get_input_shape(); + if (input_shape.empty()) return false; + + size_t expected_elems = 1; + for (int d : input_shape) { + if (d <= 0) return false; + expected_elems *= static_cast(d); + } + if (pixel_values.size() > expected_elems) return false; + + std::vector<__fp16> input_fp16(expected_elems, __fp16(0)); + for (size_t i = 0; i < pixel_values.size(); ++i) { + input_fp16[i] = static_cast<__fp16>(pixel_values[i]); + } + + const std::vector output_shape = npu_vision_encoder_->get_output_shape(); + size_t output_elems = 1; + for (int d : output_shape) { + if (d <= 0) { output_elems = 0; break; } + output_elems *= static_cast(d); + } + if (output_elems == 0) { + output_elems = npu_vision_encoder_->get_output_buffer_size(); + } + std::vector<__fp16> output_fp16(output_elems, __fp16(0)); + + size_t written = npu_vision_encoder_->encode( + input_fp16.data(), output_fp16.data(), input_shape, "x", "encoded"); + if (written == 0) return false; + + // Copy NPU output into the CPU vision_encoder's primary output buffer so + // downstream code (dynamic walker / media_features_) sees the same path. + for (size_t i = 0; i < vision_encoder_->output_node_ids.size() + && i < vision_encoder_->logical_outputs.size(); ++i) { + const std::string& name = vision_encoder_->logical_outputs[i]; + size_t node_id = static_cast(vision_encoder_->output_node_ids[i]); + const auto& desc = vision_encoder_->graph->get_output_buffer(node_id); + const size_t copy_bytes = std::min(desc.byte_size, written * sizeof(__fp16)); + auto& slot = media_features_[name]; + slot.assign(copy_bytes, 0); + if (desc.precision == Precision::FP16) { + std::memcpy(slot.data(), output_fp16.data(), copy_bytes); + } else if (desc.precision == Precision::FP32) { + const size_t n = copy_bytes / sizeof(__fp16); + slot.assign(n * sizeof(float), 0); + float* dst = reinterpret_cast(slot.data()); + for (size_t k = 0; k < n; ++k) dst[k] = static_cast(output_fp16[k]); + } else { + std::memcpy(slot.data(), output_fp16.data(), copy_bytes); + } + std::vector shape; + for (int d : output_shape) shape.push_back(static_cast(d)); + media_feature_shapes_[name] = std::move(shape); + media_feature_precisions_[name] = desc.precision; + break; // vision encoder has one logical output + } + return true; +} + +bool Model::prefill_via_npu(const std::vector& tokens) { + if (!npu_prefill_ || !npu_prefill_->is_available() || !decoder_) { + return false; + } + if (tokens.empty()) return true; + + const int chunk_size = npu_prefill_->get_chunk_size(); + if (chunk_size <= 0) return false; + + const size_t num_tokens = tokens.size(); + const size_t num_chunks = (num_tokens + chunk_size - 1) / chunk_size; + + std::vector chunk_ids(chunk_size, 0); + std::vector chunk_positions(chunk_size, 0); + + for (size_t c = 0; c < num_chunks; ++c) { + const size_t start = c * static_cast(chunk_size); + const size_t actual_tokens = std::min(static_cast(chunk_size), num_tokens - start); + + std::fill(chunk_ids.begin(), chunk_ids.end(), 0); + std::fill(chunk_positions.begin(), chunk_positions.end(), 0); + for (size_t i = 0; i < actual_tokens; ++i) { + chunk_ids[i] = static_cast(tokens[start + i]); + chunk_positions[i] = static_cast(cache_total_seq_len_ + start + i); + } + + auto direct = npu_prefill_->prefill_chunk_tokens(chunk_ids, chunk_positions); + if (!direct.valid) { + CACTUS_LOG_WARN("model", "NPU prefill chunk " << c << " returned invalid result"); + return false; + } + write_npu_kv_to_cache(direct, actual_tokens); + } + cache_total_seq_len_ += num_tokens; + (void)is_gemma_family(family_); // family info still useful for future scaling work + return true; +} + +void Model::write_npu_kv_to_cache(const npu::NPUPrefillDirectResult& result, size_t actual_tokens) { + if (!decoder_) return; + if (actual_tokens == 0) return; + + const size_t n_layers = std::min(result.k_caches.size(), decoder_->cache_states.size()); + if (n_layers == 0) return; + + for (size_t layer_idx = 0; layer_idx < n_layers; ++layer_idx) { + const auto& cs = decoder_->cache_states[layer_idx]; + const auto& k_ref = result.k_caches[layer_idx]; + const auto& v_ref = result.v_caches[layer_idx]; + if (!k_ref.data || !v_ref.data) continue; + + const int node_ids[2] = {cs.key_node_id, cs.value_node_id}; + const npu::NPUBufferRef refs[2] = {k_ref, v_ref}; + for (int side = 0; side < 2; ++side) { + const int node_id = node_ids[side]; + if (node_id < 0) continue; + const auto& desc = decoder_->graph->get_output_buffer(static_cast(node_id)); + if (desc.byte_size <= kNPUCacheHeaderBytes || !desc.get_data()) continue; + + void* raw = decoder_->graph->get_output(static_cast(node_id)); + if (!raw) continue; + auto* hdr = static_cast(raw); + + const size_t kv_heads = hdr->num_kv_heads; + const size_t hdim = hdr->head_dim; + const size_t max_len = hdr->max_seq_len; + const size_t sink = hdr->sink_size; + if (kv_heads == 0 || hdim == 0 || max_len == 0) continue; + + const size_t current_len = hdr->current_seq_len; + const size_t new_total = current_len + actual_tokens; + const size_t window = max_len; + const size_t token_elems = kv_heads * hdim; + const __fp16* source = refs[side].data; + + if (desc.precision == Precision::FP16) { + __fp16* base = reinterpret_cast<__fp16*>(static_cast(raw) + kNPUCacheHeaderBytes); + if (new_total <= window) { + std::memcpy(base + current_len * token_elems, source, + actual_tokens * token_elems * sizeof(__fp16)); + hdr->current_seq_len = new_total; + } else if (actual_tokens >= window) { + const size_t skip = actual_tokens - window; + std::memcpy(base, source + skip * token_elems, + window * token_elems * sizeof(__fp16)); + hdr->current_seq_len = window; + } else { + const size_t sink_eff = std::min(sink, window); + const size_t remaining = (actual_tokens + sink_eff < window) + ? window - sink_eff - actual_tokens : 0; + const size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; + if (remaining > 0 && shift_src > sink_eff) { + std::memmove(base + sink_eff * token_elems, + base + shift_src * token_elems, + remaining * token_elems * sizeof(__fp16)); + } + const size_t append_offset = window - actual_tokens; + std::memcpy(base + append_offset * token_elems, source, + actual_tokens * token_elems * sizeof(__fp16)); + hdr->current_seq_len = window; + } + } else { + int8_t* int8_base = reinterpret_cast(static_cast(raw) + kNPUCacheHeaderBytes); + const size_t num_groups = (hdim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + const size_t scale_stride = kv_heads * num_groups; + float* scale_base = reinterpret_cast(static_cast(raw) + kNPUCacheHeaderBytes + + max_len * kv_heads * hdim); + if (new_total <= window) { + cactus_quantize_kv_fp16_to_int8( + source, + int8_base + current_len * token_elems, + scale_base + current_len * scale_stride, + actual_tokens, kv_heads, hdim); + hdr->current_seq_len = new_total; + } else if (actual_tokens >= window) { + const size_t skip = actual_tokens - window; + cactus_quantize_kv_fp16_to_int8( + source + skip * token_elems, + int8_base, + scale_base, + window, kv_heads, hdim); + hdr->current_seq_len = window; + } else { + const size_t sink_eff = std::min(sink, window); + const size_t remaining = (actual_tokens + sink_eff < window) + ? window - sink_eff - actual_tokens : 0; + const size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; + if (remaining > 0 && shift_src > sink_eff) { + std::memmove(int8_base + sink_eff * token_elems, + int8_base + shift_src * token_elems, + remaining * token_elems); + std::memmove(scale_base + sink_eff * scale_stride, + scale_base + shift_src * scale_stride, + remaining * scale_stride * sizeof(float)); + } + const size_t append_offset = window - actual_tokens; + cactus_quantize_kv_fp16_to_int8( + source, + int8_base + append_offset * token_elems, + scale_base + append_offset * scale_stride, + actual_tokens, kv_heads, hdim); + hdr->current_seq_len = window; + } + } + } + } +} + +} // namespace engine +} // namespace cactus diff --git a/cactus-engine/src/npu_ane.h b/cactus-engine/src/npu_ane.h index 29f0e8c8c..f38dfb6bf 100644 --- a/cactus-engine/src/npu_ane.h +++ b/cactus-engine/src/npu_ane.h @@ -77,10 +77,9 @@ class ANEPrefill : public NPUPrefill { int get_num_kv_heads() const override; int get_head_dim() const override; - NPUPrefillDirectResult prefill_chunk_direct( - const std::vector<__fp16>& embeddings, - int position_offset = 0, - const std::string& input_name = "x") override; + NPUPrefillDirectResult prefill_chunk_tokens( + const std::vector& input_ids, + const std::vector& position_ids) override; private: void* impl_; @@ -140,10 +139,9 @@ class ANEPrefill : public NPUPrefill { int get_num_kv_heads() const override { return 0; } int get_head_dim() const override { return 0; } - NPUPrefillDirectResult prefill_chunk_direct( - const std::vector<__fp16>&, - int = 0, - const std::string& = "x") override { return {}; } + NPUPrefillDirectResult prefill_chunk_tokens( + const std::vector&, + const std::vector&) override { return {}; } }; #endif // CACTUS_HAS_ANE diff --git a/cactus-engine/src/npu_ane.mm b/cactus-engine/src/npu_ane.mm index 26ffb74cf..ab3616362 100644 --- a/cactus-engine/src/npu_ane.mm +++ b/cactus-engine/src/npu_ane.mm @@ -961,17 +961,19 @@ @interface CactusANEPrefillImpl : NSObject @property (nonatomic, assign) int numKvHeads; @property (nonatomic, assign) int headDim; -@property (nonatomic, strong) MLMultiArray* cachedInputArray; -@property (nonatomic, strong) MLMultiArray* cachedOffsetArray; +@property (nonatomic, strong) MLMultiArray* cachedInputIdsArray; +@property (nonatomic, strong) MLMultiArray* cachedPositionIdsArray; +@property (nonatomic, strong) NSString* inputIdsName; +@property (nonatomic, strong) NSString* positionIdsName; @property (nonatomic, strong) NSMutableDictionary* cachedOutputArrays; @property (nonatomic, strong) MLPredictionOptions* predictionOptions; @property (nonatomic, assign) BOOL hasOutputBackings; - (instancetype)initWithModelPath:(NSString*)path; - (void)preallocateBuffers; -- (BOOL)predictDirectWithInput:(NSString*)inputName - data:(const __fp16*)data - offset:(int)offset; +- (BOOL)predictWithInputIds:(const int32_t*)inputIds + positionIds:(const int32_t*)positionIds + count:(NSUInteger)count; - (MLMultiArray*)getOutputArray:(NSString*)name; @end @@ -1006,13 +1008,34 @@ - (instancetype)initWithModelPath:(NSString*)path { - (void)inferModelDimensions { if (!_modelDescription) return; - NSString* inputName = _modelDescription.inputDescriptionsByName.allKeys.firstObject; - MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; - if (inputDesc && inputDesc.type == MLFeatureTypeMultiArray) { - NSArray* shape = inputDesc.multiArrayConstraint.shape; + // Token-based interface: look for input_ids (chunk size) and position_ids. + NSDictionary* inputs = _modelDescription.inputDescriptionsByName; + for (NSString* candidate in @[@"input_ids", @"tokens", @"ids"]) { + if (inputs[candidate]) { _inputIdsName = candidate; break; } + } + for (NSString* candidate in @[@"position_ids", @"positions", @"pos_ids"]) { + if (inputs[candidate]) { _positionIdsName = candidate; break; } + } + if (!_inputIdsName) { + _inputIdsName = inputs.allKeys.firstObject; + } + MLFeatureDescription* idsDesc = inputs[_inputIdsName]; + if (idsDesc && idsDesc.type == MLFeatureTypeMultiArray) { + NSArray* shape = idsDesc.multiArrayConstraint.shape; + // expected shape: [1, chunk_size] or [chunk_size] if (shape.count >= 2) { + _chunkSize = [shape[shape.count - 1] intValue]; + } else if (shape.count == 1) { _chunkSize = [shape[0] intValue]; - _hiddenDim = [shape[1] intValue]; + } + } + + // hidden_dim from the "hidden" output (last dim). + MLFeatureDescription* hiddenDesc = _modelDescription.outputDescriptionsByName[@"hidden"]; + if (hiddenDesc && hiddenDesc.type == MLFeatureTypeMultiArray) { + NSArray* shape = hiddenDesc.multiArrayConstraint.shape; + if (shape.count >= 1) { + _hiddenDim = [shape[shape.count - 1] intValue]; } } @@ -1025,7 +1048,20 @@ - (void)inferModelDimensions { MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { NSArray* shape = outputDesc.multiArrayConstraint.shape; - if (shape.count >= 3) { + // expected K/V shape: [batch, seq, num_kv_heads, head_dim] or [batch, num_kv_heads, seq, head_dim] + if (shape.count >= 4) { + NSInteger d2 = [shape[2] intValue]; + NSInteger d3 = [shape[3] intValue]; + if (d2 == _chunkSize) { + // [b, num_kv_heads, seq, head_dim] + _numKvHeads = [shape[1] intValue]; + _headDim = (int)d3; + } else { + // assume [b, seq, num_kv_heads, head_dim] + _numKvHeads = (int)d2; + _headDim = (int)d3; + } + } else if (shape.count == 3) { _numKvHeads = [shape[1] intValue]; _headDim = [shape[2] intValue]; } @@ -1036,25 +1072,33 @@ - (void)inferModelDimensions { } - (void)preallocateBuffers { - if (!_model || !_modelDescription || _chunkSize == 0 || _hiddenDim == 0) return; + if (!_model || !_modelDescription || _chunkSize == 0) return; NSError* error = nil; - NSArray* inputShape = @[@(_chunkSize), @(_hiddenDim)]; - _cachedInputArray = [[MLMultiArray alloc] initWithShape:inputShape - dataType:MLMultiArrayDataTypeFloat16 - error:&error]; + NSArray* tokenShape = @[@1, @(_chunkSize)]; + MLMultiArrayDataType tokenDtype = MLMultiArrayDataTypeInt32; + MLFeatureDescription* idsDesc = _modelDescription.inputDescriptionsByName[_inputIdsName]; + if (idsDesc) { + tokenDtype = idsDesc.multiArrayConstraint.dataType; + tokenShape = idsDesc.multiArrayConstraint.shape; + } + _cachedInputIdsArray = [[MLMultiArray alloc] initWithShape:tokenShape + dataType:tokenDtype + error:&error]; if (error) { - _cachedInputArray = nil; + _cachedInputIdsArray = nil; return; } - - if (_modelDescription.inputDescriptionsByName[@"offset"] != nil) { - _cachedOffsetArray = [[MLMultiArray alloc] initWithShape:@[@1] - dataType:MLMultiArrayDataTypeFloat16 - error:&error]; + if (_positionIdsName) { + MLFeatureDescription* posDesc = _modelDescription.inputDescriptionsByName[_positionIdsName]; + NSArray* posShape = posDesc ? posDesc.multiArrayConstraint.shape : tokenShape; + MLMultiArrayDataType posDtype = posDesc ? posDesc.multiArrayConstraint.dataType : tokenDtype; + _cachedPositionIdsArray = [[MLMultiArray alloc] initWithShape:posShape + dataType:posDtype + error:&error]; if (error) { - _cachedOffsetArray = nil; + _cachedPositionIdsArray = nil; } } @@ -1084,24 +1128,37 @@ - (void)preallocateBuffers { } } -- (BOOL)predictDirectWithInput:(NSString*)inputName - data:(const __fp16*)data - offset:(int)offset { - if (!_model || !_cachedInputArray) return NO; - +- (BOOL)predictWithInputIds:(const int32_t*)inputIds + positionIds:(const int32_t*)positionIds + count:(NSUInteger)count { + if (!_model || !_cachedInputIdsArray) return NO; NSError* error = nil; - NSUInteger totalElements = (NSUInteger)(_chunkSize * _hiddenDim); - __fp16* inputPtr = (__fp16*)_cachedInputArray.dataPointer; - memcpy(inputPtr, data, totalElements * sizeof(__fp16)); - - MLFeatureValue* inputFeature = [MLFeatureValue featureValueWithMultiArray:_cachedInputArray]; - NSMutableDictionary* inputDict = [NSMutableDictionary dictionaryWithObject:inputFeature forKey:inputName]; + NSUInteger cap = (NSUInteger)_chunkSize; + NSUInteger n = MIN(count, cap); + auto writeInts = [&](MLMultiArray* arr, const int32_t* src) { + if (arr.dataType == MLMultiArrayDataTypeInt32) { + int32_t* dst = (int32_t*)arr.dataPointer; + memcpy(dst, src, n * sizeof(int32_t)); + if (n < cap) memset(dst + n, 0, (cap - n) * sizeof(int32_t)); + } else { + // fall back to per-element write for non-int32 + for (NSUInteger i = 0; i < cap; ++i) { + NSNumber* v = (i < n) ? @(src[i]) : @0; + arr[i] = v; + } + } + }; + writeInts(_cachedInputIdsArray, inputIds); + if (_cachedPositionIdsArray && positionIds) { + writeInts(_cachedPositionIdsArray, positionIds); + } - if (_cachedOffsetArray) { - ((__fp16*)_cachedOffsetArray.dataPointer)[0] = (__fp16)offset; - MLFeatureValue* offsetFeature = [MLFeatureValue featureValueWithMultiArray:_cachedOffsetArray]; - inputDict[@"offset"] = offsetFeature; + MLFeatureValue* idsFeature = [MLFeatureValue featureValueWithMultiArray:_cachedInputIdsArray]; + NSMutableDictionary* inputDict = [NSMutableDictionary dictionaryWithObject:idsFeature forKey:_inputIdsName]; + if (_cachedPositionIdsArray && _positionIdsName) { + MLFeatureValue* posFeature = [MLFeatureValue featureValueWithMultiArray:_cachedPositionIdsArray]; + inputDict[_positionIdsName] = posFeature; } id inputProvider = [[MLDictionaryFeatureProvider alloc] @@ -1118,15 +1175,12 @@ - (BOOL)predictDirectWithInput:(NSString*)inputName for (NSString* outputName in _cachedOutputArrays) { MLFeatureValue* outFeature = [outputProvider featureValueForName:outputName]; if (!outFeature || !outFeature.multiArrayValue) return NO; - MLMultiArray* dst = _cachedOutputArrays[outputName]; if (!dst) return NO; - size_t copied = copyMLArrayToFP16(outFeature.multiArrayValue, (__fp16*)dst.dataPointer); if (copied == 0) return NO; } } - return YES; } @@ -1210,10 +1264,9 @@ - (MLMultiArray*)getOutputArray:(NSString*)name { int ANEPrefill::get_num_kv_heads() const { return num_kv_heads_; } int ANEPrefill::get_head_dim() const { return head_dim_; } -NPUPrefillDirectResult ANEPrefill::prefill_chunk_direct( - const std::vector<__fp16>& embeddings, - int position_offset, - const std::string& input_name) { +NPUPrefillDirectResult ANEPrefill::prefill_chunk_tokens( + const std::vector& input_ids, + const std::vector& position_ids) { NPUPrefillDirectResult result; result.valid = false; @@ -1226,11 +1279,9 @@ - (MLMultiArray*)getOutputArray:(NSString*)name { @autoreleasepool { CactusANEPrefillImpl* impl = (__bridge CactusANEPrefillImpl*)impl_; - NSString* inName = [NSString stringWithUTF8String:input_name.c_str()]; - - BOOL success = [impl predictDirectWithInput:inName - data:embeddings.data() - offset:position_offset]; + BOOL success = [impl predictWithInputIds:input_ids.data() + positionIds:position_ids.empty() ? nullptr : position_ids.data() + count:input_ids.size()]; if (!success) return result; diff --git a/local_integration_tests/native_runner.cpp b/local_integration_tests/native_runner.cpp index 8cba0ce08..11cf3ef65 100644 --- a/local_integration_tests/native_runner.cpp +++ b/local_integration_tests/native_runner.cpp @@ -1088,6 +1088,10 @@ int run_transcribe(const Args& args, bool pcm) { int main(int argc, char** argv) { try { + const char* level_env = std::getenv("CACTUS_LOG_LEVEL"); + if (level_env && level_env[0]) { + cactus_log_set_level(std::atoi(level_env)); + } Args args = parse_args(argc, argv); if (args.command == "e002-tokens") return run_e002_tokens(args); if (args.command == "e002-preprocess") return run_e002_preprocess(args); diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index a1b6e8609..cf3919e28 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -296,6 +296,7 @@ def _write_component_bundle( component_io_signatures: dict[str, dict[str, tuple[str, ...]]] | None = None, graph_filename: str = "graph.cactus", npu_prefill_mlpackage: str | None = None, + npu_encoder_mlpackages: dict[str, str] | None = None, ) -> Path: bundle_dir = artifact_dir / "components" component_order = [ @@ -410,6 +411,9 @@ def _write_component_bundle( } if npu_prefill_mlpackage: manifest_payload["npu_prefill"] = npu_prefill_mlpackage + for manifest_key, mlpackage_path in (npu_encoder_mlpackages or {}).items(): + if mlpackage_path: + manifest_payload[manifest_key] = mlpackage_path _write_json(manifest_path, manifest_payload) return manifest_path @@ -667,13 +671,21 @@ def _run_component_pipeline_transpile( f"saved_optimized_component_ir_{component}=" f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" ) - from .npu import run_prefill_pipeline + from .npu import run_encoder_pipeline, run_prefill_pipeline + npu_enabled = bool(getattr(args, "npu", False)) + npu_quantize = getattr(args, "npu_quantize", None) npu_prefill_filename = run_prefill_pipeline( model, artifact_dir, chunk_size=int(getattr(args, "npu_chunk_size", 256)), - enabled=bool(getattr(args, "npu", False)), - quantize_bits=getattr(args, "npu_quantize", None), + enabled=npu_enabled, + quantize_bits=npu_quantize, + ) + npu_encoder_mlpackages = run_encoder_pipeline( + component_specs, + artifact_dir, + enabled=npu_enabled, + quantize_bits=npu_quantize, ) component_manifest_path = _write_component_bundle( @@ -689,6 +701,7 @@ def _run_component_pipeline_transpile( component_io_signatures=component_io_signatures, graph_filename=args.graph_filename, npu_prefill_mlpackage=npu_prefill_filename, + npu_encoder_mlpackages=npu_encoder_mlpackages, ) print(f"saved_component_bundle_manifest={component_manifest_path}") for component in transpiled_component_graphs: diff --git a/python/cactus/transpile/npu/README.md b/python/cactus/transpile/npu/README.md new file mode 100644 index 000000000..3885afd9a --- /dev/null +++ b/python/cactus/transpile/npu/README.md @@ -0,0 +1,110 @@ +# NPU Transpiler Pipeline + +Parallel to the graph transpiler: emits CoreML `.mlpackage`s so the runtime +engine can dispatch through the Apple Neural Engine. Triggered by +`cactus convert --npu`. Runtime side: `cactus-engine/src/model_npu.cpp`. + +Three emit paths, matching what `origin/main` historically NPU-accelerated: + +| Entry point | Emits | manifest key | Runtime dispatch | +|---|---|---|---| +| `run_prefill_pipeline` (`prefill.py`) | text-decoder prefill | `npu_prefill` | `prefill_via_npu` | +| `run_encoder_pipeline` → `audio.py` | audio encoder | `npu_audio_encoder` | `audio_encode_via_npu` | +| `run_encoder_pipeline` → `vision.py` | vision encoder | `npu_vision_encoder` | `vision_encode_via_npu` | + +`run_encoder_pipeline` reuses the exact `ComponentModuleSpec` adapter +modules + example inputs the graph transpiler captures, so the NPU encoder +stays in lockstep with the CPU component graph. Auxiliary adapter inputs +(masks, position ids) are baked into the exported model as constants — +the ANE requires a static single-input signature. + +## Status + +| Target | Works | Notes | +|---|---|---| +| Parakeet TDT audio encoder | **yes — verified end-to-end** | `cactus transcribe` via NPU, ~7.5x faster warm than CPU with `CACTUS_ANE_AUDIO_COMPUTE_UNITS=ALL`. | +| `run_encoder_pipeline` emit mechanics | **yes — verified** | Synthetic 2-input (audio) and 3-input (vision) encoders export + convert + load + predict. | +| Gemma 4 audio + vision encoder emit | wired, **untested** | Can't convert Gemma 4 locally (OOM). Coworker must run `cactus convert --npu`; see handoff note below. | +| `coremltools.optimize.linear_quantize_weights` | works | `--npu-quantize 4` / `--npu-quantize 8` post-conversion. int8 recommended for encoders. | + +## Gemma 4 coworker handoff + +Gemma 4 conversion OOMs on a 16 GB host, so the actual emit must run on a +bigger machine. To produce + test the Gemma 4 NPU bundle: + +1. `cactus convert --npu --npu-quantize 8 google/gemma-4-E2B-it` — emits + `components/model.mlpackage` (prefill), `components/audio_encoder.mlpackage`, + `components/vision_encoder.mlpackage`, and writes `npu_prefill` / + `npu_audio_encoder` / `npu_vision_encoder` keys into `components/manifest.json`. +2. The runtime auto-loads any present mlpackage at init and dispatches + automatically (`run_audio_encoder` / `run_vision_encoder` / prefill). +3. If `coremltools.convert` fails on an unsupported op, add a patch to + `coremltools_patches.py` (see the layer_norm / `new_ones` / `__and__` + patches already there for the pattern) and re-run. +4. Test: `cactus transcribe` for audio, a multimodal prompt for vision. + Try `CACTUS_ANE_AUDIO_COMPUTE_UNITS=ALL` — ANE beat CPU+GPU for Parakeet. + +## Known coremltools 9.0 gaps when converting HF Gemma 4 via `torch.export` + +Each requires a custom `@register_torch_op` in this module: + +1. `new_ones` — `tensor.new_ones(shape)`. Fix: `mb.fill(shape=size, value=1.0)`. +2. `__and__` — bitwise AND on bool tensors. Fix: `mb.logical_and(...)`. +3. Probably more once those two are fixed. + +`torch.jit.trace` is not viable on current HF transformers — fails inside +`masking_utils.sdpa_mask` because `q_length` is an int but the code +indexes it as a tensor. `torch.export` is the correct path. + +## Where the NPU perf gains actually come from + +Surveyed `origin/main` for inspiration. Main has **CPU-side** kernel +fusion (e.g., `dense_mlp_int4_fused` from PR #617 in `cactus/models/ +gemma4/model_gemma4.cpp` — gate+up+down INT4 matmul in one kernel call). +There is **no NPU-side fusion** code in main: the `.mlpackage` is +expected pre-built externally, and `cactus/npu/npu_ane.mm` is a thin +CoreML MLModel wrapper. So fused-op wins for NPU need to be added on +the convert path here. + +The four NPU-fusion levers, ranked by expected impact: + +1. **Weight quantization (int4 / int8)** — biggest single perf knob on + ANE. Smaller weights ⇒ less memory bandwidth ⇒ faster matmuls. + coremltools' `linear_quantize_weights` does post-conversion + per-channel quantization. **Wired**: pass `--npu-quantize 4` (or 8) + on `cactus convert`. Recommend int4 for big LMs. + +2. **SDPA preservation** — `torch.nn.functional.scaled_dot_product_attention` + maps to a single MIL op when coremltools sees it. If the model + decomposes attention to Q·Kᵀ + softmax + ·V manually, coremltools + has to recompose it, often losing the ANE fast path. HF defaults to + `attn_implementation="sdpa"` for compatible models — we benefit + automatically, but worth checking with `model.config._attn_implementation` + after load. + +3. **Stateful KV cache** — coremltools 9 supports `mb.state` for models + that hold K/V on ANE between calls. Avoids transferring the entire + cache through host memory per chunk. Requires: + - CoreML model authored with `mb.state` ops (the prefill wrapper + would need rewriting in MIL directly, not via PyTorch trace) + - `npu_ane.mm` would need to use the iOS 18 `MLState` API instead of + plain `MLPredictionOptions` + Significant work. **Not done.** + +4. **Pre-export module replacement** — swap HF's `LlamaAttention`, + `LlamaMLP`, etc. for NPU-friendly versions before `torch.export`. + E.g., fuse Q/K/V projections into a single matmul with stacked + weights (saves 2 matmuls per layer). Per-architecture work. **Not done.** + +## Architectural notes + +- Wrapper takes `input_ids` (not `inputs_embeds`) because Gemma 4 + computes auxiliary per-layer embeddings from the original token IDs. + Also more efficient (no host-side embedding lookup). +- Output naming matches `cactus-engine/src/npu_ane.mm` (`hidden`, + `k_cache_0`, `v_cache_0`, ...) — change in lockstep with that file. +- Text decoder located via `_find_text_decoder`: tries + `model.model.language_model` (multimodal HF), then `model.model`, + then `model`. +- Dims via `_extract_dims` — tries `config.text_config` first + (multimodal) then `config`. diff --git a/python/cactus/transpile/npu/__init__.py b/python/cactus/transpile/npu/__init__.py new file mode 100644 index 000000000..0f1a87884 --- /dev/null +++ b/python/cactus/transpile/npu/__init__.py @@ -0,0 +1,12 @@ +"""NPU transpiler pipeline. + +Parallel to the graph transpiler: takes a captured HF causal LM and emits a +CoreML `.mlpackage` for Apple Neural Engine prefill, alongside the +`.cactus` graphs the graph transpiler produces. Entry point is +`run_prefill_pipeline`, invoked from `hf_model._run_component_pipeline_transpile` +and the legacy single-graph path. +""" + +from .pipeline import run_encoder_pipeline, run_prefill_pipeline + +__all__ = ["run_prefill_pipeline", "run_encoder_pipeline"] diff --git a/python/cactus/transpile/npu/audio.py b/python/cactus/transpile/npu/audio.py new file mode 100644 index 000000000..c62a1149e --- /dev/null +++ b/python/cactus/transpile/npu/audio.py @@ -0,0 +1,149 @@ +"""CoreML audio encoder emit. + +Wraps an audio encoder PyTorch module (Conformer / Whisper-style) into an +ANE-targeted `.mlpackage`. The runtime side (`cactus-engine/src/npu_ane.mm`) +loads this with a single named input "x" and a single named output, matching +the `NPUEncoder::encode` interface. + +Targets the audio encoder NPU path — historically the highest-impact use +of the ANE for cactus (CPU was faster for text prefill in most cases). +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import torch + + +class AudioEncoderWrapper(torch.nn.Module): + """Single-input single-output wrapper around an audio encoder module. + + Matches the runtime interface in `ANEEncoder::encode(input, output, ...)`: + one fp16 input tensor, one fp16 output tensor. + + Audio encoder adapters that take auxiliary inputs (e.g. Gemma 4's + ``input_features_mask``) supply them via ``baked_inputs`` — these are + registered as constant buffers so the exported model keeps a static + single-input signature, as the ANE requires. + """ + + def __init__( + self, + audio_module: torch.nn.Module, + baked_inputs: tuple[torch.Tensor, ...] = (), + ): + super().__init__() + self.audio = audio_module + self._n_baked = len(baked_inputs) + for idx, tensor in enumerate(baked_inputs): + self.register_buffer(f"_baked_{idx}", tensor, persistent=False) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + extra = tuple(getattr(self, f"_baked_{i}") for i in range(self._n_baked)) + return self.audio(input_features, *extra) + + +def _import_coremltools() -> Any: + try: + import coremltools as ct + from .coremltools_patches import apply_all_coremltools_patches + apply_all_coremltools_patches() + return ct + except Exception: + return None + + +def _apply_weight_quantization(mlmodel: Any, bits: int) -> Any: + try: + from coremltools.optimize.coreml import ( + linear_quantize_weights, + OpLinearQuantizerConfig, + OptimizationConfig, + ) + op_config = OpLinearQuantizerConfig( + mode="linear_symmetric", + dtype=f"int{bits}", + granularity="per_channel", + ) + config = OptimizationConfig(global_config=op_config) + return linear_quantize_weights(mlmodel, config) + except Exception as exc: + print(f"npu.audio: weight quantization to int{bits} failed ({type(exc).__name__}: {exc}); keeping FP16 weights") + return mlmodel + + +def emit_audio_encoder_mlpackage( + audio_module: torch.nn.Module, + bundle_dir: Path, + *, + example_input: torch.Tensor, + baked_inputs: tuple[torch.Tensor, ...] = (), + filename: str = "audio_encoder.mlpackage", + input_name: str = "x", + output_name: str = "encoded", + minimum_deployment_target: str = "iOS18", + quantize_bits: int | None = None, +) -> str | None: + """Trace + convert + save. Returns the filename or None on failure. + + `example_input` should be the shape the runtime will use at inference + time (e.g., for Parakeet: ``[1, num_frames, mel_bins]``). The ANE + requires a fully-specified shape; dynamic dims are not supported here. + `baked_inputs` carries any auxiliary encoder inputs (masks etc.) that + get frozen into the model as constants. + """ + ct = _import_coremltools() + if ct is None: + print("npu.audio: coremltools not installed; skipping mlpackage emit") + return None + + wrapper = AudioEncoderWrapper(audio_module, baked_inputs) + wrapper.eval() + + try: + with torch.no_grad(): + exported = torch.export.export(wrapper, (example_input,)) + exported = exported.run_decompositions({}) + except Exception as exc: + print(f"npu.audio: torch.export failed ({type(exc).__name__}: {exc}); skipping mlpackage emit") + return None + + # Free the live module — ExportedProgram now owns what it needs. + del wrapper + import gc as _gc + _gc.collect() + + target_attr = getattr(ct.target, minimum_deployment_target, None) or ct.target.iOS17 + + from .coremltools_patches import build_cactus_pass_pipeline + try: + mlmodel = ct.convert( + exported, + inputs=[ct.TensorType(name=input_name, shape=tuple(example_input.shape))], + outputs=[ct.TensorType(name=output_name)], + compute_precision=ct.precision.FLOAT16, + convert_to="mlprogram", + minimum_deployment_target=target_attr, + pass_pipeline=build_cactus_pass_pipeline(), + ) + except Exception as exc: + print(f"npu.audio: coremltools.convert failed ({type(exc).__name__}: {exc})") + return None + + if quantize_bits is not None: + before_id = id(mlmodel) + mlmodel = _apply_weight_quantization(mlmodel, quantize_bits) + if id(mlmodel) != before_id: + print(f"npu.audio: applied int{quantize_bits} weight quantization") + + bundle_dir.mkdir(parents=True, exist_ok=True) + out_path = bundle_dir / filename + try: + mlmodel.save(str(out_path)) + except Exception as exc: + print(f"npu.audio: mlpackage save failed ({type(exc).__name__}: {exc})") + return None + + print(f"npu.audio: wrote {out_path} (input_shape={tuple(example_input.shape)})") + return filename diff --git a/python/cactus/transpile/npu/coremltools_patches.py b/python/cactus/transpile/npu/coremltools_patches.py new file mode 100644 index 000000000..f920c466d --- /dev/null +++ b/python/cactus/transpile/npu/coremltools_patches.py @@ -0,0 +1,373 @@ +"""Runtime patches for coremltools 9.0 to support Gemma 4 + torch.export. + +Each patch is idempotent and guarded so reimporting the module is safe. +Call `apply_all_coremltools_patches()` once before invoking `ct.convert`. + +Patches: +- register_func: allow dunder-suffixed op names (e.g. ``__and__``) past the + inplace-op rejection check. +- new_ones: register a translation that maps to ``mb.fill(shape=..., value=1.0)``. +- SDPA scalar-cast: ``mb.sub/mul/add`` auto-cast python scalars to the dtype + of the tensor input (avoids fp16/fp32 mismatch in the SDPA mask path). +- Pipeline: remove ``common::fuse_prelu`` (KeyError on float64 in 9.0). +""" +from __future__ import annotations + +_APPLIED = False + + +def apply_all_coremltools_patches() -> None: + global _APPLIED + if _APPLIED: + return + try: + _patch_register_func_allow_dunder() + _register_new_ones_op() + _register_logical_and_op() + _override_layer_norm_translator() + _patch_pipeline_remove_fuse_prelu() + _patch_mb_binops_scalar_cast() + _patch_fp16_cast_skip_layer_norm() + _APPLIED = True + print("npu.coremltools_patches: applied") + except Exception as exc: + print(f"npu.coremltools_patches: failed to apply ({type(exc).__name__}: {exc})") + + +def _patch_register_func_allow_dunder() -> None: + from coremltools.converters.mil.frontend.torch import torch_op_registry + + cls = torch_op_registry.TorchOpsRegistry + if getattr(cls.register_func, "_cactus_dunder_patched", False): + return + + original = cls.register_func + + def register_func(self, func=None, torch_alias=None, override=False): + f_name = func.__name__ + all_f_names = [f_name] + if torch_alias is not None: + all_f_names.extend(torch_alias) + for name in all_f_names: + is_dunder = name.startswith("__") and name.endswith("__") + if name.endswith("_") and not is_dunder: + raise Exception( + f'Attempting to register "{name}" op. Do not register inplace ops.' + ) + if not override and name in self.name_to_func_mapping: + raise ValueError(f"Torch op {name} already registered.") + self.set_func_by_name(func, name) + + register_func._cactus_dunder_patched = True + cls.register_func = register_func + + +def _register_new_ones_op() -> None: + """Translate Tensor.new_ones(size, ...) -> mb.fill(shape=size, value=1). + + Mirrors coremltools' built-in `ones` translator (shift by 1 to skip the + `self` tensor argument). Direct registration via set_func_by_name avoids + the decorator's redundant name lookups. + """ + from coremltools.converters.mil.frontend.torch.torch_op_registry import ( + _TORCH_OPS_REGISTRY, + ) + from coremltools.converters.mil import Builder as mb + from coremltools.converters.mil.frontend.torch.ops import _get_inputs + + if "new_ones" in _TORCH_OPS_REGISTRY.name_to_func_mapping: + return + + def _to_int32(v): + if isinstance(v, list): + casted = [_to_int32(x) for x in v] + return mb.concat(values=casted, axis=0) + dtype = getattr(v, "dtype", None) + name = getattr(dtype, "__name__", str(dtype) if dtype else "") + if "int32" in name: + return v + return mb.cast(x=v, dtype="int32") + + def new_ones(context, node): + inputs = _get_inputs(context, node, min_expected=2) + size = _to_int32(inputs[1]) + res = mb.fill(shape=size, value=1.0, name=node.name) + context.add(res, node.name) + + _TORCH_OPS_REGISTRY.set_func_by_name(new_ones, "new_ones") + + +def _register_logical_and_op() -> None: + """Register __and__ / __or__ translations. Gemma 4's attention mask path emits + `__and__.Tensor` nodes for combining causal + padding masks; coremltools 9.0 + doesn't ship a translator for those names by default.""" + from coremltools.converters.mil.frontend.torch.torch_op_registry import ( + _TORCH_OPS_REGISTRY, + register_torch_op, + ) + from coremltools.converters.mil import Builder as mb + + def _force_bool(v): + dtype = getattr(v, "dtype", None) + name = getattr(dtype, "__name__", str(dtype) if dtype else "") + if "bool" in name: + return v + # cast through fp32 -> bool to handle exotic source dtypes (e.g. fp64) + if "fp16" in name or "fp32" in name or "fp64" in name or "double" in name or "float" in name: + non_zero = mb.not_equal(x=v, y=0.0) + return non_zero + if "int" in name: + non_zero = mb.not_equal(x=v, y=0) + return non_zero + return mb.cast(x=v, dtype="bool") + + def make_logical(op_kind): + def _impl(context, node): + inputs = [context[i] for i in node.inputs] + x = _force_bool(inputs[0]) + y = _force_bool(inputs[1]) + if op_kind == "and": + out = mb.logical_and(x=x, y=y, name=node.name) + elif op_kind == "or": + out = mb.logical_or(x=x, y=y, name=node.name) + else: + out = mb.logical_xor(x=x, y=y, name=node.name) + context.add(out, node.name) + return _impl + + # Register all aliases that sanitize_op_kind + unify_inplace_and_functional + # may resolve fx node `__and__.Tensor` / `__or__.Tensor` to. The lookup + # chain for `__and__.Tensor` ends at `__and_` (one trailing underscore), + # for `aten::__and__` ends at `__and_`, and the raw key `__and__` ends at + # `and`. Cover all three so the bool-casting handler always wins. + for tag, op_kind in [("__and_", "and"), ("__or_", "or"), ("__xor_", "xor")]: + if tag not in _TORCH_OPS_REGISTRY.name_to_func_mapping: + _TORCH_OPS_REGISTRY.set_func_by_name(make_logical(op_kind), tag) + for tag, op_kind in [("__and__", "and"), ("__or__", "or"), ("__xor__", "xor")]: + if tag not in _TORCH_OPS_REGISTRY.name_to_func_mapping: + _TORCH_OPS_REGISTRY.set_func_by_name(make_logical(op_kind), tag) + # The default `bitwise_and` translator chokes on mixed bool+float inputs + # that Gemma 4's masking emits; override the lookup keys with the + # bool-casting variant. `and` and `or` are the sanitize_op_kind results + # for many aten op kinds, so override those too. + for tag in ("bitwise_and", "and"): + _TORCH_OPS_REGISTRY.set_func_by_name(make_logical("and"), tag) + for tag in ("bitwise_or", "or"): + _TORCH_OPS_REGISTRY.set_func_by_name(make_logical("or"), tag) + + +def _override_layer_norm_translator() -> None: + """Override the torch frontend's ``layer_norm`` translator so gamma/beta/ + epsilon share one dtype before reaching ``mb.layer_norm``. + + coremltools 9.0's iOS17/iOS18 layer_norm op enforces this at validation + time. FP16-traced models hit it because the python-scalar ``eps`` arg + survives as fp32 while gamma/beta come through as fp16. + """ + import numpy as np + from coremltools.converters.mil.frontend.torch.torch_op_registry import ( + _TORCH_OPS_REGISTRY, + ) + from coremltools.converters.mil.frontend.torch.ops import _get_inputs + from coremltools.converters.mil import Builder as mb + + def _dtype_name(v): + d = getattr(v, "dtype", None) + return getattr(d, "__name__", str(d) if d else "") + + def _to_typed_const(v, target): + """Force v to a const Var of dtype `target`. Eps stays a const so + later MIL passes (add_fp16_cast) don't promote it back to fp32.""" + if v is None: + return v + np_dtype = np.float16 if target == "fp16" else np.float32 + if isinstance(v, (int, float)) and not isinstance(v, bool): + return mb.const(val=np_dtype(v)) + if hasattr(v, "val") and v.val is not None and hasattr(v, "dtype"): + try: + return mb.const(val=np.asarray(v.val, dtype=np_dtype)) + except Exception: + pass + if hasattr(v, "dtype") and _dtype_name(v) != target: + return mb.cast(x=v, dtype=target) + return v + + def layer_norm(context, node): + inputs = _get_inputs(context, node, min_expected=2) + nargs = len(inputs) + x = inputs[0] + normalized_shape = inputs[1] + weight = inputs[2] if nargs > 2 else None + bias = inputs[3] if nargs > 3 else None + eps = inputs[4] if nargs > 4 else None + if eps is None: + eps = 1e-5 + + ref = weight if weight is not None else x + ref_name = _dtype_name(ref) + target = "fp16" if ("fp16" in ref_name or "float16" in ref_name) else ( + "fp32" if ("fp32" in ref_name or "float32" in ref_name) else "fp16" + ) + + # x is a runtime tensor — keep as cast Var; gamma/beta/eps are const-like. + if x is not None and hasattr(x, "dtype") and _dtype_name(x) != target: + x = mb.cast(x=x, dtype=target) + weight = _to_typed_const(weight, target) + bias = _to_typed_const(bias, target) + eps = _to_typed_const(eps, target) + + out = mb.layer_norm( + x=x, + axes=list(range(-len(normalized_shape.val), 0)), + gamma=weight, + beta=bias, + epsilon=eps, + name=node.name, + ) + + if node.kind == "native_layer_norm": + context.add((out, None, None), torch_name=node.name) + else: + context.add(out) + + _TORCH_OPS_REGISTRY.set_func_by_name(layer_norm, "layer_norm") + _TORCH_OPS_REGISTRY.set_func_by_name(layer_norm, "native_layer_norm") + + +def _patch_pipeline_remove_fuse_prelu() -> None: + """No-op placeholder. ``PassPipeline.DEFAULT`` is a classproperty that + returns a fresh instance each time, so mutating it in-place doesn't + persist. We instead build a trimmed pipeline at convert time via + :func:`build_cactus_pass_pipeline` and pass it to ``ct.convert``. + """ + return + + +_PASSES_TO_DROP = [ + # fuse_prelu crashes on fp64 const tables in coremltools 9.0. + "common::fuse_prelu", +] + + +def build_cactus_pass_pipeline(): + """Construct a fresh PassPipeline with our problem passes removed. + + Pass this to ``ct.convert(..., pass_pipeline=pipeline)`` — mutating + ``PassPipeline.DEFAULT`` doesn't work because that's a classproperty + returning a new instance every time. + """ + from coremltools.converters.mil.mil.passes.pass_pipeline import PassPipeline + pipeline = PassPipeline.DEFAULT + for pass_name in _PASSES_TO_DROP: + try: + pipeline.remove_passes([pass_name]) + except Exception: + pass + return pipeline + + +def _patch_mb_binops_scalar_cast() -> None: + """Auto-align dtypes for mb.{sub,mul,add,div} so SDPA mask paths don't + crash on mixed fp16/fp32 inputs. Casts both scalars and tensors to + match whichever operand is already in the lower-precision dtype.""" + from coremltools.converters.mil import Builder as mb + + def _dtype_name(v): + d = getattr(v, "dtype", None) + return getattr(d, "__name__", str(d) if d else "") + + def _is_fp(name): + return "fp" in name or "float" in name or "double" in name + + for op_name in ("sub", "mul", "add", "div", "real_div"): + original = getattr(mb, op_name, None) + if original is None or getattr(original, "_cactus_scalar_cast_patched", False): + continue + + def make_wrapper(orig): + def wrapper(**kwargs): + x = kwargs.get("x") + y = kwargs.get("y") + xn = _dtype_name(x) + yn = _dtype_name(y) + # If only one is a tensor, cast scalars to match it. + if xn and not yn and _is_fp(xn) and isinstance(y, (int, float)) and not isinstance(y, bool): + kwargs["y"] = mb.cast(x=y, dtype=xn) + elif yn and not xn and _is_fp(yn) and isinstance(x, (int, float)) and not isinstance(x, bool): + kwargs["x"] = mb.cast(x=x, dtype=yn) + elif xn and yn and xn != yn: + # Both tensors, mismatched. Prefer fp16 to keep memory low. + target = "fp16" if (xn == "fp16" or yn == "fp16") else xn + if xn != target: + kwargs["x"] = mb.cast(x=x, dtype=target) + if yn != target: + kwargs["y"] = mb.cast(x=y, dtype=target) + return orig(**kwargs) + wrapper._cactus_scalar_cast_patched = True + wrapper.__name__ = orig.__name__ + return wrapper + + setattr(mb, op_name, make_wrapper(original)) + + # layer_norm: epsilon (scalar) must match gamma/beta dtype. Parakeet's + # Conformer norms produce fp16 gamma but coremltools emits fp32 epsilon. + orig_ln = getattr(mb, "layer_norm", None) + if orig_ln is not None and not getattr(orig_ln, "_cactus_scalar_cast_patched", False): + def _ln_wrap(orig): + def wrapper(**kwargs): + gamma = kwargs.get("gamma") + gn = _dtype_name(gamma) + if _is_fp(gn): + eps = kwargs.get("epsilon") + if isinstance(eps, (int, float)) and not isinstance(eps, bool): + kwargs["epsilon"] = mb.cast(x=eps, dtype=gn) + elif hasattr(eps, "dtype") and _dtype_name(eps) != gn: + kwargs["epsilon"] = mb.cast(x=eps, dtype=gn) + return orig(**kwargs) + wrapper._cactus_scalar_cast_patched = True + wrapper.__name__ = orig.__name__ + return wrapper + setattr(mb, "layer_norm", _ln_wrap(orig_ln)) + + # batch_norm / instance_norm have similar epsilon shapes + for op_name in ("batch_norm", "instance_norm", "rms_norm"): + orig_op = getattr(mb, op_name, None) + if orig_op is None or getattr(orig_op, "_cactus_scalar_cast_patched", False): + continue + def _bn_wrap(orig): + def wrapper(**kwargs): + # Reference dtype: gamma if present, else x + ref = kwargs.get("gamma") or kwargs.get("x") + rn = _dtype_name(ref) + if _is_fp(rn): + eps = kwargs.get("epsilon") + if isinstance(eps, (int, float)) and not isinstance(eps, bool): + kwargs["epsilon"] = mb.cast(x=eps, dtype=rn) + elif hasattr(eps, "dtype") and _dtype_name(eps) != rn: + kwargs["epsilon"] = mb.cast(x=eps, dtype=rn) + return orig(**kwargs) + wrapper._cactus_scalar_cast_patched = True + wrapper.__name__ = orig.__name__ + return wrapper + setattr(mb, op_name, _bn_wrap(orig_op)) + + +def _patch_fp16_cast_skip_layer_norm() -> None: + """Tell the ``common::add_fp16_cast`` MIL pass to skip ``layer_norm`` and + ``batch_norm``-family ops. The pass casts gamma to fp16 but leaves + epsilon as fp32, which violates the iOS17+ ``layer_norm`` constraint that + gamma, beta, and epsilon share one dtype. + + By marking them unsupported, we keep them at their original (fp32) dtype. + """ + try: + from coremltools.converters.mil.mil.passes.defs.quantization import ( + FP16ComputePrecision, + ) + except Exception as exc: + print(f"npu.coremltools_patches: skip_layer_norm patch unavailable ({exc})") + return + base = set(FP16ComputePrecision._UNSUPPORTED_FP16_OPS) + for op in ("layer_norm", "batch_norm", "instance_norm", "rms_norm"): + base.add(op) + FP16ComputePrecision._UNSUPPORTED_FP16_OPS = base diff --git a/python/cactus/transpile/npu/pipeline.py b/python/cactus/transpile/npu/pipeline.py new file mode 100644 index 000000000..0d39a3882 --- /dev/null +++ b/python/cactus/transpile/npu/pipeline.py @@ -0,0 +1,150 @@ +"""NPU transpiler orchestrator. + +Mirrors the role of `component_pipeline.execute_component_pipeline` but +for CoreML. Invoked from `hf_model._run_component_pipeline_transpile` and +the legacy single-graph path when `--npu` is passed on the CLI. + +Two entry points: +- `run_prefill_pipeline`: emits the text-decoder prefill `.mlpackage`. +- `run_encoder_pipeline`: emits audio / vision encoder `.mlpackage`s by + reusing the same component adapter modules the graph transpiler captures. + +Historically (`origin/main`) NPU covered text prefill (Gemma/Qwen), +audio encoders (Whisper/Moonshine/Parakeet/Gemma4) and vision encoders +(SigLIP2/Gemma4). This pipeline regenerates those `.mlpackage`s through +the v2 transpiler instead of shipping them pre-built. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import torch + +from .audio import emit_audio_encoder_mlpackage +from .prefill import emit_prefill_mlpackage +from .vision import emit_vision_encoder_mlpackage + + +DEFAULT_CHUNK_SIZE = 256 + + +def _extract_dims(hf_model: torch.nn.Module) -> tuple[int, int]: + """Walk through possible config locations to find text decoder dims. + + Multimodal HF models (Gemma 4, LFM-VL, etc.) nest text dims under + `config.text_config`. Plain causal LMs expose them at the top level. + """ + candidates = [] + cfg = getattr(hf_model, "config", None) + if cfg is not None: + candidates.append(cfg) + text_cfg = getattr(cfg, "text_config", None) + if text_cfg is not None: + candidates.insert(0, text_cfg) + inner_cfg = getattr(getattr(hf_model, "model", None), "config", None) + if inner_cfg is not None and inner_cfg not in candidates: + candidates.append(inner_cfg) + + for c in candidates: + hidden_dim = int(getattr(c, "hidden_size", 0) or 0) + num_layers = int(getattr(c, "num_hidden_layers", 0) or 0) + if hidden_dim > 0 and num_layers > 0: + return hidden_dim, num_layers + return 0, 0 + + +def run_prefill_pipeline( + hf_model: torch.nn.Module, + artifact_dir: Path, + *, + chunk_size: int = DEFAULT_CHUNK_SIZE, + enabled: bool = True, + quantize_bits: int | None = None, +) -> str | None: + """Run the NPU prefill transpilation. Returns the manifest-relative + filename to embed in `manifest.json["npu_prefill"]`, or None. + """ + if not enabled: + return None + + hidden_dim, num_layers = _extract_dims(hf_model) + if hidden_dim <= 0 or num_layers <= 0: + print("npu.pipeline: model config missing hidden_size/num_hidden_layers; skipping") + return None + + bundle_root = artifact_dir / "components" + bundle_root.mkdir(parents=True, exist_ok=True) + + qbits: int | None = quantize_bits + if qbits == 0: + qbits = None + + filename = emit_prefill_mlpackage( + hf_model, + bundle_root, + hidden_dim=hidden_dim, + num_layers=num_layers, + chunk_size=chunk_size, + quantize_bits=qbits, + ) + return f"components/{filename}" if filename else None + + +# Component name -> (emit function, output mlpackage filename). +_ENCODER_COMPONENTS = { + "audio_encoder": (emit_audio_encoder_mlpackage, "audio_encoder.mlpackage"), + "vision_encoder": (emit_vision_encoder_mlpackage, "vision_encoder.mlpackage"), +} + + +def run_encoder_pipeline( + component_specs, + artifact_dir: Path, + *, + enabled: bool = True, + quantize_bits: int | None = None, +) -> dict[str, str]: + """Emit audio / vision encoder `.mlpackage`s from captured component specs. + + Reuses the exact adapter modules + example inputs the graph transpiler + captured (``ComponentModuleSpec``), so the NPU encoder stays in lockstep + with the CPU component graph. Auxiliary adapter inputs (masks, position + ids) are baked into the exported model as constants. + + Returns a dict mapping ``manifest.json`` key (``npu_audio_encoder`` / + ``npu_vision_encoder``) to the manifest-relative `.mlpackage` path. + """ + results: dict[str, str] = {} + if not enabled or not component_specs: + return results + + bundle_root = artifact_dir / "components" + bundle_root.mkdir(parents=True, exist_ok=True) + + qbits: int | None = quantize_bits + if qbits == 0: + qbits = None + + for spec in component_specs: + component = getattr(spec, "component", None) + if component not in _ENCODER_COMPONENTS: + continue + emit_fn, filename = _ENCODER_COMPONENTS[component] + example_inputs = tuple(getattr(spec, "example_inputs", ()) or ()) + if not example_inputs: + print(f"npu.pipeline: {component} spec has no example inputs; skipping") + continue + primary = example_inputs[0] + baked = example_inputs[1:] + emitted = emit_fn( + spec.module, + bundle_root, + example_input=primary, + baked_inputs=baked, + filename=filename, + quantize_bits=qbits, + ) + if emitted: + results[f"npu_{component}"] = f"components/{emitted}" + return results diff --git a/python/cactus/transpile/npu/prefill.py b/python/cactus/transpile/npu/prefill.py new file mode 100644 index 000000000..94f7aa643 --- /dev/null +++ b/python/cactus/transpile/npu/prefill.py @@ -0,0 +1,213 @@ +"""CoreML prefill emit. + +Wraps an HF causal LM so its forward pass exposes per-layer K/V outputs, +then runs the wrapper through `coremltools.convert` to produce a +`.mlpackage`. Output naming matches what `cactus-engine/src/npu_ane.mm` +reads at load time: `hidden`, `k_0`, `v_0`, ... (the runtime infers +`num_layers` by scanning for outputs prefixed with `k_`). +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import torch + + +def _find_text_decoder(hf_model: torch.nn.Module) -> torch.nn.Module: + """Locate the inner text decoder. + + Handles three layouts: + - Plain `AutoModelForCausalLM`: returns `model.model` (e.g., LlamaModel). + - Multimodal HF model: `model.model.language_model` (e.g., Gemma4Model.language_model). + - Already-text model: returns the model itself. + """ + inner = getattr(hf_model, "model", None) or hf_model + nested = getattr(inner, "language_model", None) + if nested is not None and isinstance(nested, torch.nn.Module): + return nested + return inner + + +class PrefillWrapper(torch.nn.Module): + """Wraps an HF causal LM into a (input_ids, position_ids) -> (hidden, k_0, v_0, ...) module. + + Input is token IDs (not embeddings) because some architectures (Gemma 4) + require the original input_ids to compute auxiliary per-layer embeddings + that get mixed into each transformer layer. Passing input_ids lets the + model handle embedding internally and is also more efficient. + """ + + def __init__(self, hf_model: torch.nn.Module): + super().__init__() + self.inner = _find_text_decoder(hf_model) + + def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, ...]: + outputs = self.inner( + input_ids=input_ids, + position_ids=position_ids, + use_cache=True, + output_hidden_states=False, + output_attentions=False, + return_dict=True, + ) + hidden = outputs.last_hidden_state + flat: list[torch.Tensor] = [hidden] + for layer_kv in outputs.past_key_values: + flat.append(layer_kv[0]) + flat.append(layer_kv[1]) + return tuple(flat) + + +def _estimate_save_bytes(hidden_dim: int, num_layers: int, chunk_size: int, quantize_bits: int | None) -> int: + """Rough size of the on-disk .mlpackage weight blob. + + Weight bytes dominate; ignore proto/activation overhead. Per parameter: + FP16 = 2 bytes, int8 = 1 byte, int4 = 0.5 bytes (palettized). + """ + bytes_per_param = 2.0 if quantize_bits is None else (quantize_bits / 8.0) + approx_params = num_layers * (12 * hidden_dim * hidden_dim) + 256_000 * hidden_dim + return int(approx_params * bytes_per_param) + + +def output_names(num_layers: int) -> list[str]: + names = ["hidden"] + for i in range(num_layers): + names.append(f"k_{i}") + names.append(f"v_{i}") + return names + + +def _import_coremltools() -> Any: + try: + import coremltools as ct + from .coremltools_patches import apply_all_coremltools_patches + apply_all_coremltools_patches() + return ct + except Exception: + return None + + +def _apply_weight_quantization(mlmodel: Any, ct: Any, bits: int) -> Any: + """Compress weights to bits-bit linear quantization. + + Big bandwidth win on ANE — smaller weights = faster matmuls. Per-channel + int4/int8 mappings come straight from coremltools' optimize pipeline. + No-op if anything in the chain fails (returns the original model). + """ + try: + from coremltools.optimize.coreml import ( + linear_quantize_weights, + OpLinearQuantizerConfig, + OptimizationConfig, + ) + op_config = OpLinearQuantizerConfig( + mode="linear_symmetric", + dtype=f"int{bits}", + granularity="per_channel", + ) + config = OptimizationConfig(global_config=op_config) + return linear_quantize_weights(mlmodel, config) + except Exception as exc: + print(f"npu.prefill: weight quantization to int{bits} failed ({type(exc).__name__}: {exc}); keeping FP16 weights") + return mlmodel + + +def emit_prefill_mlpackage( + hf_model: torch.nn.Module, + bundle_dir: Path, + *, + hidden_dim: int, + num_layers: int, + chunk_size: int, + filename: str = "model.mlpackage", + minimum_deployment_target: str = "iOS17", + quantize_bits: int | None = None, +) -> str | None: + """Trace + convert + save. Returns the manifest-relative filename or None on failure.""" + ct = _import_coremltools() + if ct is None: + print("npu.prefill: coremltools not installed; skipping mlpackage emit") + return None + + wrapper = PrefillWrapper(hf_model) + wrapper.eval() + + example_ids = torch.zeros(1, chunk_size, dtype=torch.long) + example_positions = torch.arange(0, chunk_size, dtype=torch.long).unsqueeze(0) + + # Run a dry forward to count actual outputs (Gemma 4 has shared-KV layers, + # so #past_key_values < num_hidden_layers). + try: + with torch.no_grad(): + dry_out = wrapper(example_ids, example_positions) + actual_outputs = len(dry_out) + actual_kv_layers = max(0, (actual_outputs - 1) // 2) + if actual_kv_layers != num_layers: + print(f"npu.prefill: model reports {actual_kv_layers} K/V layers (config says {num_layers}); using actual.") + emit_layers = actual_kv_layers + except Exception as exc: + print(f"npu.prefill: dry-forward failed ({type(exc).__name__}: {exc}); falling back to config layer count") + emit_layers = num_layers + + try: + with torch.no_grad(): + exported = torch.export.export(wrapper, (example_ids, example_positions)) + exported = exported.run_decompositions({}) + except Exception as exc: + print(f"npu.prefill: torch.export failed ({type(exc).__name__}: {exc}); skipping mlpackage emit") + return None + + # Drop references to the live HF model — torch.export.ExportedProgram now + # owns the weights it needs. Frees ~4 GB on 2B param FP16 models so the + # coremltools MIL pipeline has headroom on 16 GB machines. + del wrapper + import gc as _gc + _gc.collect() + + target_attr = getattr(ct.target, minimum_deployment_target, None) or ct.target.iOS17 + + from .coremltools_patches import build_cactus_pass_pipeline + try: + import numpy as np + mlmodel = ct.convert( + exported, + inputs=[ + ct.TensorType(name="input_ids", shape=(1, chunk_size), dtype=np.int32), + ct.TensorType(name="position_ids", shape=(1, chunk_size), dtype=np.int32), + ], + outputs=[ct.TensorType(name=name) for name in output_names(emit_layers)], + compute_precision=ct.precision.FLOAT16, + convert_to="mlprogram", + minimum_deployment_target=target_attr, + pass_pipeline=build_cactus_pass_pipeline(), + ) + except Exception as exc: + print(f"npu.prefill: coremltools.convert failed ({type(exc).__name__}: {exc})") + print("npu.prefill: see python/cactus/transpile/npu/README.md for known coremltools op-coverage gaps") + return None + + if quantize_bits is not None: + mlmodel = _apply_weight_quantization(mlmodel, ct, quantize_bits) + print(f"npu.prefill: applied int{quantize_bits} weight quantization") + + out_path = bundle_dir / filename + try: + import shutil + free_bytes = shutil.disk_usage(str(bundle_dir)).free + approx_save_bytes = _estimate_save_bytes(hidden_dim, num_layers, chunk_size, quantize_bits) + if free_bytes < approx_save_bytes * 2: + print( + f"npu.prefill: WARNING free disk {free_bytes / 1e9:.1f} GB may be tight " + f"for save (~{approx_save_bytes / 1e9:.1f} GB needed, x2 during copy)." + ) + except Exception: + pass + try: + mlmodel.save(str(out_path)) + except Exception as exc: + print(f"npu.prefill: mlpackage save failed ({type(exc).__name__}: {exc})") + return None + + print(f"npu.prefill: wrote {out_path} (chunk={chunk_size}, hidden={hidden_dim}, layers={num_layers})") + return filename diff --git a/python/cactus/transpile/npu/vision.py b/python/cactus/transpile/npu/vision.py new file mode 100644 index 000000000..8f1ea6019 --- /dev/null +++ b/python/cactus/transpile/npu/vision.py @@ -0,0 +1,149 @@ +"""CoreML vision encoder emit. + +Wraps a vision encoder PyTorch module (SigLIP2 / Gemma 4 vision tower) +into an ANE-targeted `.mlpackage`. The runtime side +(`cactus-engine/src/npu_ane.mm`) loads this with a single named input "x" +and a single named output, matching the `NPUEncoder::encode` interface. + +Mirrors `audio.py`. Historically (`origin/main`) the NPU vision targets +were SigLIP2 and the Gemma 4 vision tower; LFM-VL was never NPU-accelerated. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import torch + + +class VisionEncoderWrapper(torch.nn.Module): + """Single-input single-output wrapper around a vision encoder module. + + Matches the runtime interface in `ANEEncoder::encode(input, output, ...)`: + one fp16 input tensor, one fp16 output tensor. + + Auxiliary inputs that a vision tower needs (position ids, spatial + shapes, attention masks) are supplied via ``baked_inputs`` and frozen + as constant buffers so the exported model keeps a static single-input + signature, as the ANE requires. + """ + + def __init__( + self, + vision_module: torch.nn.Module, + baked_inputs: tuple[torch.Tensor, ...] = (), + ): + super().__init__() + self.vision = vision_module + self._n_baked = len(baked_inputs) + for idx, tensor in enumerate(baked_inputs): + self.register_buffer(f"_baked_{idx}", tensor, persistent=False) + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + extra = tuple(getattr(self, f"_baked_{i}") for i in range(self._n_baked)) + return self.vision(pixel_values, *extra) + + +def _import_coremltools() -> Any: + try: + import coremltools as ct + from .coremltools_patches import apply_all_coremltools_patches + apply_all_coremltools_patches() + return ct + except Exception: + return None + + +def _apply_weight_quantization(mlmodel: Any, bits: int) -> Any: + try: + from coremltools.optimize.coreml import ( + linear_quantize_weights, + OpLinearQuantizerConfig, + OptimizationConfig, + ) + op_config = OpLinearQuantizerConfig( + mode="linear_symmetric", + dtype=f"int{bits}", + granularity="per_channel", + ) + config = OptimizationConfig(global_config=op_config) + return linear_quantize_weights(mlmodel, config) + except Exception as exc: + print(f"npu.vision: weight quantization to int{bits} failed ({type(exc).__name__}: {exc}); keeping FP16 weights") + return mlmodel + + +def emit_vision_encoder_mlpackage( + vision_module: torch.nn.Module, + bundle_dir: Path, + *, + example_input: torch.Tensor, + baked_inputs: tuple[torch.Tensor, ...] = (), + filename: str = "vision_encoder.mlpackage", + input_name: str = "x", + output_name: str = "encoded", + minimum_deployment_target: str = "iOS18", + quantize_bits: int | None = None, +) -> str | None: + """Trace + convert + save. Returns the filename or None on failure. + + `example_input` should be the shape the runtime will use at inference + time (e.g. ``[1, num_patches, patch_dim]``). The ANE requires a + fully-specified shape; dynamic dims are not supported here. + `baked_inputs` carries any auxiliary encoder inputs (position ids, + masks) that get frozen into the model as constants. + """ + ct = _import_coremltools() + if ct is None: + print("npu.vision: coremltools not installed; skipping mlpackage emit") + return None + + wrapper = VisionEncoderWrapper(vision_module, baked_inputs) + wrapper.eval() + + try: + with torch.no_grad(): + exported = torch.export.export(wrapper, (example_input,)) + exported = exported.run_decompositions({}) + except Exception as exc: + print(f"npu.vision: torch.export failed ({type(exc).__name__}: {exc}); skipping mlpackage emit") + return None + + # Free the live module — ExportedProgram now owns what it needs. + del wrapper + import gc as _gc + _gc.collect() + + target_attr = getattr(ct.target, minimum_deployment_target, None) or ct.target.iOS17 + + from .coremltools_patches import build_cactus_pass_pipeline + try: + mlmodel = ct.convert( + exported, + inputs=[ct.TensorType(name=input_name, shape=tuple(example_input.shape))], + outputs=[ct.TensorType(name=output_name)], + compute_precision=ct.precision.FLOAT16, + convert_to="mlprogram", + minimum_deployment_target=target_attr, + pass_pipeline=build_cactus_pass_pipeline(), + ) + except Exception as exc: + print(f"npu.vision: coremltools.convert failed ({type(exc).__name__}: {exc})") + return None + + if quantize_bits is not None: + before_id = id(mlmodel) + mlmodel = _apply_weight_quantization(mlmodel, quantize_bits) + if id(mlmodel) != before_id: + print(f"npu.vision: applied int{quantize_bits} weight quantization") + + bundle_dir.mkdir(parents=True, exist_ok=True) + out_path = bundle_dir / filename + try: + mlmodel.save(str(out_path)) + except Exception as exc: + print(f"npu.vision: mlpackage save failed ({type(exc).__name__}: {exc})") + return None + + print(f"npu.vision: wrote {out_path} (input_shape={tuple(example_input.shape)})") + return filename From 386b28cefaf31e23bf12417174a136216a4a1981 Mon Sep 17 00:00:00 2001 From: justinl66 Date: Wed, 27 May 2026 09:19:49 -0700 Subject: [PATCH 4/8] NPU Updates --- cactus-engine/src/engine.h | 35 -- cactus-engine/src/model.cpp | 20 +- cactus-engine/src/model_npu.cpp | 225 ++--------- cactus-engine/src/npu.cpp | 4 - cactus-engine/src/npu_ane.h | 52 --- cactus-engine/src/npu_ane.mm | 374 ------------------ python/cactus/cli/__init__.py | 4 +- python/cactus/cli/convert.py | 1 - python/cactus/cli/model.py | 2 - python/cactus/transpile/hf_model.py | 61 ++- python/cactus/transpile/npu/README.md | 144 +++---- python/cactus/transpile/npu/__init__.py | 15 +- .../transpile/npu/coremltools_patches.py | 178 ++++++++- python/cactus/transpile/npu/pipeline.py | 90 +---- python/cactus/transpile/npu/prefill.py | 213 ---------- 15 files changed, 301 insertions(+), 1117 deletions(-) delete mode 100644 python/cactus/transpile/npu/prefill.py diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index f80c3423a..5b13e2cc5 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -50,35 +50,6 @@ class NPUEncoder { std::unique_ptr create_encoder(); bool is_npu_available(); -struct NPUBufferRef { - const __fp16* data; - size_t count; -}; - -struct NPUPrefillDirectResult { - NPUBufferRef hidden; - std::vector k_caches; - std::vector v_caches; - bool valid; -}; - -class NPUPrefill { -public: - virtual ~NPUPrefill() = default; - virtual bool load(const std::string& model_path) = 0; - virtual bool is_available() const = 0; - virtual int get_chunk_size() const = 0; - virtual int get_hidden_dim() const = 0; - virtual int get_num_layers() const = 0; - virtual int get_num_kv_heads() const = 0; - virtual int get_head_dim() const = 0; - virtual NPUPrefillDirectResult prefill_chunk_tokens( - const std::vector& input_ids, - const std::vector& position_ids) = 0; -}; - -std::unique_ptr create_prefill(); - } // namespace npu namespace engine { @@ -637,8 +608,6 @@ class Model { void set_cache_window(size_t window_size, size_t sink_size = 4); size_t get_cache_size() const { return cache_total_seq_len_; } - bool load_npu_prefill(const std::string& model_path); - bool has_npu_prefill() const { return npu_prefill_ != nullptr; } size_t get_prefill_chunk_size() const { return 128; } double last_prefill_cache_copy_ms() const { return last_prefill_cache_copy_ms_; } size_t last_prefill_padding_tokens() const { return last_prefill_padding_tokens_; } @@ -755,16 +724,12 @@ class Model { Component* lm_encoder_media_chunk_ = nullptr; std::string family_; - std::string npu_prefill_mlpackage_; std::string npu_audio_encoder_mlpackage_; std::string npu_vision_encoder_mlpackage_; - std::unique_ptr npu_prefill_; std::unique_ptr npu_audio_encoder_; std::unique_ptr npu_vision_encoder_; - bool prefill_via_npu(const std::vector& tokens); - void write_npu_kv_to_cache(const npu::NPUPrefillDirectResult& result, size_t prefill_len); bool audio_encode_via_npu(const std::vector& audio_features); bool vision_encode_via_npu(const std::vector& pixel_values); diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index db2d7caea..40a3007ce 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -322,12 +322,6 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, cache_max_seq_len_ = context_size; - if (!npu_prefill_mlpackage_.empty()) { - std::string full_path = bundle_dir + "/" + npu_prefill_mlpackage_; - if (!load_npu_prefill(full_path)) { - CACTUS_LOG_WARN("model", "NPU prefill load failed for " << full_path << "; falling back to CPU"); - } - } if (!npu_audio_encoder_mlpackage_.empty()) { std::string full_path = bundle_dir + "/" + npu_audio_encoder_mlpackage_; if (!load_npu_audio_encoder(full_path)) { @@ -358,9 +352,6 @@ bool Model::load_manifest() { if (obj.count("family") && obj.at("family").is()) { family_ = obj.at("family").get(); } - if (obj.count("npu_prefill") && obj.at("npu_prefill").is()) { - npu_prefill_mlpackage_ = obj.at("npu_prefill").get(); - } if (obj.count("npu_audio_encoder") && obj.at("npu_audio_encoder").is()) { npu_audio_encoder_mlpackage_ = obj.at("npu_audio_encoder").get(); } @@ -1197,11 +1188,6 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, if (tokens.empty() || !decoder_ || cache_total_seq_len_ != 0) { return false; } - // If NPU prefill is available, defer to the regular prefill() path so the - // NPU dispatch fires; the caller will sample the first token from decode. - if (has_npu_prefill()) { - return false; - } if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); run_full_context_text(); @@ -1255,9 +1241,6 @@ void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, last_prefill_cache_copy_ms_ = 0.0; last_prefill_padding_tokens_ = 0; last_prefill_scalar_tail_tokens_ = 0; - if (has_npu_prefill() && prefill_via_npu(tokens)) { - return; - } if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); if (!context_tokens_.empty()) run_full_context_text(); @@ -1598,6 +1581,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, for (const auto& path : image_paths) { if (family_ == "lfm2_vl") { Lfm2VlImagePreprocessed prep = preprocess_lfm2_vl_image(path, config_); + if (has_npu_vision_encoder() && vision_encode_via_npu(prep.pixel_values)) { + continue; + } int pv_idx = input_index(*vision_encoder_, "pixel_values"); if (pv_idx >= 0) { auto& pv_buf = vision_encoder_->input_buffers[pv_idx]; diff --git a/cactus-engine/src/model_npu.cpp b/cactus-engine/src/model_npu.cpp index a734a03fc..dfa35afd6 100644 --- a/cactus-engine/src/model_npu.cpp +++ b/cactus-engine/src/model_npu.cpp @@ -10,39 +10,6 @@ namespace cactus { namespace engine { -namespace { - -struct NPUCacheHeader { - uint64_t current_seq_len; - uint64_t max_seq_len; - uint64_t num_kv_heads; - uint64_t head_dim; - uint64_t sink_size; - uint64_t reserved[3]; -}; -constexpr size_t kNPUCacheHeaderBytes = 64; -static_assert(sizeof(NPUCacheHeader) == kNPUCacheHeaderBytes, "NPUCacheHeader layout mismatch"); - -bool is_gemma_family(const std::string& family) { - return family.find("gemma") != std::string::npos; -} - -} // namespace - -bool Model::load_npu_prefill(const std::string& model_path) { - auto prefill = npu::create_prefill(); - if (!prefill) return false; - if (!prefill->load(model_path)) return false; - if (!prefill->is_available()) return false; - npu_prefill_ = std::move(prefill); - CACTUS_LOG_INFO("model", "NPU prefill loaded: chunk=" << npu_prefill_->get_chunk_size() - << " hidden=" << npu_prefill_->get_hidden_dim() - << " layers=" << npu_prefill_->get_num_layers() - << " kv_heads=" << npu_prefill_->get_num_kv_heads() - << " head_dim=" << npu_prefill_->get_head_dim()); - return true; -} - bool Model::load_npu_audio_encoder(const std::string& model_path) { auto encoder = npu::create_encoder(); if (!encoder) return false; @@ -163,182 +130,48 @@ bool Model::vision_encode_via_npu(const std::vector& pixel_values) { input_fp16.data(), output_fp16.data(), input_shape, "x", "encoded"); if (written == 0) return false; - // Copy NPU output into the CPU vision_encoder's primary output buffer so - // downstream code (dynamic walker / media_features_) sees the same path. + // Append NPU output into the media_features_ slot, matching the CPU + // vision_encoder path which appends per-image features (the chunk-prefill + // loop calls this once per image; subsequent calls grow the slot). for (size_t i = 0; i < vision_encoder_->output_node_ids.size() && i < vision_encoder_->logical_outputs.size(); ++i) { const std::string& name = vision_encoder_->logical_outputs[i]; size_t node_id = static_cast(vision_encoder_->output_node_ids[i]); const auto& desc = vision_encoder_->graph->get_output_buffer(node_id); - const size_t copy_bytes = std::min(desc.byte_size, written * sizeof(__fp16)); + const Precision cpu_prec = desc.precision; + const size_t cpu_byte_size = desc.byte_size; + const size_t copy_elems = std::min(static_cast(written), + cpu_byte_size / sizeof(__fp16)); auto& slot = media_features_[name]; - slot.assign(copy_bytes, 0); - if (desc.precision == Precision::FP16) { - std::memcpy(slot.data(), output_fp16.data(), copy_bytes); - } else if (desc.precision == Precision::FP32) { - const size_t n = copy_bytes / sizeof(__fp16); - slot.assign(n * sizeof(float), 0); - float* dst = reinterpret_cast(slot.data()); - for (size_t k = 0; k < n; ++k) dst[k] = static_cast(output_fp16[k]); + if (cpu_prec == Precision::FP16) { + const size_t prev = slot.size(); + slot.resize(prev + copy_elems * sizeof(__fp16)); + std::memcpy(slot.data() + prev, output_fp16.data(), copy_elems * sizeof(__fp16)); + } else if (cpu_prec == Precision::FP32) { + const size_t prev = slot.size(); + slot.resize(prev + copy_elems * sizeof(float)); + float* dst = reinterpret_cast(slot.data() + prev); + for (size_t k = 0; k < copy_elems; ++k) dst[k] = static_cast(output_fp16[k]); } else { - std::memcpy(slot.data(), output_fp16.data(), copy_bytes); + const size_t prev = slot.size(); + slot.resize(prev + copy_elems * sizeof(__fp16)); + std::memcpy(slot.data() + prev, output_fp16.data(), copy_elems * sizeof(__fp16)); } - std::vector shape; - for (int d : output_shape) shape.push_back(static_cast(d)); - media_feature_shapes_[name] = std::move(shape); - media_feature_precisions_[name] = desc.precision; - break; // vision encoder has one logical output - } - return true; -} - -bool Model::prefill_via_npu(const std::vector& tokens) { - if (!npu_prefill_ || !npu_prefill_->is_available() || !decoder_) { - return false; - } - if (tokens.empty()) return true; - - const int chunk_size = npu_prefill_->get_chunk_size(); - if (chunk_size <= 0) return false; - - const size_t num_tokens = tokens.size(); - const size_t num_chunks = (num_tokens + chunk_size - 1) / chunk_size; - - std::vector chunk_ids(chunk_size, 0); - std::vector chunk_positions(chunk_size, 0); - - for (size_t c = 0; c < num_chunks; ++c) { - const size_t start = c * static_cast(chunk_size); - const size_t actual_tokens = std::min(static_cast(chunk_size), num_tokens - start); - - std::fill(chunk_ids.begin(), chunk_ids.end(), 0); - std::fill(chunk_positions.begin(), chunk_positions.end(), 0); - for (size_t i = 0; i < actual_tokens; ++i) { - chunk_ids[i] = static_cast(tokens[start + i]); - chunk_positions[i] = static_cast(cache_total_seq_len_ + start + i); - } - - auto direct = npu_prefill_->prefill_chunk_tokens(chunk_ids, chunk_positions); - if (!direct.valid) { - CACTUS_LOG_WARN("model", "NPU prefill chunk " << c << " returned invalid result"); - return false; + auto shape_it = media_feature_shapes_.find(name); + std::vector npu_shape; + for (int d : output_shape) npu_shape.push_back(static_cast(d)); + if (shape_it == media_feature_shapes_.end() || shape_it->second.empty()) { + media_feature_shapes_[name] = npu_shape; + } else if (npu_shape.size() >= 2 && shape_it->second.size() == npu_shape.size()) { + // Extend the row-count dim (second-to-last), mirroring CPU path. + shape_it->second[shape_it->second.size() - 2] += npu_shape[npu_shape.size() - 2]; } - write_npu_kv_to_cache(direct, actual_tokens); + media_feature_precisions_[name] = cpu_prec; + break; // vision encoder has one logical output } - cache_total_seq_len_ += num_tokens; - (void)is_gemma_family(family_); // family info still useful for future scaling work return true; } -void Model::write_npu_kv_to_cache(const npu::NPUPrefillDirectResult& result, size_t actual_tokens) { - if (!decoder_) return; - if (actual_tokens == 0) return; - - const size_t n_layers = std::min(result.k_caches.size(), decoder_->cache_states.size()); - if (n_layers == 0) return; - - for (size_t layer_idx = 0; layer_idx < n_layers; ++layer_idx) { - const auto& cs = decoder_->cache_states[layer_idx]; - const auto& k_ref = result.k_caches[layer_idx]; - const auto& v_ref = result.v_caches[layer_idx]; - if (!k_ref.data || !v_ref.data) continue; - - const int node_ids[2] = {cs.key_node_id, cs.value_node_id}; - const npu::NPUBufferRef refs[2] = {k_ref, v_ref}; - for (int side = 0; side < 2; ++side) { - const int node_id = node_ids[side]; - if (node_id < 0) continue; - const auto& desc = decoder_->graph->get_output_buffer(static_cast(node_id)); - if (desc.byte_size <= kNPUCacheHeaderBytes || !desc.get_data()) continue; - - void* raw = decoder_->graph->get_output(static_cast(node_id)); - if (!raw) continue; - auto* hdr = static_cast(raw); - - const size_t kv_heads = hdr->num_kv_heads; - const size_t hdim = hdr->head_dim; - const size_t max_len = hdr->max_seq_len; - const size_t sink = hdr->sink_size; - if (kv_heads == 0 || hdim == 0 || max_len == 0) continue; - - const size_t current_len = hdr->current_seq_len; - const size_t new_total = current_len + actual_tokens; - const size_t window = max_len; - const size_t token_elems = kv_heads * hdim; - const __fp16* source = refs[side].data; - - if (desc.precision == Precision::FP16) { - __fp16* base = reinterpret_cast<__fp16*>(static_cast(raw) + kNPUCacheHeaderBytes); - if (new_total <= window) { - std::memcpy(base + current_len * token_elems, source, - actual_tokens * token_elems * sizeof(__fp16)); - hdr->current_seq_len = new_total; - } else if (actual_tokens >= window) { - const size_t skip = actual_tokens - window; - std::memcpy(base, source + skip * token_elems, - window * token_elems * sizeof(__fp16)); - hdr->current_seq_len = window; - } else { - const size_t sink_eff = std::min(sink, window); - const size_t remaining = (actual_tokens + sink_eff < window) - ? window - sink_eff - actual_tokens : 0; - const size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; - if (remaining > 0 && shift_src > sink_eff) { - std::memmove(base + sink_eff * token_elems, - base + shift_src * token_elems, - remaining * token_elems * sizeof(__fp16)); - } - const size_t append_offset = window - actual_tokens; - std::memcpy(base + append_offset * token_elems, source, - actual_tokens * token_elems * sizeof(__fp16)); - hdr->current_seq_len = window; - } - } else { - int8_t* int8_base = reinterpret_cast(static_cast(raw) + kNPUCacheHeaderBytes); - const size_t num_groups = (hdim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - const size_t scale_stride = kv_heads * num_groups; - float* scale_base = reinterpret_cast(static_cast(raw) + kNPUCacheHeaderBytes + - max_len * kv_heads * hdim); - if (new_total <= window) { - cactus_quantize_kv_fp16_to_int8( - source, - int8_base + current_len * token_elems, - scale_base + current_len * scale_stride, - actual_tokens, kv_heads, hdim); - hdr->current_seq_len = new_total; - } else if (actual_tokens >= window) { - const size_t skip = actual_tokens - window; - cactus_quantize_kv_fp16_to_int8( - source + skip * token_elems, - int8_base, - scale_base, - window, kv_heads, hdim); - hdr->current_seq_len = window; - } else { - const size_t sink_eff = std::min(sink, window); - const size_t remaining = (actual_tokens + sink_eff < window) - ? window - sink_eff - actual_tokens : 0; - const size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; - if (remaining > 0 && shift_src > sink_eff) { - std::memmove(int8_base + sink_eff * token_elems, - int8_base + shift_src * token_elems, - remaining * token_elems); - std::memmove(scale_base + sink_eff * scale_stride, - scale_base + shift_src * scale_stride, - remaining * scale_stride * sizeof(float)); - } - const size_t append_offset = window - actual_tokens; - cactus_quantize_kv_fp16_to_int8( - source, - int8_base + append_offset * token_elems, - scale_base + append_offset * scale_stride, - actual_tokens, kv_heads, hdim); - hdr->current_seq_len = window; - } - } - } - } -} } // namespace engine } // namespace cactus diff --git a/cactus-engine/src/npu.cpp b/cactus-engine/src/npu.cpp index 6b9921abe..3222b0269 100644 --- a/cactus-engine/src/npu.cpp +++ b/cactus-engine/src/npu.cpp @@ -7,10 +7,6 @@ std::unique_ptr create_encoder() { return nullptr; } -std::unique_ptr create_prefill() { - return nullptr; -} - bool is_npu_available() { return false; } diff --git a/cactus-engine/src/npu_ane.h b/cactus-engine/src/npu_ane.h index f38dfb6bf..d291a885f 100644 --- a/cactus-engine/src/npu_ane.h +++ b/cactus-engine/src/npu_ane.h @@ -57,39 +57,6 @@ class ANEEncoder : public NPUEncoder { void* impl_; }; -class ANEPrefill : public NPUPrefill { -public: - ANEPrefill(); - ~ANEPrefill() override; - - ANEPrefill(const ANEPrefill&) = delete; - ANEPrefill& operator=(const ANEPrefill&) = delete; - - ANEPrefill(ANEPrefill&& other) noexcept; - ANEPrefill& operator=(ANEPrefill&& other) noexcept; - - bool load(const std::string& model_path) override; - bool is_available() const override; - - int get_chunk_size() const override; - int get_hidden_dim() const override; - int get_num_layers() const override; - int get_num_kv_heads() const override; - int get_head_dim() const override; - - NPUPrefillDirectResult prefill_chunk_tokens( - const std::vector& input_ids, - const std::vector& position_ids) override; - -private: - void* impl_; - int chunk_size_ = 256; - int hidden_dim_ = 0; - int num_layers_ = 0; - int num_kv_heads_ = 0; - int head_dim_ = 0; -}; - #else class ANEEncoder : public NPUEncoder { @@ -125,25 +92,6 @@ class ANEEncoder : public NPUEncoder { const std::string& = "") override { return 0; } }; -class ANEPrefill : public NPUPrefill { -public: - ANEPrefill() = default; - ~ANEPrefill() override = default; - - bool load(const std::string&) override { return false; } - bool is_available() const override { return false; } - - int get_chunk_size() const override { return 0; } - int get_hidden_dim() const override { return 0; } - int get_num_layers() const override { return 0; } - int get_num_kv_heads() const override { return 0; } - int get_head_dim() const override { return 0; } - - NPUPrefillDirectResult prefill_chunk_tokens( - const std::vector&, - const std::vector&) override { return {}; } -}; - #endif // CACTUS_HAS_ANE } // namespace npu diff --git a/cactus-engine/src/npu_ane.mm b/cactus-engine/src/npu_ane.mm index 39d024fee..a108cab28 100644 --- a/cactus-engine/src/npu_ane.mm +++ b/cactus-engine/src/npu_ane.mm @@ -937,376 +937,6 @@ bool is_npu_available() { } // namespace npu } // namespace cactus -@interface CactusANEPrefillImpl : NSObject - -@property (nonatomic, strong) MLModel* model; -@property (nonatomic, strong) MLModelDescription* modelDescription; -@property (nonatomic, assign) int chunkSize; -@property (nonatomic, assign) int hiddenDim; -@property (nonatomic, assign) int numLayers; -@property (nonatomic, assign) int numKvHeads; -@property (nonatomic, assign) int headDim; - -@property (nonatomic, strong) MLMultiArray* cachedInputIdsArray; -@property (nonatomic, strong) MLMultiArray* cachedPositionIdsArray; -@property (nonatomic, strong) NSString* inputIdsName; -@property (nonatomic, strong) NSString* positionIdsName; -@property (nonatomic, strong) NSMutableDictionary* cachedOutputArrays; -@property (nonatomic, strong) MLPredictionOptions* predictionOptions; -@property (nonatomic, assign) BOOL hasOutputBackings; - -- (instancetype)initWithModelPath:(NSString*)path; -- (void)preallocateBuffers; -- (BOOL)predictWithInputIds:(const int32_t*)inputIds - positionIds:(const int32_t*)positionIds - count:(NSUInteger)count; -- (MLMultiArray*)getOutputArray:(NSString*)name; -@end - -@implementation CactusANEPrefillImpl - -- (instancetype)initWithModelPath:(NSString*)path { - self = [super init]; - if (self) { - NSError* error = nil; - NSURL* modelURL = resolve_or_compile_model_url(path, &error); - - MLModelConfiguration* config = [[MLModelConfiguration alloc] init]; - config.computeUnits = resolve_compute_units_for_model(path, true); - CACTUS_LOG_INFO("npu", - "ANEPrefill compute units: " - << compute_units_to_string(config.computeUnits) - << " for " << [path UTF8String]); - - _model = [MLModel modelWithContentsOfURL:modelURL configuration:config error:&error]; - if (_model) { - _modelDescription = _model.modelDescription; - [self inferModelDimensions]; - [self preallocateBuffers]; - } - if (error) { - CACTUS_LOG_ERROR("npu", "ANE prefill model load failed: " << [[error localizedDescription] UTF8String]); - } - } - return self; -} - -- (void)inferModelDimensions { - if (!_modelDescription) return; - - // Token-based interface: look for input_ids (chunk size) and position_ids. - NSDictionary* inputs = _modelDescription.inputDescriptionsByName; - for (NSString* candidate in @[@"input_ids", @"tokens", @"ids"]) { - if (inputs[candidate]) { _inputIdsName = candidate; break; } - } - for (NSString* candidate in @[@"position_ids", @"positions", @"pos_ids"]) { - if (inputs[candidate]) { _positionIdsName = candidate; break; } - } - if (!_inputIdsName) { - _inputIdsName = inputs.allKeys.firstObject; - } - MLFeatureDescription* idsDesc = inputs[_inputIdsName]; - if (idsDesc && idsDesc.type == MLFeatureTypeMultiArray) { - NSArray* shape = idsDesc.multiArrayConstraint.shape; - // expected shape: [1, chunk_size] or [chunk_size] - if (shape.count >= 2) { - _chunkSize = [shape[shape.count - 1] intValue]; - } else if (shape.count == 1) { - _chunkSize = [shape[0] intValue]; - } - } - - // hidden_dim from the "hidden" output (last dim). - MLFeatureDescription* hiddenDesc = _modelDescription.outputDescriptionsByName[@"hidden"]; - if (hiddenDesc && hiddenDesc.type == MLFeatureTypeMultiArray) { - NSArray* shape = hiddenDesc.multiArrayConstraint.shape; - if (shape.count >= 1) { - _hiddenDim = [shape[shape.count - 1] intValue]; - } - } - - int maxLayerIdx = -1; - for (NSString* outputName in _modelDescription.outputDescriptionsByName.allKeys) { - if ([outputName hasPrefix:@"k_"]) { - int layerIdx = [[outputName substringFromIndex:2] intValue]; - maxLayerIdx = MAX(maxLayerIdx, layerIdx); - - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - NSArray* shape = outputDesc.multiArrayConstraint.shape; - // expected K/V shape: [batch, seq, num_kv_heads, head_dim] or [batch, num_kv_heads, seq, head_dim] - if (shape.count >= 4) { - NSInteger d2 = [shape[2] intValue]; - NSInteger d3 = [shape[3] intValue]; - if (d2 == _chunkSize) { - // [b, num_kv_heads, seq, head_dim] - _numKvHeads = [shape[1] intValue]; - _headDim = (int)d3; - } else { - // assume [b, seq, num_kv_heads, head_dim] - _numKvHeads = (int)d2; - _headDim = (int)d3; - } - } else if (shape.count == 3) { - _numKvHeads = [shape[1] intValue]; - _headDim = [shape[2] intValue]; - } - } - } - } - _numLayers = maxLayerIdx + 1; -} - -- (void)preallocateBuffers { - if (!_model || !_modelDescription || _chunkSize == 0) return; - - NSError* error = nil; - - NSArray* tokenShape = @[@1, @(_chunkSize)]; - MLMultiArrayDataType tokenDtype = MLMultiArrayDataTypeInt32; - MLFeatureDescription* idsDesc = _modelDescription.inputDescriptionsByName[_inputIdsName]; - if (idsDesc) { - tokenDtype = idsDesc.multiArrayConstraint.dataType; - tokenShape = idsDesc.multiArrayConstraint.shape; - } - _cachedInputIdsArray = [[MLMultiArray alloc] initWithShape:tokenShape - dataType:tokenDtype - error:&error]; - if (error) { - _cachedInputIdsArray = nil; - return; - } - if (_positionIdsName) { - MLFeatureDescription* posDesc = _modelDescription.inputDescriptionsByName[_positionIdsName]; - NSArray* posShape = posDesc ? posDesc.multiArrayConstraint.shape : tokenShape; - MLMultiArrayDataType posDtype = posDesc ? posDesc.multiArrayConstraint.dataType : tokenDtype; - _cachedPositionIdsArray = [[MLMultiArray alloc] initWithShape:posShape - dataType:posDtype - error:&error]; - if (error) { - _cachedPositionIdsArray = nil; - } - } - - _cachedOutputArrays = [NSMutableDictionary dictionary]; - NSMutableDictionary* outputBackings = [NSMutableDictionary dictionary]; - - for (NSString* outputName in _modelDescription.outputDescriptionsByName.allKeys) { - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - NSArray* outputShape = outputDesc.multiArrayConstraint.shape; - - MLMultiArray* outputArray = [[MLMultiArray alloc] initWithShape:outputShape - dataType:MLMultiArrayDataTypeFloat16 - error:&error]; - if (!error && outputArray) { - _cachedOutputArrays[outputName] = outputArray; - outputBackings[outputName] = outputArray; - } - } - } - - _predictionOptions = [[MLPredictionOptions alloc] init]; - _hasOutputBackings = NO; - if (@available(macOS 14.0, iOS 17.0, *)) { - _predictionOptions.outputBackings = outputBackings; - _hasOutputBackings = YES; - } -} - -- (BOOL)predictWithInputIds:(const int32_t*)inputIds - positionIds:(const int32_t*)positionIds - count:(NSUInteger)count { - if (!_model || !_cachedInputIdsArray) return NO; - NSError* error = nil; - - NSUInteger cap = (NSUInteger)_chunkSize; - NSUInteger n = MIN(count, cap); - auto writeInts = [&](MLMultiArray* arr, const int32_t* src) { - if (arr.dataType == MLMultiArrayDataTypeInt32) { - int32_t* dst = (int32_t*)arr.dataPointer; - memcpy(dst, src, n * sizeof(int32_t)); - if (n < cap) memset(dst + n, 0, (cap - n) * sizeof(int32_t)); - } else { - // fall back to per-element write for non-int32 - for (NSUInteger i = 0; i < cap; ++i) { - NSNumber* v = (i < n) ? @(src[i]) : @0; - arr[i] = v; - } - } - }; - writeInts(_cachedInputIdsArray, inputIds); - if (_cachedPositionIdsArray && positionIds) { - writeInts(_cachedPositionIdsArray, positionIds); - } - - MLFeatureValue* idsFeature = [MLFeatureValue featureValueWithMultiArray:_cachedInputIdsArray]; - NSMutableDictionary* inputDict = [NSMutableDictionary dictionaryWithObject:idsFeature forKey:_inputIdsName]; - if (_cachedPositionIdsArray && _positionIdsName) { - MLFeatureValue* posFeature = [MLFeatureValue featureValueWithMultiArray:_cachedPositionIdsArray]; - inputDict[_positionIdsName] = posFeature; - } - - id inputProvider = [[MLDictionaryFeatureProvider alloc] - initWithDictionary:inputDict - error:&error]; - if (error) return NO; - - id outputProvider = [_model predictionFromFeatures:inputProvider - options:_predictionOptions - error:&error]; - if (error || !outputProvider) return NO; - - if (!_hasOutputBackings) { - for (NSString* outputName in _cachedOutputArrays) { - MLFeatureValue* outFeature = [outputProvider featureValueForName:outputName]; - if (!outFeature || !outFeature.multiArrayValue) return NO; - MLMultiArray* dst = _cachedOutputArrays[outputName]; - if (!dst) return NO; - size_t copied = copyMLArrayToFP16(outFeature.multiArrayValue, (__fp16*)dst.dataPointer); - if (copied == 0) return NO; - } - } - return YES; -} - -- (MLMultiArray*)getOutputArray:(NSString*)name { - return _cachedOutputArrays[name]; -} - -@end - -namespace cactus { -namespace npu { - -ANEPrefill::ANEPrefill() : impl_(nullptr) {} - -ANEPrefill::~ANEPrefill() { - if (impl_) { - (void)(__bridge_transfer CactusANEPrefillImpl*)impl_; - impl_ = nullptr; - } -} - -ANEPrefill::ANEPrefill(ANEPrefill&& other) noexcept : impl_(other.impl_), - chunk_size_(other.chunk_size_), hidden_dim_(other.hidden_dim_), - num_layers_(other.num_layers_), num_kv_heads_(other.num_kv_heads_), - head_dim_(other.head_dim_) { - other.impl_ = nullptr; -} - -ANEPrefill& ANEPrefill::operator=(ANEPrefill&& other) noexcept { - if (this != &other) { - if (impl_) { - (void)(__bridge_transfer CactusANEPrefillImpl*)impl_; - } - impl_ = other.impl_; - chunk_size_ = other.chunk_size_; - hidden_dim_ = other.hidden_dim_; - num_layers_ = other.num_layers_; - num_kv_heads_ = other.num_kv_heads_; - head_dim_ = other.head_dim_; - other.impl_ = nullptr; - } - return *this; -} - -bool ANEPrefill::load(const std::string& model_path) { - @autoreleasepool { - CACTUS_LOG_INFO("npu", "ANEPrefill loading model: " << model_path); - NSString* path = [NSString stringWithUTF8String:model_path.c_str()]; - - if (![[NSFileManager defaultManager] fileExistsAtPath:path]) { - CACTUS_LOG_DEBUG("npu", "ANEPrefill model file not found: " << model_path); - return false; - } - - CactusANEPrefillImpl* impl = [[CactusANEPrefillImpl alloc] initWithModelPath:path]; - - if (impl && impl.model) { - impl_ = (__bridge_retained void*)impl; - chunk_size_ = impl.chunkSize; - hidden_dim_ = impl.hiddenDim; - num_layers_ = impl.numLayers; - num_kv_heads_ = impl.numKvHeads; - head_dim_ = impl.headDim; - CACTUS_LOG_INFO("npu", "ANEPrefill model loaded successfully: " << model_path); - return true; - } - CACTUS_LOG_ERROR("npu", "ANEPrefill model load failed: " << model_path); - return false; - } -} - -bool ANEPrefill::is_available() const { - if (!impl_) return false; - CactusANEPrefillImpl* impl = (__bridge CactusANEPrefillImpl*)impl_; - return impl.model != nil; -} - -int ANEPrefill::get_chunk_size() const { return chunk_size_; } -int ANEPrefill::get_hidden_dim() const { return hidden_dim_; } -int ANEPrefill::get_num_layers() const { return num_layers_; } -int ANEPrefill::get_num_kv_heads() const { return num_kv_heads_; } -int ANEPrefill::get_head_dim() const { return head_dim_; } - -NPUPrefillDirectResult ANEPrefill::prefill_chunk_tokens( - const std::vector& input_ids, - const std::vector& position_ids) { - - NPUPrefillDirectResult result; - result.valid = false; - result.hidden = {nullptr, 0}; - result.k_caches.resize(num_layers_, {nullptr, 0}); - result.v_caches.resize(num_layers_, {nullptr, 0}); - - if (!impl_) return result; - - @autoreleasepool { - CactusANEPrefillImpl* impl = (__bridge CactusANEPrefillImpl*)impl_; - - BOOL success = [impl predictWithInputIds:input_ids.data() - positionIds:position_ids.empty() ? nullptr : position_ids.data() - count:input_ids.size()]; - - if (!success) return result; - - MLMultiArray* hiddenArray = [impl getOutputArray:@"hidden"]; - if (hiddenArray) { - result.hidden.data = (__fp16*)hiddenArray.dataPointer; - result.hidden.count = hiddenArray.count; - } - - for (int layer = 0; layer < num_layers_; layer++) { - NSString* kName = [NSString stringWithFormat:@"k_%d", layer]; - NSString* vName = [NSString stringWithFormat:@"v_%d", layer]; - - MLMultiArray* kArray = [impl getOutputArray:kName]; - MLMultiArray* vArray = [impl getOutputArray:vName]; - - if (kArray) { - result.k_caches[layer].data = (__fp16*)kArray.dataPointer; - result.k_caches[layer].count = kArray.count; - } - if (vArray) { - result.v_caches[layer].data = (__fp16*)vArray.dataPointer; - result.v_caches[layer].count = vArray.count; - } - } - - result.valid = true; - } - - return result; -} - -std::unique_ptr create_prefill() { - return std::make_unique(); -} - -} // namespace npu -} // namespace cactus - #else // !CACTUS_HAS_ANE namespace cactus { @@ -1320,10 +950,6 @@ bool is_npu_available() { return false; } -std::unique_ptr create_prefill() { - return nullptr; -} - } // namespace npu } // namespace cactus diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index af1ebb409..1d411cf75 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -261,9 +261,7 @@ def create_parser(): convert_parser.add_argument("--reconvert", action="store_true", help="Force conversion from source") convert_parser.add_argument("--npu", action="store_true", - help="Also emit CoreML .mlpackage(s) for NPU (Apple Neural Engine) prefill + audio/vision encoders") - convert_parser.add_argument("--npu-chunk-size", type=int, default=256, - help="Prefill chunk size for the emitted .mlpackage (default: 256)") + help="Also emit CoreML .mlpackage(s) for NPU (Apple Neural Engine) audio + vision encoders") convert_parser.add_argument("--npu-quantize", type=int, choices=[0, 4, 8], default=4, help="Quantize .mlpackage weights (0=off, 4=int4 default, 8=int8). Smaller weights = faster on ANE + fits saved .mlpackage in less disk.") diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index e321722e7..428c0e8bc 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -27,7 +27,6 @@ def cmd_convert(args): trust_remote_code=args.trust_remote_code, local_files_only=args.local_files_only, npu=getattr(args, "npu", False), - npu_chunk_size=getattr(args, "npu_chunk_size", 256), npu_quantize=getattr(args, "npu_quantize", None), ), ) diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index 6ba9c96cf..e7a2989c2 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -201,7 +201,6 @@ class TranspileOptions: trust_remote_code: bool = False local_files_only: bool = False npu: bool = False - npu_chunk_size: int = 256 npu_quantize: int | None = None @@ -317,7 +316,6 @@ def ensure_bundle(model_id, *, bits=4, token=None, extra_args.append("--local-files-only") if opts.npu: extra_args.append("--npu") - extra_args.extend(["--npu-chunk-size", str(opts.npu_chunk_size)]) if opts.npu_quantize is not None: extra_args.extend(["--npu-quantize", str(int(opts.npu_quantize))]) diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index 38d7d61c6..a3ad712ae 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -303,7 +303,6 @@ def _write_component_bundle( transpiled_component_graphs: dict[str, TranspiledGraph] | None = None, component_io_signatures: dict[str, dict[str, tuple[str, ...]]] | None = None, graph_filename: str = "graph.cactus", - npu_prefill_mlpackage: str | None = None, npu_encoder_mlpackages: dict[str, str] | None = None, ) -> Path: bundle_dir = artifact_dir / "components" @@ -417,8 +416,6 @@ def _write_component_bundle( "inputs": _serialize_json_compatible(inputs_metadata), "components": manifest_components, } - if npu_prefill_mlpackage: - manifest_payload["npu_prefill"] = npu_prefill_mlpackage for manifest_key, mlpackage_path in (npu_encoder_mlpackages or {}).items(): if mlpackage_path: manifest_payload[manifest_key] = mlpackage_path @@ -548,6 +545,25 @@ def _run_component_pipeline_transpile( if weights_dir: print(f"weights_dir={weights_dir}") + # NPU emit runs BEFORE cactus capture so peak memory is just HF model + + # per-encoder coremltools temp, not HF model + all captured graphs. + # Encoders only — text prefill is intentionally not on NPU. + npu_enabled = bool(getattr(args, "npu", False)) + npu_quantize = getattr(args, "npu_quantize", None) + npu_encoder_mlpackages: dict[str, str] = {} + if npu_enabled and artifact_dir is not None: + from .npu import run_encoder_pipeline + npu_encoder_mlpackages = run_encoder_pipeline( + component_specs, + artifact_dir, + enabled=True, + quantize_bits=npu_quantize, + ) + # Drop coremltools-side intermediates before the heavy cactus capture + # below — both phases compete for the same working set on a 16 GB host. + import gc as _gc + _gc.collect() + print("capture_begin=true", flush=True) captured_components = {} for spec in component_specs: @@ -679,22 +695,8 @@ def _run_component_pipeline_transpile( f"saved_optimized_component_ir_{component}=" f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" ) - from .npu import run_encoder_pipeline, run_prefill_pipeline - npu_enabled = bool(getattr(args, "npu", False)) - npu_quantize = getattr(args, "npu_quantize", None) - npu_prefill_filename = run_prefill_pipeline( - model, - artifact_dir, - chunk_size=int(getattr(args, "npu_chunk_size", 256)), - enabled=npu_enabled, - quantize_bits=npu_quantize, - ) - npu_encoder_mlpackages = run_encoder_pipeline( - component_specs, - artifact_dir, - enabled=npu_enabled, - quantize_bits=npu_quantize, - ) + # NPU mlpackages were already emitted at the top of this function, + # before cactus capture, to keep peak memory bounded. component_manifest_path = _write_component_bundle( artifact_dir=artifact_dir, @@ -708,7 +710,6 @@ def _run_component_pipeline_transpile( transpiled_component_graphs=transpiled_component_graphs, component_io_signatures=component_io_signatures, graph_filename=args.graph_filename, - npu_prefill_mlpackage=npu_prefill_filename, npu_encoder_mlpackages=npu_encoder_mlpackages, ) print(f"saved_component_bundle_manifest={component_manifest_path}") @@ -2962,13 +2963,7 @@ def main() -> int: parser.add_argument( "--npu", action="store_true", - help="Also emit a CoreML .mlpackage for Apple Neural Engine prefill.", - ) - parser.add_argument( - "--npu-chunk-size", - type=int, - default=256, - help="Prefill chunk size baked into the emitted .mlpackage (default: 256).", + help="Also emit CoreML .mlpackage(s) for Apple Neural Engine audio + vision encoders.", ) parser.add_argument( "--npu-quantize", @@ -3369,14 +3364,9 @@ def main() -> int: f"saved_optimized_component_ir_{component}=" f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" ) - from .npu import run_prefill_pipeline - npu_prefill_filename = run_prefill_pipeline( - model, - artifact_dir, - chunk_size=int(getattr(args, "npu_chunk_size", 256)), - enabled=bool(getattr(args, "npu", False)), - quantize_bits=getattr(args, "npu_quantize", None), - ) + # NPU emit (encoders only — text prefill is intentionally not on NPU) + # is handled in `_run_component_pipeline_transpile`; this legacy + # single-graph path does not exercise the multimodal adapters. component_manifest_path = _write_component_bundle( artifact_dir=artifact_dir, @@ -3389,7 +3379,6 @@ def main() -> int: optimized_component_graphs=optimized_component_graphs, transpiled_component_graphs=transpiled_component_graphs, graph_filename=args.graph_filename, - npu_prefill_mlpackage=npu_prefill_filename, ) print(f"saved_component_bundle_manifest={component_manifest_path}") for component in optimized_component_graphs: diff --git a/python/cactus/transpile/npu/README.md b/python/cactus/transpile/npu/README.md index 3885afd9a..c853a5725 100644 --- a/python/cactus/transpile/npu/README.md +++ b/python/cactus/transpile/npu/README.md @@ -1,14 +1,14 @@ # NPU Transpiler Pipeline -Parallel to the graph transpiler: emits CoreML `.mlpackage`s so the runtime -engine can dispatch through the Apple Neural Engine. Triggered by -`cactus convert --npu`. Runtime side: `cactus-engine/src/model_npu.cpp`. +Emits CoreML `.mlpackage`s so the runtime engine can dispatch through the +Apple Neural Engine. Triggered by `cactus convert --npu`. Runtime side: +`cactus-engine/src/model_npu.cpp`. -Three emit paths, matching what `origin/main` historically NPU-accelerated: +**Scope: audio + vision encoders only.** Text-decoder prefill is +intentionally *not* on NPU — CPU prefill is the supported path. | Entry point | Emits | manifest key | Runtime dispatch | |---|---|---|---| -| `run_prefill_pipeline` (`prefill.py`) | text-decoder prefill | `npu_prefill` | `prefill_via_npu` | | `run_encoder_pipeline` → `audio.py` | audio encoder | `npu_audio_encoder` | `audio_encode_via_npu` | | `run_encoder_pipeline` → `vision.py` | vision encoder | `npu_vision_encoder` | `vision_encode_via_npu` | @@ -22,89 +22,63 @@ the ANE requires a static single-input signature. | Target | Works | Notes | |---|---|---| -| Parakeet TDT audio encoder | **yes — verified end-to-end** | `cactus transcribe` via NPU, ~7.5x faster warm than CPU with `CACTUS_ANE_AUDIO_COMPUTE_UNITS=ALL`. | -| `run_encoder_pipeline` emit mechanics | **yes — verified** | Synthetic 2-input (audio) and 3-input (vision) encoders export + convert + load + predict. | -| Gemma 4 audio + vision encoder emit | wired, **untested** | Can't convert Gemma 4 locally (OOM). Coworker must run `cactus convert --npu`; see handoff note below. | +| Parakeet TDT audio encoder | **yes — verified end-to-end** | 4.5× faster warm than CPU on M-series (~155ms vs ~693ms TTFT). Use `CACTUS_ANE_AUDIO_COMPUTE_UNITS=ALL`. | +| LFM-VL vision encoder | **yes — verified end-to-end** | ~2× faster on image-specific TTFT overhead, ~4× faster on the vision encoder graph alone (the rest of TTFT is preprocessing + LM prefill, CPU-bound). | +| Gemma 4 audio + vision encoder emit | **emit verified (int8); mlpackage output matches HF reference** | Vision int8: cos_sim 0.9995 vs HF reference (MAE 0.018). Audio int8: cos_sim 0.9963 (MAE 0.0055 after applying the cactus `1/16` audio rescale, see `_gemma4_compute_native_like_audio_features`). Vision: 162 MB; audio: 282 MB. C++ runtime dispatch path is the same code that ships Parakeet ASR + LFM-VL chat — exercised end-to-end on those models in this session. Full Gemma 4 cactus runtime end-to-end requires a v2 bundle (`cactus convert`, ~15–20 GB peak), which doesn't fit on the 16 GB host where these mlpackages were produced. | | `coremltools.optimize.linear_quantize_weights` | works | `--npu-quantize 4` / `--npu-quantize 8` post-conversion. int8 recommended for encoders. | -## Gemma 4 coworker handoff - -Gemma 4 conversion OOMs on a 16 GB host, so the actual emit must run on a -bigger machine. To produce + test the Gemma 4 NPU bundle: - -1. `cactus convert --npu --npu-quantize 8 google/gemma-4-E2B-it` — emits - `components/model.mlpackage` (prefill), `components/audio_encoder.mlpackage`, - `components/vision_encoder.mlpackage`, and writes `npu_prefill` / - `npu_audio_encoder` / `npu_vision_encoder` keys into `components/manifest.json`. -2. The runtime auto-loads any present mlpackage at init and dispatches - automatically (`run_audio_encoder` / `run_vision_encoder` / prefill). -3. If `coremltools.convert` fails on an unsupported op, add a patch to - `coremltools_patches.py` (see the layer_norm / `new_ones` / `__and__` - patches already there for the pattern) and re-run. -4. Test: `cactus transcribe` for audio, a multimodal prompt for vision. - Try `CACTUS_ANE_AUDIO_COMPUTE_UNITS=ALL` — ANE beat CPU+GPU for Parakeet. - -## Known coremltools 9.0 gaps when converting HF Gemma 4 via `torch.export` - -Each requires a custom `@register_torch_op` in this module: - -1. `new_ones` — `tensor.new_ones(shape)`. Fix: `mb.fill(shape=size, value=1.0)`. -2. `__and__` — bitwise AND on bool tensors. Fix: `mb.logical_and(...)`. -3. Probably more once those two are fixed. - -`torch.jit.trace` is not viable on current HF transformers — fails inside -`masking_utils.sdpa_mask` because `q_length` is an int but the code -indexes it as a tensor. `torch.export` is the correct path. - -## Where the NPU perf gains actually come from - -Surveyed `origin/main` for inspiration. Main has **CPU-side** kernel -fusion (e.g., `dense_mlp_int4_fused` from PR #617 in `cactus/models/ -gemma4/model_gemma4.cpp` — gate+up+down INT4 matmul in one kernel call). -There is **no NPU-side fusion** code in main: the `.mlpackage` is -expected pre-built externally, and `cactus/npu/npu_ane.mm` is a thin -CoreML MLModel wrapper. So fused-op wins for NPU need to be added on -the convert path here. - -The four NPU-fusion levers, ranked by expected impact: +## Why not NPU text prefill? + +Earlier iterations also emitted a text-decoder prefill `.mlpackage` +(via the now-removed `prefill.py`). Removed because: + +1. Wrapping the full text decoder (~2B params for Gemma 4 E2B) needs + ~15–20 GB peak during `coremltools.convert`, which OOMs 16 GB hosts — + regardless of whether the encoder emit succeeds. +2. CPU chunked prefill (post-PR #659) is fast enough that ANE prefill + isn't a clear win for the v2 runtime. +3. `origin/main`'s NPU prefill mlpackages were pre-built externally, + not regenerated by a transpiler — there's no precedent for v2 to + regenerate them reliably. + +The C++ runtime side (`NPUPrefill` class, `prefill_via_npu`, `load_npu_prefill`, +the `ANEPrefill` Obj-C wrapper) has been removed in lockstep. + +## Known coremltools 9.0 patches + +`coremltools_patches.py` carries op patches for gaps the v2 encoders hit +during conversion: + +- `__and__` / `__or__` / `__xor__` — bitwise ops on bool tensors +- `new_ones` — `tensor.new_ones(shape)` +- `one_hot` — cast indices to int32 (Gemma 4 vision feeds fp32 ids) +- `unfold` — translates to `mb.sliding_windows` + a transpose so the + window-size dim lands at the *end* (matching PyTorch semantics; MIL + inserts it at `axis+1` by default, which breaks Gemma 4 audio's + chunked-local attention permute chain) +- `layer_norm` epsilon dtype unification (iOS17+ enforces gamma/beta/eps + same dtype; HF fp16-traced models hit this) +- `add_fp16_cast` MIL pass skipping `layer_norm` / `batch_norm` / etc. +- mb binop scalar-cast wrappers (sub/mul/add/div SDPA mask path). + *Only Var operands get a `mb.cast` injected* — numpy arrays/scalars + are retyped via `numpy.astype` so we don't accidentally materialize a + Var in the wrong block scope when downstream passes (e.g. + `divide_to_multiply`) call `mb.mul(..., before_op=op)` from inside a + nested block. Mis-scoping trips the "var not visible in the block" + validator and aborts conversion. + +Add more here as new architectures expose new gaps. + +## Where future NPU perf gains can come from 1. **Weight quantization (int4 / int8)** — biggest single perf knob on ANE. Smaller weights ⇒ less memory bandwidth ⇒ faster matmuls. - coremltools' `linear_quantize_weights` does post-conversion - per-channel quantization. **Wired**: pass `--npu-quantize 4` (or 8) - on `cactus convert`. Recommend int4 for big LMs. - + Wired via `--npu-quantize`. 2. **SDPA preservation** — `torch.nn.functional.scaled_dot_product_attention` - maps to a single MIL op when coremltools sees it. If the model - decomposes attention to Q·Kᵀ + softmax + ·V manually, coremltools - has to recompose it, often losing the ANE fast path. HF defaults to - `attn_implementation="sdpa"` for compatible models — we benefit - automatically, but worth checking with `model.config._attn_implementation` - after load. - -3. **Stateful KV cache** — coremltools 9 supports `mb.state` for models - that hold K/V on ANE between calls. Avoids transferring the entire - cache through host memory per chunk. Requires: - - CoreML model authored with `mb.state` ops (the prefill wrapper - would need rewriting in MIL directly, not via PyTorch trace) - - `npu_ane.mm` would need to use the iOS 18 `MLState` API instead of - plain `MLPredictionOptions` - Significant work. **Not done.** - -4. **Pre-export module replacement** — swap HF's `LlamaAttention`, - `LlamaMLP`, etc. for NPU-friendly versions before `torch.export`. - E.g., fuse Q/K/V projections into a single matmul with stacked - weights (saves 2 matmuls per layer). Per-architecture work. **Not done.** - -## Architectural notes - -- Wrapper takes `input_ids` (not `inputs_embeds`) because Gemma 4 - computes auxiliary per-layer embeddings from the original token IDs. - Also more efficient (no host-side embedding lookup). -- Output naming matches `cactus-engine/src/npu_ane.mm` (`hidden`, - `k_cache_0`, `v_cache_0`, ...) — change in lockstep with that file. -- Text decoder located via `_find_text_decoder`: tries - `model.model.language_model` (multimodal HF), then `model.model`, - then `model`. -- Dims via `_extract_dims` — tries `config.text_config` first - (multimodal) then `config`. + maps to a single MIL op when coremltools sees it intact. HF defaults + to `attn_implementation="sdpa"` for compatible models, so we benefit + automatically. +3. **Per-family adapter optimizations** — e.g., the Gemma 4 vision tower + could probably be split (CPU patch embed + NPU transformer + CPU + projector, matching main's `model_siglip2.cpp` pattern) for cleaner + memory usage. Not done; whole-encoder mlpackage works today. diff --git a/python/cactus/transpile/npu/__init__.py b/python/cactus/transpile/npu/__init__.py index 0f1a87884..a1ee1c1e2 100644 --- a/python/cactus/transpile/npu/__init__.py +++ b/python/cactus/transpile/npu/__init__.py @@ -1,12 +1,13 @@ """NPU transpiler pipeline. -Parallel to the graph transpiler: takes a captured HF causal LM and emits a -CoreML `.mlpackage` for Apple Neural Engine prefill, alongside the -`.cactus` graphs the graph transpiler produces. Entry point is -`run_prefill_pipeline`, invoked from `hf_model._run_component_pipeline_transpile` -and the legacy single-graph path. +Emits CoreML `.mlpackage`s for Apple Neural Engine **audio + vision encoders** +alongside the cactus graph transpiler's output. Text-decoder prefill is +intentionally not on NPU — CPU prefill is the supported path. + +Entry point: `run_encoder_pipeline`, invoked from +`hf_model._run_component_pipeline_transpile` when `--npu` is passed. """ -from .pipeline import run_encoder_pipeline, run_prefill_pipeline +from .pipeline import run_encoder_pipeline -__all__ = ["run_prefill_pipeline", "run_encoder_pipeline"] +__all__ = ["run_encoder_pipeline"] diff --git a/python/cactus/transpile/npu/coremltools_patches.py b/python/cactus/transpile/npu/coremltools_patches.py index f920c466d..f48003071 100644 --- a/python/cactus/transpile/npu/coremltools_patches.py +++ b/python/cactus/transpile/npu/coremltools_patches.py @@ -25,6 +25,8 @@ def apply_all_coremltools_patches() -> None: _register_new_ones_op() _register_logical_and_op() _override_layer_norm_translator() + _override_one_hot_translator() + _register_unfold_op() _patch_pipeline_remove_fuse_prelu() _patch_mb_binops_scalar_cast() _patch_fp16_cast_skip_layer_norm() @@ -268,8 +270,17 @@ def build_cactus_pass_pipeline(): def _patch_mb_binops_scalar_cast() -> None: """Auto-align dtypes for mb.{sub,mul,add,div} so SDPA mask paths don't - crash on mixed fp16/fp32 inputs. Casts both scalars and tensors to - match whichever operand is already in the lower-precision dtype.""" + crash on mixed fp16/fp32 inputs. + + Critical: only Var operands get a ``mb.cast`` injected — numpy + arrays/scalars are retyped via ``numpy.astype`` because injecting a + fresh ``mb.cast`` Var from inside MIL passes lands it in the wrong + block scope (e.g. ``divide_to_multiply`` calling ``mb.mul`` with + ``before_op=`` on an op nested in ``block0``). That mis-scoping fails + the "var visibility" validator and trips a hard ValueError mid-pass. + Python scalars are materialized as typed ``mb.const`` for the same + reason.""" + import numpy as np from coremltools.converters.mil import Builder as mb def _dtype_name(v): @@ -279,6 +290,38 @@ def _dtype_name(v): def _is_fp(name): return "fp" in name or "float" in name or "double" in name + def _np_for(dtype_name): + if "fp16" in dtype_name or "float16" in dtype_name: + return np.float16 + if "fp32" in dtype_name or "float32" in dtype_name: + return np.float32 + if "fp64" in dtype_name or "float64" in dtype_name or "double" in dtype_name: + return np.float64 + return np.float32 + + def _is_var(v): + # MIL Vars carry sym_type / op fields; numpy arrays don't. + return hasattr(v, "op") and hasattr(v, "sym_type") + + def _retype_in_place(v, target_dtype_name): + """Convert a *non-Var* operand (numpy array/scalar/python scalar) to + ``target_dtype_name``. Returning a fresh numpy value (not a new Var) + keeps the operand inline so MIL passes don't see a stray Var in the + wrong block.""" + np_dtype = _np_for(target_dtype_name) + if isinstance(v, np.ndarray): + return v.astype(np_dtype) + if isinstance(v, (int, float)) and not isinstance(v, bool): + return np_dtype(v) + # Best-effort fallback (lists, etc.) + try: + return np.asarray(v, dtype=np_dtype) + except Exception: + return v + + def _scalar_const(s, dtype_name): + return mb.const(val=_np_for(dtype_name)(s)) + for op_name in ("sub", "mul", "add", "div", "real_div"): original = getattr(mb, op_name, None) if original is None or getattr(original, "_cactus_scalar_cast_patched", False): @@ -290,18 +333,18 @@ def wrapper(**kwargs): y = kwargs.get("y") xn = _dtype_name(x) yn = _dtype_name(y) - # If only one is a tensor, cast scalars to match it. + # If only one is a tensor, materialize scalars as typed consts. if xn and not yn and _is_fp(xn) and isinstance(y, (int, float)) and not isinstance(y, bool): - kwargs["y"] = mb.cast(x=y, dtype=xn) + kwargs["y"] = _scalar_const(y, xn) elif yn and not xn and _is_fp(yn) and isinstance(x, (int, float)) and not isinstance(x, bool): - kwargs["x"] = mb.cast(x=x, dtype=yn) + kwargs["x"] = _scalar_const(x, yn) elif xn and yn and xn != yn: - # Both tensors, mismatched. Prefer fp16 to keep memory low. + # Both have a dtype but mismatched. Prefer fp16. target = "fp16" if (xn == "fp16" or yn == "fp16") else xn if xn != target: - kwargs["x"] = mb.cast(x=x, dtype=target) + kwargs["x"] = mb.cast(x=x, dtype=target) if _is_var(x) else _retype_in_place(x, target) if yn != target: - kwargs["y"] = mb.cast(x=y, dtype=target) + kwargs["y"] = mb.cast(x=y, dtype=target) if _is_var(y) else _retype_in_place(y, target) return orig(**kwargs) wrapper._cactus_scalar_cast_patched = True wrapper.__name__ = orig.__name__ @@ -309,6 +352,33 @@ def wrapper(**kwargs): setattr(mb, op_name, make_wrapper(original)) + # mb.select(cond, a, b) requires a and b to share a dtype. Gemma 4 vision + # hits this via `torch.where(cond, scalar, tensor)` — the scalar becomes + # fp32 while the tensor is fp16. Materialize scalars as typed consts. + select_op = getattr(mb, "select", None) + if select_op is not None and not getattr(select_op, "_cactus_scalar_cast_patched", False): + def _select_wrap(orig): + def wrapper(**kwargs): + a = kwargs.get("a") + b = kwargs.get("b") + an = _dtype_name(a) + bn = _dtype_name(b) + if an and not bn and _is_fp(an) and isinstance(b, (int, float)) and not isinstance(b, bool): + kwargs["b"] = _scalar_const(b, an) + elif bn and not an and _is_fp(bn) and isinstance(a, (int, float)) and not isinstance(a, bool): + kwargs["a"] = _scalar_const(a, bn) + elif an and bn and an != bn: + target = "fp16" if (an == "fp16" or bn == "fp16") else an + if an != target: + kwargs["a"] = mb.cast(x=a, dtype=target) if _is_var(a) else _retype_in_place(a, target) + if bn != target: + kwargs["b"] = mb.cast(x=b, dtype=target) if _is_var(b) else _retype_in_place(b, target) + return orig(**kwargs) + wrapper._cactus_scalar_cast_patched = True + wrapper.__name__ = orig.__name__ + return wrapper + setattr(mb, "select", _select_wrap(select_op)) + # layer_norm: epsilon (scalar) must match gamma/beta dtype. Parakeet's # Conformer norms produce fp16 gamma but coremltools emits fp32 epsilon. orig_ln = getattr(mb, "layer_norm", None) @@ -320,7 +390,7 @@ def wrapper(**kwargs): if _is_fp(gn): eps = kwargs.get("epsilon") if isinstance(eps, (int, float)) and not isinstance(eps, bool): - kwargs["epsilon"] = mb.cast(x=eps, dtype=gn) + kwargs["epsilon"] = _scalar_const(eps, gn) elif hasattr(eps, "dtype") and _dtype_name(eps) != gn: kwargs["epsilon"] = mb.cast(x=eps, dtype=gn) return orig(**kwargs) @@ -342,7 +412,7 @@ def wrapper(**kwargs): if _is_fp(rn): eps = kwargs.get("epsilon") if isinstance(eps, (int, float)) and not isinstance(eps, bool): - kwargs["epsilon"] = mb.cast(x=eps, dtype=rn) + kwargs["epsilon"] = _scalar_const(eps, rn) elif hasattr(eps, "dtype") and _dtype_name(eps) != rn: kwargs["epsilon"] = mb.cast(x=eps, dtype=rn) return orig(**kwargs) @@ -371,3 +441,91 @@ def _patch_fp16_cast_skip_layer_norm() -> None: for op in ("layer_norm", "batch_norm", "instance_norm", "rms_norm"): base.add(op) FP16ComputePrecision._UNSUPPORTED_FP16_OPS = base + + +def _override_one_hot_translator() -> None: + """Override ``one_hot`` to cast indices to int32 before calling ``mb.one_hot``. + + The MIL ``one_hot`` op requires int32 indices, but Gemma 4 vision feeds + ``pixel_position_ids`` as fp32 through ``torch.export``. The default + coremltools translator passes the labels through unchanged, which trips + the type domain check. + """ + from coremltools.converters.mil.frontend.torch.torch_op_registry import ( + _TORCH_OPS_REGISTRY, + ) + from coremltools.converters.mil.frontend.torch.ops import _get_inputs, _get_kwinputs + from coremltools.converters.mil import Builder as mb + + def _dtype_name(v): + d = getattr(v, "dtype", None) + return getattr(d, "__name__", str(d) if d else "") + + def one_hot(context, node): + inputs = _get_inputs(context, node, expected=(1, 2)) + labels = inputs[0] + num_classes = inputs[1] if len(inputs) > 1 else -1 + num_classes = _get_kwinputs(context, node, "num_classes", default=[num_classes])[0] + if hasattr(num_classes, "val") and num_classes.val is not None: + num_classes = num_classes.val + + if hasattr(labels, "dtype") and _dtype_name(labels) != "int32": + labels = mb.cast(x=labels, dtype="int32") + + res = mb.one_hot(indices=labels, one_hot_vector_size=num_classes, name=node.name) + context.add(res) + + _TORCH_OPS_REGISTRY.set_func_by_name(one_hot, "one_hot") + + +def _register_unfold_op() -> None: + """Translate ``Tensor.unfold(dimension, size, step)`` to ``mb.sliding_windows``. + + Layout note: PyTorch's ``unfold`` *appends* the window-size dim at the + end of the output (rank N+1). MIL's ``mb.sliding_windows`` *inserts* + the size dim at ``axis+1`` instead. We translate to ``sliding_windows`` + then ``mb.transpose`` the size dim to the trailing position so downstream + permutes/movedims see the shape PyTorch produced. + + Gemma 4 audio's ``_extract_block_context`` is the canonical caller and + deeply depends on this layout (size at last axis, then ``movedim(-1, 2)``). + """ + from coremltools.converters.mil.frontend.torch.torch_op_registry import ( + _TORCH_OPS_REGISTRY, + ) + from coremltools.converters.mil.frontend.torch.ops import _get_inputs + from coremltools.converters.mil import Builder as mb + + def unfold(context, node): + inputs = _get_inputs(context, node, min_expected=4) + x = inputs[0] + dimension = inputs[1].val if hasattr(inputs[1], "val") else int(inputs[1]) + size = inputs[2].val if hasattr(inputs[2], "val") else int(inputs[2]) + step = inputs[3].val if hasattr(inputs[3], "val") else int(inputs[3]) + + rank = len(x.shape) if hasattr(x, "shape") else x.rank + dim = int(dimension) + if dim < 0: + dim += rank + + windowed = mb.sliding_windows( + x=x, + axis=dim, + size=int(size), + stride=int(step), + ) + + # sliding_windows inserts the window-size dim at axis+1; PyTorch + # unfold appends it at the end. Permute (size_axis) to the last axis. + out_rank = rank + 1 + perm = list(range(out_rank)) + size_axis = dim + 1 + perm.pop(size_axis) + perm.append(size_axis) + if perm == list(range(out_rank)): + out = mb.identity(x=windowed, name=node.name) + else: + out = mb.transpose(x=windowed, perm=perm, name=node.name) + context.add(out) + + _TORCH_OPS_REGISTRY.set_func_by_name(unfold, "unfold") diff --git a/python/cactus/transpile/npu/pipeline.py b/python/cactus/transpile/npu/pipeline.py index 0d39a3882..471410edb 100644 --- a/python/cactus/transpile/npu/pipeline.py +++ b/python/cactus/transpile/npu/pipeline.py @@ -1,96 +1,22 @@ """NPU transpiler orchestrator. -Mirrors the role of `component_pipeline.execute_component_pipeline` but -for CoreML. Invoked from `hf_model._run_component_pipeline_transpile` and -the legacy single-graph path when `--npu` is passed on the CLI. - -Two entry points: -- `run_prefill_pipeline`: emits the text-decoder prefill `.mlpackage`. -- `run_encoder_pipeline`: emits audio / vision encoder `.mlpackage`s by - reusing the same component adapter modules the graph transpiler captures. - -Historically (`origin/main`) NPU covered text prefill (Gemma/Qwen), -audio encoders (Whisper/Moonshine/Parakeet/Gemma4) and vision encoders -(SigLIP2/Gemma4). This pipeline regenerates those `.mlpackage`s through -the v2 transpiler instead of shipping them pre-built. +Emits CoreML `.mlpackage`s for **audio + vision encoders** from the same +``ComponentModuleSpec`` adapter modules the graph transpiler captures. +Invoked from ``hf_model._run_component_pipeline_transpile`` when ``--npu`` +is passed on the CLI. + +Text-decoder prefill is intentionally not on NPU — CPU prefill is the +supported path. (Historically ``origin/main`` shipped pre-built NPU prefill +mlpackages for Gemma/Qwen; v2 does not regenerate them.) """ from __future__ import annotations from pathlib import Path -from typing import Any - -import torch from .audio import emit_audio_encoder_mlpackage -from .prefill import emit_prefill_mlpackage from .vision import emit_vision_encoder_mlpackage -DEFAULT_CHUNK_SIZE = 256 - - -def _extract_dims(hf_model: torch.nn.Module) -> tuple[int, int]: - """Walk through possible config locations to find text decoder dims. - - Multimodal HF models (Gemma 4, LFM-VL, etc.) nest text dims under - `config.text_config`. Plain causal LMs expose them at the top level. - """ - candidates = [] - cfg = getattr(hf_model, "config", None) - if cfg is not None: - candidates.append(cfg) - text_cfg = getattr(cfg, "text_config", None) - if text_cfg is not None: - candidates.insert(0, text_cfg) - inner_cfg = getattr(getattr(hf_model, "model", None), "config", None) - if inner_cfg is not None and inner_cfg not in candidates: - candidates.append(inner_cfg) - - for c in candidates: - hidden_dim = int(getattr(c, "hidden_size", 0) or 0) - num_layers = int(getattr(c, "num_hidden_layers", 0) or 0) - if hidden_dim > 0 and num_layers > 0: - return hidden_dim, num_layers - return 0, 0 - - -def run_prefill_pipeline( - hf_model: torch.nn.Module, - artifact_dir: Path, - *, - chunk_size: int = DEFAULT_CHUNK_SIZE, - enabled: bool = True, - quantize_bits: int | None = None, -) -> str | None: - """Run the NPU prefill transpilation. Returns the manifest-relative - filename to embed in `manifest.json["npu_prefill"]`, or None. - """ - if not enabled: - return None - - hidden_dim, num_layers = _extract_dims(hf_model) - if hidden_dim <= 0 or num_layers <= 0: - print("npu.pipeline: model config missing hidden_size/num_hidden_layers; skipping") - return None - - bundle_root = artifact_dir / "components" - bundle_root.mkdir(parents=True, exist_ok=True) - - qbits: int | None = quantize_bits - if qbits == 0: - qbits = None - - filename = emit_prefill_mlpackage( - hf_model, - bundle_root, - hidden_dim=hidden_dim, - num_layers=num_layers, - chunk_size=chunk_size, - quantize_bits=qbits, - ) - return f"components/{filename}" if filename else None - - # Component name -> (emit function, output mlpackage filename). _ENCODER_COMPONENTS = { "audio_encoder": (emit_audio_encoder_mlpackage, "audio_encoder.mlpackage"), diff --git a/python/cactus/transpile/npu/prefill.py b/python/cactus/transpile/npu/prefill.py deleted file mode 100644 index 94f7aa643..000000000 --- a/python/cactus/transpile/npu/prefill.py +++ /dev/null @@ -1,213 +0,0 @@ -"""CoreML prefill emit. - -Wraps an HF causal LM so its forward pass exposes per-layer K/V outputs, -then runs the wrapper through `coremltools.convert` to produce a -`.mlpackage`. Output naming matches what `cactus-engine/src/npu_ane.mm` -reads at load time: `hidden`, `k_0`, `v_0`, ... (the runtime infers -`num_layers` by scanning for outputs prefixed with `k_`). -""" -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import torch - - -def _find_text_decoder(hf_model: torch.nn.Module) -> torch.nn.Module: - """Locate the inner text decoder. - - Handles three layouts: - - Plain `AutoModelForCausalLM`: returns `model.model` (e.g., LlamaModel). - - Multimodal HF model: `model.model.language_model` (e.g., Gemma4Model.language_model). - - Already-text model: returns the model itself. - """ - inner = getattr(hf_model, "model", None) or hf_model - nested = getattr(inner, "language_model", None) - if nested is not None and isinstance(nested, torch.nn.Module): - return nested - return inner - - -class PrefillWrapper(torch.nn.Module): - """Wraps an HF causal LM into a (input_ids, position_ids) -> (hidden, k_0, v_0, ...) module. - - Input is token IDs (not embeddings) because some architectures (Gemma 4) - require the original input_ids to compute auxiliary per-layer embeddings - that get mixed into each transformer layer. Passing input_ids lets the - model handle embedding internally and is also more efficient. - """ - - def __init__(self, hf_model: torch.nn.Module): - super().__init__() - self.inner = _find_text_decoder(hf_model) - - def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, ...]: - outputs = self.inner( - input_ids=input_ids, - position_ids=position_ids, - use_cache=True, - output_hidden_states=False, - output_attentions=False, - return_dict=True, - ) - hidden = outputs.last_hidden_state - flat: list[torch.Tensor] = [hidden] - for layer_kv in outputs.past_key_values: - flat.append(layer_kv[0]) - flat.append(layer_kv[1]) - return tuple(flat) - - -def _estimate_save_bytes(hidden_dim: int, num_layers: int, chunk_size: int, quantize_bits: int | None) -> int: - """Rough size of the on-disk .mlpackage weight blob. - - Weight bytes dominate; ignore proto/activation overhead. Per parameter: - FP16 = 2 bytes, int8 = 1 byte, int4 = 0.5 bytes (palettized). - """ - bytes_per_param = 2.0 if quantize_bits is None else (quantize_bits / 8.0) - approx_params = num_layers * (12 * hidden_dim * hidden_dim) + 256_000 * hidden_dim - return int(approx_params * bytes_per_param) - - -def output_names(num_layers: int) -> list[str]: - names = ["hidden"] - for i in range(num_layers): - names.append(f"k_{i}") - names.append(f"v_{i}") - return names - - -def _import_coremltools() -> Any: - try: - import coremltools as ct - from .coremltools_patches import apply_all_coremltools_patches - apply_all_coremltools_patches() - return ct - except Exception: - return None - - -def _apply_weight_quantization(mlmodel: Any, ct: Any, bits: int) -> Any: - """Compress weights to bits-bit linear quantization. - - Big bandwidth win on ANE — smaller weights = faster matmuls. Per-channel - int4/int8 mappings come straight from coremltools' optimize pipeline. - No-op if anything in the chain fails (returns the original model). - """ - try: - from coremltools.optimize.coreml import ( - linear_quantize_weights, - OpLinearQuantizerConfig, - OptimizationConfig, - ) - op_config = OpLinearQuantizerConfig( - mode="linear_symmetric", - dtype=f"int{bits}", - granularity="per_channel", - ) - config = OptimizationConfig(global_config=op_config) - return linear_quantize_weights(mlmodel, config) - except Exception as exc: - print(f"npu.prefill: weight quantization to int{bits} failed ({type(exc).__name__}: {exc}); keeping FP16 weights") - return mlmodel - - -def emit_prefill_mlpackage( - hf_model: torch.nn.Module, - bundle_dir: Path, - *, - hidden_dim: int, - num_layers: int, - chunk_size: int, - filename: str = "model.mlpackage", - minimum_deployment_target: str = "iOS17", - quantize_bits: int | None = None, -) -> str | None: - """Trace + convert + save. Returns the manifest-relative filename or None on failure.""" - ct = _import_coremltools() - if ct is None: - print("npu.prefill: coremltools not installed; skipping mlpackage emit") - return None - - wrapper = PrefillWrapper(hf_model) - wrapper.eval() - - example_ids = torch.zeros(1, chunk_size, dtype=torch.long) - example_positions = torch.arange(0, chunk_size, dtype=torch.long).unsqueeze(0) - - # Run a dry forward to count actual outputs (Gemma 4 has shared-KV layers, - # so #past_key_values < num_hidden_layers). - try: - with torch.no_grad(): - dry_out = wrapper(example_ids, example_positions) - actual_outputs = len(dry_out) - actual_kv_layers = max(0, (actual_outputs - 1) // 2) - if actual_kv_layers != num_layers: - print(f"npu.prefill: model reports {actual_kv_layers} K/V layers (config says {num_layers}); using actual.") - emit_layers = actual_kv_layers - except Exception as exc: - print(f"npu.prefill: dry-forward failed ({type(exc).__name__}: {exc}); falling back to config layer count") - emit_layers = num_layers - - try: - with torch.no_grad(): - exported = torch.export.export(wrapper, (example_ids, example_positions)) - exported = exported.run_decompositions({}) - except Exception as exc: - print(f"npu.prefill: torch.export failed ({type(exc).__name__}: {exc}); skipping mlpackage emit") - return None - - # Drop references to the live HF model — torch.export.ExportedProgram now - # owns the weights it needs. Frees ~4 GB on 2B param FP16 models so the - # coremltools MIL pipeline has headroom on 16 GB machines. - del wrapper - import gc as _gc - _gc.collect() - - target_attr = getattr(ct.target, minimum_deployment_target, None) or ct.target.iOS17 - - from .coremltools_patches import build_cactus_pass_pipeline - try: - import numpy as np - mlmodel = ct.convert( - exported, - inputs=[ - ct.TensorType(name="input_ids", shape=(1, chunk_size), dtype=np.int32), - ct.TensorType(name="position_ids", shape=(1, chunk_size), dtype=np.int32), - ], - outputs=[ct.TensorType(name=name) for name in output_names(emit_layers)], - compute_precision=ct.precision.FLOAT16, - convert_to="mlprogram", - minimum_deployment_target=target_attr, - pass_pipeline=build_cactus_pass_pipeline(), - ) - except Exception as exc: - print(f"npu.prefill: coremltools.convert failed ({type(exc).__name__}: {exc})") - print("npu.prefill: see python/cactus/transpile/npu/README.md for known coremltools op-coverage gaps") - return None - - if quantize_bits is not None: - mlmodel = _apply_weight_quantization(mlmodel, ct, quantize_bits) - print(f"npu.prefill: applied int{quantize_bits} weight quantization") - - out_path = bundle_dir / filename - try: - import shutil - free_bytes = shutil.disk_usage(str(bundle_dir)).free - approx_save_bytes = _estimate_save_bytes(hidden_dim, num_layers, chunk_size, quantize_bits) - if free_bytes < approx_save_bytes * 2: - print( - f"npu.prefill: WARNING free disk {free_bytes / 1e9:.1f} GB may be tight " - f"for save (~{approx_save_bytes / 1e9:.1f} GB needed, x2 during copy)." - ) - except Exception: - pass - try: - mlmodel.save(str(out_path)) - except Exception as exc: - print(f"npu.prefill: mlpackage save failed ({type(exc).__name__}: {exc})") - return None - - print(f"npu.prefill: wrote {out_path} (chunk={chunk_size}, hidden={hidden_dim}, layers={num_layers})") - return filename From 32cc372639b2469f36f055783e1a89b4a5f78216 Mon Sep 17 00:00:00 2001 From: justinl66 Date: Wed, 27 May 2026 21:00:56 -0700 Subject: [PATCH 5/8] Add new npu quantization flags, new default quant bits --- python/cactus/cli/__init__.py | 8 +++-- python/cactus/cli/convert.py | 2 ++ python/cactus/cli/model.py | 6 ++++ python/cactus/transpile/hf_model.py | 24 +++++++++++-- python/cactus/transpile/npu/README.md | 2 +- python/cactus/transpile/npu/pipeline.py | 45 ++++++++++++++++++++----- 6 files changed, 73 insertions(+), 14 deletions(-) diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 1d411cf75..502e74f9d 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -262,8 +262,12 @@ def create_parser(): help="Force conversion from source") convert_parser.add_argument("--npu", action="store_true", help="Also emit CoreML .mlpackage(s) for NPU (Apple Neural Engine) audio + vision encoders") - convert_parser.add_argument("--npu-quantize", type=int, choices=[0, 4, 8], default=4, - help="Quantize .mlpackage weights (0=off, 4=int4 default, 8=int8). Smaller weights = faster on ANE + fits saved .mlpackage in less disk.") + convert_parser.add_argument("--npu-quantize", type=int, choices=[0, 4, 8], default=None, + help="Legacy override that forces BOTH audio and vision .mlpackages to the same quant (0=fp16, 4=int4, 8=int8). When unset, per-component defaults apply: audio=int8, vision=fp16.") + convert_parser.add_argument("--npu-audio-quantize", type=int, choices=[0, 4, 8], default=None, + help="Audio encoder weight quant (0=fp16, 4=int4, 8=int8). Default int8.") + convert_parser.add_argument("--npu-vision-quantize", type=int, choices=[0, 4, 8], default=None, + help="Vision encoder weight quant (0=fp16, 4=int4, 8=int8). Default fp16 — int4 is known to degrade Gemma 4 vision output.") return parser diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index 428c0e8bc..2eaef3c40 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -28,6 +28,8 @@ def cmd_convert(args): local_files_only=args.local_files_only, npu=getattr(args, "npu", False), npu_quantize=getattr(args, "npu_quantize", None), + npu_audio_quantize=getattr(args, "npu_audio_quantize", None), + npu_vision_quantize=getattr(args, "npu_vision_quantize", None), ), ) return 0 diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index e7a2989c2..f5e90ac94 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -202,6 +202,8 @@ class TranspileOptions: local_files_only: bool = False npu: bool = False npu_quantize: int | None = None + npu_audio_quantize: int | None = None + npu_vision_quantize: int | None = None def ensure_bundle(model_id, *, bits=4, token=None, @@ -318,6 +320,10 @@ def ensure_bundle(model_id, *, bits=4, token=None, extra_args.append("--npu") if opts.npu_quantize is not None: extra_args.extend(["--npu-quantize", str(int(opts.npu_quantize))]) + if opts.npu_audio_quantize is not None: + extra_args.extend(["--npu-audio-quantize", str(int(opts.npu_audio_quantize))]) + if opts.npu_vision_quantize is not None: + extra_args.extend(["--npu-vision-quantize", str(int(opts.npu_vision_quantize))]) rc = run_transpile(model_id, extra_args=extra_args) if rc != 0: diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index a3ad712ae..7b381b7ba 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -549,7 +549,9 @@ def _run_component_pipeline_transpile( # per-encoder coremltools temp, not HF model + all captured graphs. # Encoders only — text prefill is intentionally not on NPU. npu_enabled = bool(getattr(args, "npu", False)) - npu_quantize = getattr(args, "npu_quantize", None) + npu_quantize = getattr(args, "npu_quantize", None) # legacy override + npu_audio_quantize = getattr(args, "npu_audio_quantize", None) # default 8 if neither set + npu_vision_quantize = getattr(args, "npu_vision_quantize", None) # default 0 (fp16) if neither set npu_encoder_mlpackages: dict[str, str] = {} if npu_enabled and artifact_dir is not None: from .npu import run_encoder_pipeline @@ -558,6 +560,8 @@ def _run_component_pipeline_transpile( artifact_dir, enabled=True, quantize_bits=npu_quantize, + audio_quantize_bits=npu_audio_quantize, + vision_quantize_bits=npu_vision_quantize, ) # Drop coremltools-side intermediates before the heavy cactus capture # below — both phases compete for the same working set on a 16 GB host. @@ -2968,9 +2972,23 @@ def main() -> int: parser.add_argument( "--npu-quantize", type=int, - default=4, + default=None, + choices=[0, 4, 8], + help="Legacy override that forces BOTH audio and vision encoders to the same weight quant (0=fp16, 4=int4, 8=int8). When unset, per-component defaults apply: audio=int8, vision=fp16.", + ) + parser.add_argument( + "--npu-audio-quantize", + type=int, + default=None, + choices=[0, 4, 8], + help="Quantization for the audio encoder .mlpackage (0=fp16, 4=int4, 8=int8). Default int8: Conformer-style encoders absorb int8 quant noise well.", + ) + parser.add_argument( + "--npu-vision-quantize", + type=int, + default=None, choices=[0, 4, 8], - help="Post-conversion weight quantization for the .mlpackage (0=off, 4=int4 default, 8=int8). Smaller weights = faster on ANE and smaller .mlpackage on disk.", + help="Quantization for the vision encoder .mlpackage (0=fp16, 4=int4, 8=int8). Default fp16: ViT-style towers are small enough that fp16 is the safer correctness default — int4 visibly degrades on Gemma 4 vision.", ) parser.add_argument("--no-fuse-gated-deltanet", action="store_true") parser.add_argument("--no-fuse-rms-norm", action="store_true") diff --git a/python/cactus/transpile/npu/README.md b/python/cactus/transpile/npu/README.md index c853a5725..40057fb42 100644 --- a/python/cactus/transpile/npu/README.md +++ b/python/cactus/transpile/npu/README.md @@ -25,7 +25,7 @@ the ANE requires a static single-input signature. | Parakeet TDT audio encoder | **yes — verified end-to-end** | 4.5× faster warm than CPU on M-series (~155ms vs ~693ms TTFT). Use `CACTUS_ANE_AUDIO_COMPUTE_UNITS=ALL`. | | LFM-VL vision encoder | **yes — verified end-to-end** | ~2× faster on image-specific TTFT overhead, ~4× faster on the vision encoder graph alone (the rest of TTFT is preprocessing + LM prefill, CPU-bound). | | Gemma 4 audio + vision encoder emit | **emit verified (int8); mlpackage output matches HF reference** | Vision int8: cos_sim 0.9995 vs HF reference (MAE 0.018). Audio int8: cos_sim 0.9963 (MAE 0.0055 after applying the cactus `1/16` audio rescale, see `_gemma4_compute_native_like_audio_features`). Vision: 162 MB; audio: 282 MB. C++ runtime dispatch path is the same code that ships Parakeet ASR + LFM-VL chat — exercised end-to-end on those models in this session. Full Gemma 4 cactus runtime end-to-end requires a v2 bundle (`cactus convert`, ~15–20 GB peak), which doesn't fit on the 16 GB host where these mlpackages were produced. | -| `coremltools.optimize.linear_quantize_weights` | works | `--npu-quantize 4` / `--npu-quantize 8` post-conversion. int8 recommended for encoders. | +| `coremltools.optimize.linear_quantize_weights` | works | Per-component defaults: **audio int8, vision fp16**. ANE compute is fp16 internally either way; quant only affects weight storage + memory bandwidth. Audio encoders (Conformer-style) absorb int8 quant noise well — verified on Parakeet + Gemma 4. Vision encoders (ViT-style: SigLIP2, Gemma 4 vision) are more sensitive — int4 visibly corrupts Gemma 4 vision output; fp16 is the rock-solid default given the towers are small (one-shot per request, bandwidth not critical). Override via `--npu-quantize` (forces both) or `--npu-audio-quantize` / `--npu-vision-quantize` (per-component). | ## Why not NPU text prefill? diff --git a/python/cactus/transpile/npu/pipeline.py b/python/cactus/transpile/npu/pipeline.py index 471410edb..3e6d4946a 100644 --- a/python/cactus/transpile/npu/pipeline.py +++ b/python/cactus/transpile/npu/pipeline.py @@ -29,14 +29,31 @@ def run_encoder_pipeline( artifact_dir: Path, *, enabled: bool = True, - quantize_bits: int | None = None, + quantize_bits: int | None = None, # legacy override: forces both + audio_quantize_bits: int | None = None, # per-component, defaults to int8 + vision_quantize_bits: int | None = None, # per-component, defaults to fp16 ) -> dict[str, str]: """Emit audio / vision encoder `.mlpackage`s from captured component specs. - Reuses the exact adapter modules + example inputs the graph transpiler - captured (``ComponentModuleSpec``), so the NPU encoder stays in lockstep - with the CPU component graph. Auxiliary adapter inputs (masks, position - ids) are baked into the exported model as constants. + Per-component quantization defaults — set independently because the two + encoder families have very different sensitivity to weight quant: + + - **Audio** (Conformer-style: Parakeet, Gemma 4 audio): wide hidden + dims, conv subsampling that averages noise across many input frames, + long sequences. int8 holds up cleanly (verified cos_sim 0.9963 vs HF + on Gemma 4); even int4 generally works. + - **Vision** (ViT-style: LFM-VL SigLIP2, Gemma 4 vision): narrow + attention heads, one patch-embedding conv is a single point of + failure, short sequence (256-1024 tokens). int4 visibly degrades + outputs; int8 is fine but fp16 is the rock-solid default since the + vision tower is small (one-shot per request, bandwidth not critical). + + Quantization knob is per-component (``audio_quantize_bits`` / + ``vision_quantize_bits``). The legacy ``quantize_bits`` arg, when set, + overrides BOTH per-component defaults — preserves prior behavior where + one ``--npu-quantize 4`` flag controlled the whole conversion. + + Defaults: audio → int8, vision → fp16. ``0`` means fp16 (no quant). Returns a dict mapping ``manifest.json`` key (``npu_audio_encoder`` / ``npu_vision_encoder``) to the manifest-relative `.mlpackage` path. @@ -48,9 +65,18 @@ def run_encoder_pipeline( bundle_root = artifact_dir / "components" bundle_root.mkdir(parents=True, exist_ok=True) - qbits: int | None = quantize_bits - if qbits == 0: - qbits = None + # Resolve final quant per component. Per-component arg wins if given; + # else the legacy `quantize_bits` override; else per-component defaults. + def _resolve(per_component: int | None, default: int | None) -> int | None: + chosen = per_component if per_component is not None else ( + quantize_bits if quantize_bits is not None else default + ) + return None if chosen == 0 else chosen + + component_quants = { + "audio_encoder": _resolve(audio_quantize_bits, default=8), + "vision_encoder": _resolve(vision_quantize_bits, default=None), # default fp16 + } for spec in component_specs: component = getattr(spec, "component", None) @@ -63,6 +89,9 @@ def run_encoder_pipeline( continue primary = example_inputs[0] baked = example_inputs[1:] + qbits = component_quants[component] + qdesc = f"int{qbits}" if qbits else "fp16" + print(f"npu.pipeline: emitting {component} quant={qdesc}") emitted = emit_fn( spec.module, bundle_root, From 9ecca514e000b63dd2516838dc480b5a154520fa Mon Sep 17 00:00:00 2001 From: justinl66 Date: Thu, 28 May 2026 12:21:19 -0700 Subject: [PATCH 6/8] Cleanup NPU work --- .gitignore | 6 - local_integration_tests/CMakeLists.txt | 62 - local_integration_tests/README.md | 43 - local_integration_tests/conftest.py | 156 --- local_integration_tests/native_runner.cpp | 1114 ----------------- local_integration_tests/test_modalities.py | 210 ---- .../gemma4_multimodal/BACKGROUND.md | 53 - .../gemma4_multimodal/EXPERIMENTS.md | 80 -- .../gemma4_multimodal/FIXES.md | 52 - .../gemma4_multimodal/HYPOTHESES.md | 71 -- .../gemma4_multimodal/PLAN.md | 29 - .../gemma4_multimodal/README.md | 23 - .../gemma4_multimodal/SUMMARY.md | 9 - .../gemma4_multimodal_v2/00_INDEX.md | 48 - .../gemma4_multimodal_v2/01_BACKGROUND.md | 48 - .../gemma4_multimodal_v2/02_QUESTIONS.md | 41 - .../gemma4_multimodal_v2/03_HYPOTHESES.md | 66 - .../gemma4_multimodal_v2/04_EXPERIMENTS.md | 356 ------ .../gemma4_multimodal_v2/05_DECISIONS.md | 118 -- .../gemma4_multimodal_v2/06_FIX_LOG.md | 156 --- .../gemma4_multimodal_v2/07_ACCEPTANCE.md | 50 - .../gemma4_multimodal_v2/08_HANDOFF.md | 44 - .../09_STATUS_FOR_NEXT_AGENT.md | 74 -- .../10_CONTINUATION_PLAN.md | 58 - .../gemma4_multimodal_v2/scripts/run_probe.py | 785 ------------ 25 files changed, 3752 deletions(-) delete mode 100644 local_integration_tests/CMakeLists.txt delete mode 100644 local_integration_tests/README.md delete mode 100644 local_integration_tests/conftest.py delete mode 100644 local_integration_tests/native_runner.cpp delete mode 100644 local_integration_tests/test_modalities.py delete mode 100644 local_investigations/gemma4_multimodal/BACKGROUND.md delete mode 100644 local_investigations/gemma4_multimodal/EXPERIMENTS.md delete mode 100644 local_investigations/gemma4_multimodal/FIXES.md delete mode 100644 local_investigations/gemma4_multimodal/HYPOTHESES.md delete mode 100644 local_investigations/gemma4_multimodal/PLAN.md delete mode 100644 local_investigations/gemma4_multimodal/README.md delete mode 100644 local_investigations/gemma4_multimodal/SUMMARY.md delete mode 100644 local_investigations/gemma4_multimodal_v2/00_INDEX.md delete mode 100644 local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md delete mode 100644 local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md delete mode 100644 local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md delete mode 100644 local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md delete mode 100644 local_investigations/gemma4_multimodal_v2/05_DECISIONS.md delete mode 100644 local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md delete mode 100644 local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md delete mode 100644 local_investigations/gemma4_multimodal_v2/08_HANDOFF.md delete mode 100644 local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md delete mode 100644 local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md delete mode 100644 local_investigations/gemma4_multimodal_v2/scripts/run_probe.py diff --git a/.gitignore b/.gitignore index b93712487..e0918fde3 100644 --- a/.gitignore +++ b/.gitignore @@ -50,9 +50,3 @@ vision_activations.json gemma-4-eap-extras/ .githooks/ ruy - -# Local integration/investigation outputs -local_integration_tests/.tmp/ -local_integration_tests/__pycache__/ -local_investigations/**/__pycache__/ -local_investigations/**/artifacts/ diff --git a/local_integration_tests/CMakeLists.txt b/local_integration_tests/CMakeLists.txt deleted file mode 100644 index 317318503..000000000 --- a/local_integration_tests/CMakeLists.txt +++ /dev/null @@ -1,62 +0,0 @@ -cmake_minimum_required(VERSION 3.16) -project(CactusLocalIntegrationTests LANGUAGES CXX) - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED True) - -if(NOT CACTUS_REPO_ROOT) - get_filename_component(CACTUS_REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.." ABSOLUTE) -endif() - -set(CACTUS_LIB "${CACTUS_REPO_ROOT}/cactus/build/libcactus.a") -if(NOT EXISTS "${CACTUS_LIB}") - message(FATAL_ERROR "Missing ${CACTUS_LIB}; run ./cactus/build.sh first") -endif() - -add_executable(integration_runner native_runner.cpp) -target_link_libraries(integration_runner PRIVATE "${CACTUS_LIB}") -target_include_directories(integration_runner PRIVATE - "${CACTUS_REPO_ROOT}/cactus-engine" - "${CACTUS_REPO_ROOT}/cactus-engine/src" - "${CACTUS_REPO_ROOT}/cactus-graph" - "${CACTUS_REPO_ROOT}/cactus-kernels" - "${CACTUS_REPO_ROOT}/cactus-kernels/src" - "${CACTUS_REPO_ROOT}/cactus-kernels/libs" -) - -find_package(Threads REQUIRED) -target_link_libraries(integration_runner PRIVATE Threads::Threads) - -if(APPLE) - find_library(COREML_FRAMEWORK CoreML REQUIRED) - find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) - find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) - find_library(SECURITY_FRAMEWORK Security REQUIRED) - find_library(SYSTEMCONFIGURATION_FRAMEWORK SystemConfiguration REQUIRED) - find_library(CFNETWORK_FRAMEWORK CFNetwork REQUIRED) - target_link_libraries(integration_runner PRIVATE - "${ACCELERATE_FRAMEWORK}" - "${COREML_FRAMEWORK}" - "${FOUNDATION_FRAMEWORK}" - "${SECURITY_FRAMEWORK}" - "${SYSTEMCONFIGURATION_FRAMEWORK}" - "${CFNETWORK_FRAMEWORK}" - ) - set(CACTUS_CURL_ROOT "${CACTUS_REPO_ROOT}/cactus-engine/libs/curl") - set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") - if(EXISTS "${CACTUS_CURL_LIBRARY_MACOS}") - target_link_libraries(integration_runner PRIVATE "${CACTUS_CURL_LIBRARY_MACOS}") - target_include_directories(integration_runner PRIVATE "${CACTUS_CURL_ROOT}/include") - else() - find_library(CURL_LIBRARY NAMES curl) - if(CURL_LIBRARY) - target_link_libraries(integration_runner PRIVATE "${CURL_LIBRARY}") - endif() - endif() -else() - find_package(CURL) - if(CURL_FOUND) - target_link_libraries(integration_runner PRIVATE ${CURL_LIBRARIES}) - target_include_directories(integration_runner PRIVATE ${CURL_INCLUDE_DIRS}) - endif() -endif() diff --git a/local_integration_tests/README.md b/local_integration_tests/README.md deleted file mode 100644 index c6442606a..000000000 --- a/local_integration_tests/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Local integration tests - -These tests are intentionally local and uncommitted until the suite is ready to become part of the repository. - -## Requirements - -- Run from the repository root. -- Use existing converted model artifacts only. -- Set `CACTUS_INTEGRATION_WEIGHTS` to override model discovery. -- Default discovery checks `../cactus/weights`, then `./weights`. - -## Commands - -```bash -PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -m smoke -PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -m full -PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -k parakeet -``` - -The pytest session builds `./cactus/build.sh` once and compiles `local_integration_tests/.tmp/build/integration_runner`. - -## Coverage - -- Gemma4 E2B: text and multiturn reuse with the matrix Cactus artifact, `gemma-4-e2b-it-sharedkv`. -- Gemma4 E2B: image, audio, and image+audio combined with the latest local multimodal artifact, preferring `gemma-4-e2b-it-alt-fixed-rowtuned-mm`. -- Qwen3-VL: text and long-prompt chunked prefill smoke. -- Qwen3-VL: image with the matrix Cactus artifact, `qwen3-vl-2b-instruct-reconvert`. -- LFM2-VL: text, image, long-prompt chunked prefill smoke. -- Parakeet TDT: file and PCM transcription. -- Whisper small: full-mode file and PCM transcription when a component-backed local bundle is available. - -## Current local result - -```bash -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s -# 7 passed, 2 deselected - -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -k parakeet -q -s -# 1 passed, 8 deselected - -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m full -q -s -# 2 skipped, 7 deselected -``` diff --git a/local_integration_tests/conftest.py b/local_integration_tests/conftest.py deleted file mode 100644 index adcfe06a4..000000000 --- a/local_integration_tests/conftest.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -import json -import os -import subprocess -from pathlib import Path -from typing import Any - -import pytest - - -REPO_ROOT = Path(__file__).resolve().parents[1] -TMP_DIR = REPO_ROOT / "local_integration_tests" / ".tmp" -ASSET_DIR = REPO_ROOT / "cactus-engine" / "tests" / "assets" - - -def pytest_configure(config: pytest.Config) -> None: - config.addinivalue_line("markers", "smoke: fast local integration coverage") - config.addinivalue_line("markers", "full: slower coverage for every available key model") - - -def _weights_root() -> Path: - explicit = os.environ.get("CACTUS_INTEGRATION_WEIGHTS") - candidates = [explicit] if explicit else [] - candidates.extend([str(REPO_ROOT.parent / "cactus" / "weights"), str(REPO_ROOT / "weights")]) - for candidate in candidates: - if not candidate: - continue - path = Path(candidate).expanduser().resolve() - if path.exists(): - return path - return Path(candidates[0]).expanduser().resolve() - - -@pytest.fixture(scope="session") -def weights_root() -> Path: - root = _weights_root() - if not root.exists(): - pytest.fail(f"weights root does not exist: {root}") - return root - - -@pytest.fixture(scope="session") -def assets() -> dict[str, Path]: - result = { - "audio": ASSET_DIR / "test.wav", - "image_monkey": ASSET_DIR / "test_monkey.png", - "image_thing": ASSET_DIR / "test_thing.png", - } - missing = [str(path) for path in result.values() if not path.exists()] - if missing: - pytest.fail("missing integration fixtures: " + ", ".join(missing)) - return result - - -@pytest.fixture(scope="session") -def runner() -> Path: - env = os.environ.copy() - env["CACTUS_NO_CLOUD_TELE"] = "1" - subprocess.run([str(REPO_ROOT / "cactus" / "build.sh")], cwd=REPO_ROOT, env=env, check=True) - - build_dir = TMP_DIR / "build" - build_dir.mkdir(parents=True, exist_ok=True) - subprocess.run( - [ - "cmake", - "-S", - str(REPO_ROOT / "local_integration_tests"), - "-B", - str(build_dir), - f"-DCACTUS_REPO_ROOT={REPO_ROOT}", - "-DCMAKE_BUILD_TYPE=Release", - ], - cwd=REPO_ROOT, - check=True, - ) - subprocess.run( - ["cmake", "--build", str(build_dir), "--target", "integration_runner", "-j", str(os.cpu_count() or 4)], - cwd=REPO_ROOT, - check=True, - ) - binary = build_dir / "integration_runner" - if not binary.exists(): - pytest.fail(f"integration runner was not built: {binary}") - return binary - - -def _is_transpiled_bundle(path: Path) -> bool: - return (path / "components" / "manifest.json").exists() or (path / "manifest.json").exists() - - -def model_path( - weights_root: Path, - *candidates: str, - required: bool = True, - transpiled: bool = True, - native_config: bool = False, -) -> Path | None: - for name in candidates: - path = weights_root / name - if not path.exists(): - continue - if transpiled and not _is_transpiled_bundle(path): - continue - if native_config and not (path / "config.txt").exists(): - continue - if path.exists(): - return path - if required: - suffixes = [] - if transpiled: - suffixes.append("components/manifest.json") - if native_config: - suffixes.append("config.txt") - suffix = " with " + " and ".join(suffixes) if suffixes else "" - pytest.fail("missing required model artifact" + suffix + "; tried: " + ", ".join(candidates)) - pytest.skip("optional model artifact not available; tried: " + ", ".join(candidates)) - return None - - -def run_runner(runner: Path, *args: object, timeout: int = 180) -> dict[str, Any]: - env = os.environ.copy() - env["CACTUS_NO_CLOUD_TELE"] = "1" - env.pop("CACTUS_KV_CACHE_FP16", None) - completed = subprocess.run( - [str(runner), *(str(arg) for arg in args)], - cwd=REPO_ROOT, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - check=False, - ) - if completed.returncode != 0: - pytest.fail( - f"integration runner failed with {completed.returncode}\n" - f"stdout:\n{completed.stdout}\n" - f"stderr:\n{completed.stderr}" - ) - try: - return json.loads(completed.stdout) - except json.JSONDecodeError as exc: - pytest.fail(f"runner did not return JSON: {exc}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}") - - -def assert_completion(result: dict[str, Any]) -> None: - response = str(result.get("response") or result.get("text") or "").strip() - token_count = int(result.get("completion_tokens") or result.get("tokens_predicted") or result.get("decode_tokens") or 0) - assert response or token_count > 0, result - - -def assert_transcript(result: dict[str, Any], fragments: tuple[str, ...] = ("hello", "testing", "voice")) -> None: - transcript = str(result.get("response") or result.get("transcript") or "").strip().lower() - assert len(transcript) > 5, result - assert any(fragment in transcript for fragment in fragments), transcript diff --git a/local_integration_tests/native_runner.cpp b/local_integration_tests/native_runner.cpp deleted file mode 100644 index 11cf3ef65..000000000 --- a/local_integration_tests/native_runner.cpp +++ /dev/null @@ -1,1114 +0,0 @@ -#include "cactus_engine.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define private public -#include "engine.h" -#undef private -#include "cactus_kernels.h" -#include "stb_image_resize2.h" -#include "utils.h" -#include "wav.h" - -namespace { - -constexpr size_t kResponseBufferSize = 1 << 20; -constexpr uint64_t kFnvOffset = 1469598103934665603ull; -constexpr uint64_t kFnvPrime = 1099511628211ull; - -struct Args { - std::string command; - std::string model; - std::string prompt; - std::string image; - std::string audio; - std::string messages_json; - int max_tokens = 32; -}; - -std::string arg_value(int& i, int argc, char** argv) { - if (i + 1 >= argc) throw std::runtime_error(std::string("missing value for ") + argv[i]); - return argv[++i]; -} - -Args parse_args(int argc, char** argv) { - if (argc < 2) throw std::runtime_error("usage: integration_runner [options]"); - Args args; - args.command = argv[1]; - for (int i = 2; i < argc; ++i) { - std::string flag = argv[i]; - if (flag == "--model") args.model = arg_value(i, argc, argv); - else if (flag == "--prompt") args.prompt = arg_value(i, argc, argv); - else if (flag == "--image") args.image = arg_value(i, argc, argv); - else if (flag == "--audio") args.audio = arg_value(i, argc, argv); - else if (flag == "--messages-json") args.messages_json = arg_value(i, argc, argv); - else if (flag == "--max-tokens") args.max_tokens = std::max(0, std::stoi(arg_value(i, argc, argv))); - else throw std::runtime_error("unknown flag: " + flag); - } - if (args.model.empty()) throw std::runtime_error("--model is required"); - return args; -} - -std::string escape_json(const std::string& value) { - std::ostringstream out; - for (char ch : value) { - switch (ch) { - case '\\': out << "\\\\"; break; - case '"': out << "\\\""; break; - case '\n': out << "\\n"; break; - case '\r': out << "\\r"; break; - case '\t': out << "\\t"; break; - default: - if (static_cast(ch) < 0x20) { - out << "\\u00"; - const char* hex = "0123456789abcdef"; - out << hex[(ch >> 4) & 0x0f] << hex[ch & 0x0f]; - } else { - out << ch; - } - } - } - return out.str(); -} - -template -std::string json_array(const std::vector& values, size_t limit = std::numeric_limits::max()) { - std::ostringstream out; - out << "["; - const size_t n = std::min(values.size(), limit); - for (size_t i = 0; i < n; ++i) { - if (i) out << ","; - out << values[i]; - } - out << "]"; - return out.str(); -} - -std::string json_float_array(const std::vector& values, size_t limit = std::numeric_limits::max()) { - std::ostringstream out; - out << "["; - const size_t n = std::min(values.size(), limit); - for (size_t i = 0; i < n; ++i) { - if (i) out << ","; - if (std::isfinite(values[i])) out << values[i]; - else out << "null"; - } - out << "]"; - return out.str(); -} - -uint64_t fnv1a_bytes(const void* data, size_t size) { - const auto* bytes = static_cast(data); - uint64_t hash = kFnvOffset; - for (size_t i = 0; i < size; ++i) { - hash ^= static_cast(bytes[i]); - hash *= kFnvPrime; - } - return hash; -} - -std::string hex64(uint64_t value) { - std::ostringstream out; - out << std::hex << std::setw(16) << std::setfill('0') << value; - return out.str(); -} - -template -std::vector first_values(const std::vector& values, size_t n) { - return std::vector(values.begin(), values.begin() + std::min(values.size(), n)); -} - -std::vector last_values(const std::vector& values, size_t n) { - if (values.size() <= n) return values; - return std::vector(values.end() - static_cast(n), values.end()); -} - -int run_e002_tokens(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for token probe: " + args.model); - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() ? "Respond briefly." : args.prompt; - message.images.push_back(args.image); - std::vector messages{message}; - - const std::string full_prompt = tokenizer->format_chat_prompt(messages, true); - const std::string no_generation_prompt = tokenizer->format_chat_prompt(messages, false); - const std::vector tokens = tokenizer->encode(full_prompt); - const std::vector prefix_tokens = tokenizer->encode(no_generation_prompt); - const uint32_t image_token_id = model->get_config().image_token_id ? model->get_config().image_token_id : tokenizer->get_image_token_id(); - - size_t image_start = tokens.size(); - size_t image_count = 0; - for (size_t i = 0; i < tokens.size(); ++i) { - if (tokens[i] == image_token_id) { - if (image_count == 0) image_start = i; - ++image_count; - } - } - - std::ostringstream out; - out << "{" - << "\"token_count\":" << tokens.size() << "," - << "\"assistant_generation_start\":" << prefix_tokens.size() << "," - << "\"image_token_id\":" << image_token_id << "," - << "\"image_token_start\":" << (image_count ? static_cast(image_start) : -1) << "," - << "\"image_token_count\":" << image_count << "," - << "\"first_20\":" << json_array(first_values(tokens, 20)) << "," - << "\"last_20\":" << json_array(last_values(tokens, 20)) << "," - << "\"input_ids\":" << json_array(tokens) << "," - << "\"prompt_prefix\":\"" << escape_json(full_prompt.substr(0, 320)) << "\"" - << "}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_preprocess(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - cactus::engine::Config config; - if (!config.from_json(args.model + "/config.txt")) { - throw std::runtime_error("failed to read config.txt for preprocess probe: " + args.model); - } - auto prep = cactus::engine::preprocess_gemma4_image(args.image, config); - double sum = 0.0; - float min_v = prep.pixel_values.empty() ? 0.0f : prep.pixel_values[0]; - float max_v = min_v; - for (float v : prep.pixel_values) { - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - size_t valid_positions = 0; - for (size_t i = 0; i + 1 < prep.pixel_position_ids.size(); i += 2) { - if (prep.pixel_position_ids[i] >= 0 && prep.pixel_position_ids[i + 1] >= 0) ++valid_positions; - } - std::vector first_positions; - const size_t position_rows = std::min(16, prep.pixel_position_ids.size() / 2); - first_positions.reserve(position_rows * 2); - for (size_t i = 0; i < position_rows * 2; ++i) first_positions.push_back(prep.pixel_position_ids[i]); - - std::ostringstream out; - out << std::setprecision(9) - << "{" - << "\"pixel_values_shape\":[1," << prep.max_patches << "," << prep.patch_dim << "]," - << "\"pixel_position_ids_shape\":[1," << prep.max_patches << ",2]," - << "\"num_patches\":" << prep.num_patches << "," - << "\"max_patches\":" << prep.max_patches << "," - << "\"patch_dim\":" << prep.patch_dim << "," - << "\"pixel_values_min\":" << min_v << "," - << "\"pixel_values_max\":" << max_v << "," - << "\"pixel_values_mean\":" << (prep.pixel_values.empty() ? 0.0 : sum / static_cast(prep.pixel_values.size())) << "," - << "\"pixel_values_hash\":\"" << hex64(fnv1a_bytes(prep.pixel_values.data(), prep.pixel_values.size() * sizeof(float))) << "\"," - << "\"pixel_position_ids_hash\":\"" << hex64(fnv1a_bytes(prep.pixel_position_ids.data(), prep.pixel_position_ids.size() * sizeof(int64_t))) << "\"," - << "\"valid_position_count\":" << valid_positions << "," - << "\"first_pixel_values\":" << json_array(first_values(prep.pixel_values, 32)) << "," - << "\"first_position_rows_flat\":" << json_array(first_positions) << "," - << "\"pixel_values\":" << json_array(prep.pixel_values) << "," - << "\"pixel_position_ids\":" << json_array(prep.pixel_position_ids) - << "}\n"; - std::cout << out.str(); - return 0; -} - -struct E003Variant { - std::string name; - std::vector resized; - std::vector pixel_values; -}; - -void resize_float_default(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { - std::vector src_float(static_cast(width) * height * 3); - for (size_t i = 0; i < src_float.size(); ++i) src_float[i] = static_cast(raw[i]); - if (!stbir_resize_float_linear(src_float.data(), width, height, 0, resized.data(), target_w, target_h, 0, STBIR_RGB)) { - throw std::runtime_error("stbir_resize_float_linear failed"); - } -} - -void resize_uint8_default(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { - std::vector tmp(static_cast(target_w) * target_h * 3); - if (!stbir_resize_uint8_linear(raw, width, height, 0, tmp.data(), target_w, target_h, 0, STBIR_RGB)) { - throw std::runtime_error("stbir_resize_uint8_linear failed"); - } - for (size_t i = 0; i < tmp.size(); ++i) resized[i] = static_cast(tmp[i]); -} - -void resize_float_triangle(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { - std::vector src_float(static_cast(width) * height * 3); - for (size_t i = 0; i < src_float.size(); ++i) src_float[i] = static_cast(raw[i]); - STBIR_RESIZE resize; - stbir_resize_init(&resize, src_float.data(), width, height, 0, resized.data(), target_w, target_h, 0, STBIR_RGB, STBIR_TYPE_FLOAT); - stbir_set_filters(&resize, STBIR_FILTER_TRIANGLE, STBIR_FILTER_TRIANGLE); - if (!stbir_resize_extended(&resize)) throw std::runtime_error("triangle float resize failed"); -} - -void resize_uint8_triangle(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { - std::vector tmp(static_cast(target_w) * target_h * 3); - STBIR_RESIZE resize; - stbir_resize_init(&resize, raw, width, height, 0, tmp.data(), target_w, target_h, 0, STBIR_RGB, STBIR_TYPE_UINT8); - stbir_set_filters(&resize, STBIR_FILTER_TRIANGLE, STBIR_FILTER_TRIANGLE); - if (!stbir_resize_extended(&resize)) throw std::runtime_error("triangle uint8 resize failed"); - for (size_t i = 0; i < tmp.size(); ++i) resized[i] = static_cast(tmp[i]); -} - -std::vector pack_gemma4_patches(const std::vector& resized, int target_w, int target_h, int patch_size, - size_t max_patches, size_t patch_dim, float rescale_factor) { - const int patch_h = target_h / patch_size; - const int patch_w = target_w / patch_size; - std::vector pixel_values(max_patches * patch_dim, 0.0f); - for (int py = 0; py < patch_h; ++py) { - for (int px = 0; px < patch_w; ++px) { - const size_t patch_idx = static_cast(py) * patch_w + px; - float* dst = pixel_values.data() + patch_idx * patch_dim; - for (int y = 0; y < patch_size; ++y) { - const int img_y = py * patch_size + y; - for (int x = 0; x < patch_size; ++x) { - const int img_x = px * patch_size + x; - const size_t src_off = (static_cast(img_y) * target_w + img_x) * 3; - const size_t dst_off = (static_cast(y) * patch_size + x) * 3; - dst[dst_off + 0] = resized[src_off + 0] * rescale_factor; - dst[dst_off + 1] = resized[src_off + 1] * rescale_factor; - dst[dst_off + 2] = resized[src_off + 2] * rescale_factor; - } - } - } - } - return pixel_values; -} - -void write_variant_json(std::ostream& out, const E003Variant& variant, bool comma) { - double sum = 0.0; - float min_v = variant.pixel_values.empty() ? 0.0f : variant.pixel_values[0]; - float max_v = min_v; - for (float v : variant.pixel_values) { - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - float resized_min = variant.resized.empty() ? 0.0f : variant.resized[0]; - float resized_max = resized_min; - for (float v : variant.resized) { - resized_min = std::min(resized_min, v); - resized_max = std::max(resized_max, v); - } - if (comma) out << ","; - out << "\"" << variant.name << "\":{" - << "\"resized_min\":" << resized_min << "," - << "\"resized_max\":" << resized_max << "," - << "\"first_resized_values\":" << json_array(first_values(variant.resized, 32)) << "," - << "\"pixel_values_min\":" << min_v << "," - << "\"pixel_values_max\":" << max_v << "," - << "\"pixel_values_mean\":" << (variant.pixel_values.empty() ? 0.0 : sum / static_cast(variant.pixel_values.size())) << "," - << "\"first_pixel_values\":" << json_array(first_values(variant.pixel_values, 32)) << "," - << "\"pixel_values\":" << json_array(variant.pixel_values) - << "}"; -} - -std::string precision_name(Precision precision) { - switch (precision) { - case Precision::FP32: return "FP32"; - case Precision::FP16: return "FP16"; - case Precision::INT8: return "INT8"; - default: return "UNKNOWN"; - } -} - -std::string op_type_name(OpType op) { - switch (op) { - case OpType::INPUT: return "INPUT"; - case OpType::PRECISION_CAST: return "PRECISION_CAST"; - case OpType::ADD: return "ADD"; - case OpType::ADD_CLIPPED: return "ADD_CLIPPED"; - case OpType::SUBTRACT: return "SUBTRACT"; - case OpType::MULTIPLY: return "MULTIPLY"; - case OpType::DIVIDE: return "DIVIDE"; - case OpType::ABS: return "ABS"; - case OpType::POW: return "POW"; - case OpType::FLATTEN: return "FLATTEN"; - case OpType::VIEW: return "VIEW"; - case OpType::MATMUL: return "MATMUL"; - case OpType::TRANSPOSE: return "TRANSPOSE"; - case OpType::RESHAPE: return "RESHAPE"; - case OpType::SLICE: return "SLICE"; - case OpType::GATHER: return "GATHER"; - case OpType::EMBEDDING: return "EMBEDDING"; - case OpType::BILINEAR_INTERPOLATION: return "BILINEAR_INTERPOLATION"; - case OpType::SUM: return "SUM"; - case OpType::MEAN: return "MEAN"; - case OpType::VARIANCE: return "VARIANCE"; - case OpType::MIN: return "MIN"; - case OpType::MAX: return "MAX"; - case OpType::CUMSUM: return "CUMSUM"; - case OpType::RMS_NORM: return "RMS_NORM"; - case OpType::ROPE: return "ROPE"; - case OpType::ROPE_GPTJ: return "ROPE_GPTJ"; - case OpType::SOFTMAX: return "SOFTMAX"; - case OpType::ATTENTION: return "ATTENTION"; - case OpType::ATTENTION_INT8_HYBRID: return "ATTENTION_INT8_HYBRID"; - case OpType::REL_POS_BIAS: return "REL_POS_BIAS"; - case OpType::CONV1D_CAUSAL: return "CONV1D_CAUSAL"; - case OpType::CONV1D_K3: return "CONV1D_K3"; - case OpType::CONV1D_K7S3: return "CONV1D_K7S3"; - case OpType::CONV1D: return "CONV1D"; - case OpType::CONV1D_SAME_DEPTHWISE_K9: return "CONV1D_SAME_DEPTHWISE_K9"; - case OpType::CONV1D_POINTWISE: return "CONV1D_POINTWISE"; - case OpType::CONV2D_K3S2P1: return "CONV2D_K3S2P1"; - case OpType::CONV2D_DEPTHWISE_K3S2P1: return "CONV2D_DEPTHWISE_K3S2P1"; - case OpType::CONV2D_POINTWISE_1X1: return "CONV2D_POINTWISE_1X1"; - case OpType::GLU: return "GLU"; - case OpType::BATCHNORM: return "BATCHNORM"; - case OpType::SCALAR_ADD: return "SCALAR_ADD"; - case OpType::SCALAR_SUBTRACT: return "SCALAR_SUBTRACT"; - case OpType::SCALAR_MULTIPLY: return "SCALAR_MULTIPLY"; - case OpType::SCALAR_DIVIDE: return "SCALAR_DIVIDE"; - case OpType::SCALAR_EXP: return "SCALAR_EXP"; - case OpType::SCALAR_SQRT: return "SCALAR_SQRT"; - case OpType::SCALAR_COS: return "SCALAR_COS"; - case OpType::SCALAR_SIN: return "SCALAR_SIN"; - case OpType::SCALAR_LOG: return "SCALAR_LOG"; - case OpType::RELU: return "RELU"; - case OpType::SILU: return "SILU"; - case OpType::GELU: return "GELU"; - case OpType::GELU_ERF: return "GELU_ERF"; - case OpType::SIGMOID: return "SIGMOID"; - case OpType::TANH: return "TANH"; - case OpType::SAMPLE: return "SAMPLE"; - case OpType::CONCAT: return "CONCAT"; - case OpType::CAT: return "CAT"; - case OpType::SCATTER_TOPK: return "SCATTER_TOPK"; - case OpType::TOPK: return "TOPK"; - case OpType::LAYERNORM: return "LAYERNORM"; - case OpType::GROUPNORM: return "GROUPNORM"; - case OpType::MOE_LAYER: return "MOE_LAYER"; - case OpType::INDEX: return "INDEX"; - case OpType::PERSISTENT: return "PERSISTENT"; - case OpType::LSTM_CELL: return "LSTM_CELL"; - case OpType::GATED_DELTANET_DECODE: return "GATED_DELTANET_DECODE"; - case OpType::GATED_DELTANET_PREFILL: return "GATED_DELTANET_PREFILL"; - case OpType::STFT: return "STFT"; - case OpType::ALTUP_PREDICT: return "ALTUP_PREDICT"; - case OpType::ALTUP_CORRECT: return "ALTUP_CORRECT"; - case OpType::GAUSSIAN_TOPK: return "GAUSSIAN_TOPK"; - case OpType::MAXPOOL1D: return "MAXPOOL1D"; - case OpType::BILSTM_SEQUENCE: return "BILSTM_SEQUENCE"; - case OpType::LEAKY_RELU: return "LEAKY_RELU"; - case OpType::CONV2D_K3S1P1: return "CONV2D_K3S1P1"; - case OpType::STATS_POOL: return "STATS_POOL"; - case OpType::WEIGHTED_STATS_POOL: return "WEIGHTED_STATS_POOL"; - case OpType::KV_CACHE_STATE: return "KV_CACHE_STATE"; - case OpType::KV_CACHE_APPEND: return "KV_CACHE_APPEND"; - case OpType::ATTENTION_CACHED: return "ATTENTION_CACHED"; - case OpType::CONV_CACHE_STATE: return "CONV_CACHE_STATE"; - case OpType::CONV_CACHE_APPEND: return "CONV_CACHE_APPEND"; - case OpType::RFFT: return "RFFT"; - case OpType::IRFFT: return "IRFFT"; - case OpType::MEL_FILTER_BANK: return "MEL_FILTER_BANK"; - case OpType::SPECTROGRAM: return "SPECTROGRAM"; - case OpType::IMAGE_PREPROCESS: return "IMAGE_PREPROCESS"; - case OpType::CLAMP: return "CLAMP"; - case OpType::DENSE_MLP_TQ_FUSED: return "DENSE_MLP_TQ_FUSED"; - case OpType::NOT_EQUAL: return "NOT_EQUAL"; - case OpType::SCALAR_NOT_EQUAL: return "SCALAR_NOT_EQUAL"; - } - return "UNKNOWN"; -} - -std::vector bytes_to_floats(const std::vector& bytes, Precision precision) { - std::vector values; - if (precision == Precision::FP32) { - const size_t count = bytes.size() / sizeof(float); - values.resize(count); - const auto* src = reinterpret_cast(bytes.data()); - for (size_t i = 0; i < count; ++i) values[i] = src[i]; - } else if (precision == Precision::FP16) { - const size_t count = bytes.size() / sizeof(__fp16); - values.resize(count); - const auto* src = reinterpret_cast(bytes.data()); - for (size_t i = 0; i < count; ++i) values[i] = static_cast(src[i]); - } else if (precision == Precision::INT8) { - const size_t count = bytes.size(); - values.resize(count); - const auto* src = reinterpret_cast(bytes.data()); - for (size_t i = 0; i < count; ++i) values[i] = static_cast(src[i]); - } - return values; -} - -std::vector buffer_to_floats(const BufferDesc& desc) { - std::vector values; - const void* data = desc.get_data(); - if (!data) return values; - if (desc.precision == Precision::FP32) { - values.resize(desc.byte_size / sizeof(float)); - const auto* src = static_cast(data); - for (size_t i = 0; i < values.size(); ++i) values[i] = src[i]; - } else if (desc.precision == Precision::FP16) { - values.resize(desc.byte_size / sizeof(__fp16)); - const auto* src = static_cast(data); - for (size_t i = 0; i < values.size(); ++i) values[i] = static_cast(src[i]); - } else if (desc.precision == Precision::INT8) { - values.resize(desc.byte_size); - const auto* src = static_cast(data); - for (size_t i = 0; i < values.size(); ++i) values[i] = static_cast(src[i]); - } - return values; -} - -void write_float_feature_json(std::ostream& out, - const std::string& name, - const std::vector& values, - const std::vector& shape, - Precision precision, - bool comma) { - double sum = 0.0; - float min_v = values.empty() ? 0.0f : values[0]; - float max_v = min_v; - double l2 = 0.0; - size_t finite_count = 0; - size_t nonfinite_count = 0; - for (float v : values) { - if (!std::isfinite(v)) { - ++nonfinite_count; - continue; - } - if (finite_count == 0) { - min_v = v; - max_v = v; - } - ++finite_count; - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - l2 += static_cast(v) * static_cast(v); - } - if (comma) out << ","; - out << "\"" << escape_json(name) << "\":{" - << "\"shape\":" << json_array(shape) << "," - << "\"precision\":\"" << precision_name(precision) << "\"," - << "\"count\":" << values.size() << "," - << "\"finite_count\":" << finite_count << "," - << "\"nonfinite_count\":" << nonfinite_count << "," - << "\"min\":" << min_v << "," - << "\"max\":" << max_v << "," - << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," - << "\"l2\":" << std::sqrt(l2) << "," - << "\"first_values\":" << json_float_array(values, 32) << "," - << "\"values\":" << json_float_array(values) - << "}"; -} - -int run_e003_preprocess_variants(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - cactus::engine::Config config; - if (!config.from_json(args.model + "/config.txt")) { - throw std::runtime_error("failed to read config.txt for E003 probe: " + args.model); - } - - const int patch_size = static_cast(config.vision_patch_size ? config.vision_patch_size : 16); - const int pooling_k = static_cast(config.vision_pooling_kernel_size ? config.vision_pooling_kernel_size : 3); - const size_t max_patches = static_cast(config.vision_default_output_length ? config.vision_default_output_length : 280) * pooling_k * pooling_k; - const int side_multiple = pooling_k * patch_size; - const size_t patch_dim = static_cast(3) * patch_size * patch_size; - const float rescale_factor = config.rescale_factor > 0.0f ? config.rescale_factor : (1.0f / 255.0f); - - int width = 0, height = 0, channels = 0; - unsigned char* raw = cactus_image_load(args.image.c_str(), &width, &height, &channels, 3); - if (!raw) throw std::runtime_error("failed to load image for E003 probe: " + args.image); - - const double target_pixels = static_cast(max_patches) * patch_size * patch_size; - const double pixel_count = std::max(1.0, static_cast(width) * static_cast(height)); - const double factor = std::sqrt(target_pixels / pixel_count); - int target_h = static_cast(std::floor(factor * height / side_multiple)) * side_multiple; - int target_w = static_cast(std::floor(factor * width / side_multiple)) * side_multiple; - if (target_h == 0) target_h = side_multiple; - if (target_w == 0) target_w = side_multiple; - - std::vector variants; - for (const std::string& name : {"stb_default_float", "stb_default_uint8", "stb_triangle_float", "stb_triangle_uint8"}) { - E003Variant variant; - variant.name = name; - variant.resized.assign(static_cast(target_w) * target_h * 3, 0.0f); - if (name == "stb_default_float") resize_float_default(raw, width, height, variant.resized, target_w, target_h); - else if (name == "stb_default_uint8") resize_uint8_default(raw, width, height, variant.resized, target_w, target_h); - else if (name == "stb_triangle_float") resize_float_triangle(raw, width, height, variant.resized, target_w, target_h); - else resize_uint8_triangle(raw, width, height, variant.resized, target_w, target_h); - variant.pixel_values = pack_gemma4_patches(variant.resized, target_w, target_h, patch_size, max_patches, patch_dim, rescale_factor); - variants.push_back(std::move(variant)); - } - cactus_image_free(raw); - - std::ostringstream out; - out << std::setprecision(9) - << "{" - << "\"source_width\":" << width << "," - << "\"source_height\":" << height << "," - << "\"target_width\":" << target_w << "," - << "\"target_height\":" << target_h << "," - << "\"patch_size\":" << patch_size << "," - << "\"pooling_kernel_size\":" << pooling_k << "," - << "\"max_patches\":" << max_patches << "," - << "\"patch_dim\":" << patch_dim << "," - << "\"rescale_factor\":" << rescale_factor << "," - << "\"variants\":{"; - for (size_t i = 0; i < variants.size(); ++i) { - write_variant_json(out, variants[i], i > 0); - } - out << "}}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_vision(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for vision probe: " + args.model); - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() - ? "What animal is in this image? Reply in one complete short sentence." - : args.prompt; - message.images.push_back(args.image); - std::vector messages{message}; - std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); - if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { - throw std::runtime_error("chunk prefill path failed for vision probe"); - } - - std::ostringstream out; - out << std::setprecision(9) << "{\"inputs\":{"; - if (model->vision_encoder_) { - for (size_t i = 0; i < model->vision_encoder_->logical_inputs.size(); ++i) { - if (i) out << ","; - const std::string& input_name = model->vision_encoder_->logical_inputs[i]; - const size_t node_id = static_cast(model->vision_encoder_->runtime_input_node_ids[i]); - const auto& desc = model->vision_encoder_->graph->get_output_buffer(node_id); - out << "\"" << escape_json(input_name) << "\":{" - << "\"node_id\":" << node_id << "," - << "\"shape\":" << json_array(desc.shape) << "," - << "\"precision\":\"" << precision_name(desc.precision) << "\"," - << "\"byte_size\":" << desc.byte_size - << "}"; - } - } - out << "},\"features\":{"; - bool first = true; - for (const auto& kv : model->media_features_) { - const std::string& name = kv.first; - Precision precision = model->media_feature_precisions_[name]; - std::vector values = bytes_to_floats(kv.second, precision); - write_float_feature_json(out, name, values, model->media_feature_shapes_[name], precision, !first); - first = false; - } - out << "}}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_vision_node_summary(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for vision node summary: " + args.model); - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() - ? "What animal is in this image? Reply in one complete short sentence." - : args.prompt; - message.images.push_back(args.image); - std::vector messages{message}; - std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); - if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { - throw std::runtime_error("chunk prefill path failed for vision node summary"); - } - if (!model->vision_encoder_ || !model->vision_encoder_->graph) { - throw std::runtime_error("model has no vision encoder graph"); - } - - const auto& graph = *model->vision_encoder_->graph; - std::ostringstream out; - out << std::setprecision(9) << "{\"nodes\":["; - bool first = true; - for (const auto& node_ptr : graph.nodes_) { - const GraphNode& node = *node_ptr; - const auto& desc = node.output_buffer; - const bool interesting = - node.id < 25 || - (node.id >= 660 && node.id <= 710) || - desc.shape == std::vector{1, 2520} || - desc.shape == std::vector{1, 2520, 1} || - desc.shape == std::vector{1, 2520, 2} || - desc.shape == std::vector{1, 2520, 768} || - desc.shape == std::vector{1, 256, 768} || - desc.shape == std::vector{256, 768} || - desc.shape == std::vector{256, 1536}; - if (!interesting) continue; - std::vector values = buffer_to_floats(desc); - double sum = 0.0; - float min_v = values.empty() ? 0.0f : values[0]; - float max_v = min_v; - size_t finite_count = 0; - size_t nonfinite_count = 0; - for (float v : values) { - if (!std::isfinite(v)) { - ++nonfinite_count; - continue; - } - if (finite_count == 0) { - min_v = v; - max_v = v; - } - ++finite_count; - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - if (!first) out << ","; - first = false; - out << "{" - << "\"node_id\":" << node.id << "," - << "\"op\":\"" << op_type_name(node.op_type) << "\"," - << "\"inputs\":" << json_array(node.input_ids) << "," - << "\"shape\":" << json_array(desc.shape) << "," - << "\"precision\":\"" << precision_name(desc.precision) << "\"," - << "\"byte_size\":" << desc.byte_size << "," - << "\"value_count\":" << values.size() << "," - << "\"finite_count\":" << finite_count << "," - << "\"nonfinite_count\":" << nonfinite_count << "," - << "\"min\":" << min_v << "," - << "\"max\":" << max_v << "," - << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," - << "\"first_values\":" << json_float_array(values, 32) << "," - << "\"values\":" << json_float_array(values) - << "}"; - } - out << "]}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_vision_selected_nodes(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for selected vision nodes: " + args.model); - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() - ? "What animal is in this image? Reply in one complete short sentence." - : args.prompt; - message.images.push_back(args.image); - std::vector messages{message}; - std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); - if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { - throw std::runtime_error("chunk prefill path failed for selected vision nodes"); - } - if (!model->vision_encoder_ || !model->vision_encoder_->graph) { - throw std::runtime_error("model has no vision encoder graph"); - } - - const std::vector selected = { - 672, 675, 699, 700, - 752, 753, 756, 757, 789, 792, 793, 825, 828, 829, 846, 847, 874, 875, - 2757, 2758, 2804, 2805, 2806, 2807, 2812, 2813, 2814, 2815, 2816, - }; - const auto& graph = *model->vision_encoder_->graph; - std::ostringstream out; - out << std::setprecision(9) << "{\"nodes\":{"; - bool first = true; - for (size_t node_id : selected) { - auto it = graph.node_index_map_.find(node_id); - if (it == graph.node_index_map_.end()) continue; - const GraphNode& node = *graph.nodes_[it->second]; - const auto& desc = node.output_buffer; - std::vector values = buffer_to_floats(desc); - if (!first) out << ","; - first = false; - out << "\"" << node_id << "\":{" - << "\"node_id\":" << node.id << "," - << "\"op\":\"" << op_type_name(node.op_type) << "\"," - << "\"inputs\":" << json_array(node.input_ids) << "," - << "\"shape\":" << json_array(desc.shape) << "," - << "\"precision\":\"" << precision_name(desc.precision) << "\"," - << "\"values\":" << json_float_array(values) - << "}"; - } - out << "}}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_audio_summary(const Args& args) { - if (args.audio.empty()) throw std::runtime_error("--audio is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for audio summary: " + args.model); - } - AudioFP32 wav = load_wav(args.audio); - std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); - auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); - model->run_audio_encoder(prep.features); - - std::ostringstream out; - out << std::setprecision(9) << "{" - << "\"num_frames\":" << prep.num_frames << "," - << "\"num_soft_tokens\":" << prep.num_soft_tokens << "," - << "\"feature_count\":" << prep.features.size() << "," - << "\"feature_first_values\":" << json_float_array(prep.features, 32) << "," - << "\"media_features\":{"; - bool first_feature = true; - for (const auto& kv : model->media_features_) { - const std::string& name = kv.first; - Precision precision = model->media_feature_precisions_[name]; - std::vector values = bytes_to_floats(kv.second, precision); - double sum = 0.0; - float min_v = values.empty() ? 0.0f : values[0]; - float max_v = min_v; - size_t finite_count = 0; - size_t nonfinite_count = 0; - for (float v : values) { - if (!std::isfinite(v)) { - ++nonfinite_count; - continue; - } - if (finite_count == 0) { - min_v = v; - max_v = v; - } - ++finite_count; - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - if (!first_feature) out << ","; - first_feature = false; - out << "\"" << name << "\":{" - << "\"shape\":" << json_array(model->media_feature_shapes_[name]) << "," - << "\"precision\":\"" << precision_name(precision) << "\"," - << "\"value_count\":" << values.size() << "," - << "\"finite_count\":" << finite_count << "," - << "\"nonfinite_count\":" << nonfinite_count << "," - << "\"min\":" << min_v << "," - << "\"max\":" << max_v << "," - << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," - << "\"first_values\":" << json_float_array(values, 32) << "," - << "\"values\":" << json_float_array(values) - << "}"; - } - out << "}}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_audio_node_summary(const Args& args) { - if (args.audio.empty()) throw std::runtime_error("--audio is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for audio node summary: " + args.model); - } - AudioFP32 wav = load_wav(args.audio); - std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); - auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); - model->run_audio_encoder(prep.features); - if (!model->audio_encoder_ || !model->audio_encoder_->graph) { - throw std::runtime_error("model has no audio encoder graph"); - } - - const auto& graph = *model->audio_encoder_->graph; - std::ostringstream out; - out << std::setprecision(9) << "{\"nodes\":["; - bool first = true; - for (size_t order = 0; order < graph.nodes_.size(); ++order) { - const GraphNode& node = *graph.nodes_[order]; - const auto& desc = node.output_buffer; - const auto& shape = desc.shape; - bool interesting = node.op_type != OpType::INPUT; - if (shape.empty()) interesting = false; - if (!interesting) continue; - - std::vector values = buffer_to_floats(desc); - double sum = 0.0; - float min_v = values.empty() ? 0.0f : values[0]; - float max_v = min_v; - size_t finite_count = 0; - size_t nonfinite_count = 0; - for (float v : values) { - if (!std::isfinite(v)) { - ++nonfinite_count; - continue; - } - if (finite_count == 0) { - min_v = v; - max_v = v; - } - ++finite_count; - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - if (!first) out << ","; - first = false; - out << "{" - << "\"order\":" << order << "," - << "\"node_id\":" << node.id << "," - << "\"op\":\"" << op_type_name(node.op_type) << "\"," - << "\"inputs\":" << json_array(node.input_ids) << "," - << "\"shape\":" << json_array(shape) << "," - << "\"precision\":\"" << precision_name(desc.precision) << "\"," - << "\"value_count\":" << values.size() << "," - << "\"finite_count\":" << finite_count << "," - << "\"nonfinite_count\":" << nonfinite_count << "," - << "\"min\":" << min_v << "," - << "\"max\":" << max_v << "," - << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," - << "\"first_values\":" << json_float_array(values, 16) - << "}"; - } - out << "]}\n"; - std::cout << out.str(); - return 0; -} - -int run_gemma_first_token(const Args& args) { - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for first token probe: " + args.model); - } - if (args.command == "gemma-first-token-media-step") { - model->lm_encoder_ = nullptr; - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() ? "Respond briefly." : args.prompt; - if (!args.image.empty()) message.images.push_back(args.image); - - std::vector audio_features; - size_t audio_num_frames = 0; - if (!args.audio.empty()) { - message.audio.push_back(args.audio); - AudioFP32 wav = load_wav(args.audio); - std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); - auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); - audio_features = std::move(prep.features); - audio_num_frames = prep.num_frames; - message.audio_soft_token_count = prep.num_soft_tokens; - } - - std::vector messages{message}; - std::string prompt = tokenizer->format_chat_prompt(messages, true); - std::vector tokens = tokenizer->encode(prompt); - if (tokens.empty()) throw std::runtime_error("first token probe produced empty prompt"); - std::vector prefill_tokens(tokens.begin(), tokens.end() - 1); - if (!prefill_tokens.empty()) { - std::vector> audio_messages; - if (!audio_features.empty()) audio_messages.push_back(std::move(audio_features)); - model->prefill_with_media(prefill_tokens, args.image.empty() ? std::vector{} : std::vector{args.image}, audio_messages); - } - uint32_t first = model->decode({tokens.back()}, 0.0f, 1.0f, 1); - std::string text = tokenizer->decode({first}); - std::vector> top_logits; - if (model->decoder_ && !model->decoder_->output_node_ids.empty()) { - size_t out_node = static_cast(model->decoder_->output_node_ids[0]); - const auto& desc = model->decoder_->graph->get_output_buffer(out_node); - std::vector logits = buffer_to_floats(desc); - if (!desc.shape.empty() && !logits.empty()) { - size_t vocab = desc.shape.back(); - size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; - size_t row = seq > 0 ? seq - 1 : 0; - size_t start = row * vocab; - size_t end = std::min(start + vocab, logits.size()); - for (size_t i = start; i < end; ++i) top_logits.emplace_back(i - start, logits[i]); - size_t keep = std::min(10, top_logits.size()); - std::partial_sort( - top_logits.begin(), - top_logits.begin() + keep, - top_logits.end(), - [](const auto& a, const auto& b) { return a.second > b.second; }); - top_logits.resize(keep); - } - } - - const uint32_t image_token_id = model->get_config().image_token_id ? model->get_config().image_token_id : tokenizer->get_image_token_id(); - const uint32_t audio_token_id = model->get_config().audio_token_id; - size_t image_token_count = 0; - size_t audio_token_count = 0; - for (uint32_t token : tokens) { - if (image_token_id != 0 && token == image_token_id) ++image_token_count; - if (audio_token_id != 0 && token == audio_token_id) ++audio_token_count; - } - - std::ostringstream out; - out << "{" - << "\"prompt_token_count\":" << tokens.size() << "," - << "\"prefill_token_count\":" << prefill_tokens.size() << "," - << "\"last_prompt_token\":" << tokens.back() << "," - << "\"first_token\":" << first << "," - << "\"first_text\":\"" << escape_json(text) << "\"," - << "\"eos_token\":" << tokenizer->get_eos_token() << "," - << "\"is_eos\":" << (first == tokenizer->get_eos_token() ? "true" : "false") << "," - << "\"image_token_count\":" << image_token_count << "," - << "\"audio_token_id\":" << audio_token_id << "," - << "\"audio_token_count\":" << audio_token_count << "," - << "\"audio_num_frames\":" << audio_num_frames << "," - << "\"top_logits\":["; - for (size_t i = 0; i < top_logits.size(); ++i) { - if (i) out << ","; - out << "{\"token\":" << top_logits[i].first - << ",\"score\":" << top_logits[i].second - << ",\"text\":\"" << escape_json(tokenizer->decode({static_cast(top_logits[i].first)})) << "\"}"; - } - out << "]," - << "\"prompt_prefix\":\"" << escape_json(prompt.substr(0, 320)) << "\"" - << "}\n"; - std::cout << out.str(); - return 0; -} - -std::string build_messages(const Args& args) { - if (!args.messages_json.empty()) return args.messages_json; - std::ostringstream out; - out << "[{\"role\":\"user\",\"content\":\"" << escape_json(args.prompt.empty() ? "Respond briefly." : args.prompt) << "\""; - if (!args.image.empty()) out << ",\"images\":[\"" << escape_json(args.image) << "\"]"; - if (!args.audio.empty()) out << ",\"audio\":[\"" << escape_json(args.audio) << "\"]"; - out << "}]"; - return out.str(); -} - -std::string complete_options(int max_tokens) { - std::ostringstream out; - out << "{" - << "\"max_tokens\":" << max_tokens << "," - << "\"temperature\":0.0," - << "\"top_p\":1.0," - << "\"top_k\":1," - << "\"telemetry_enabled\":false," - << "\"auto_handoff\":false," - << "\"confidence_threshold\":0.0," - << "\"stop_sequences\":[\"<|im_end|>\",\"\"]" - << "}"; - return out.str(); -} - -std::vector read_wav_pcm16(const std::string& path) { - std::ifstream in(path, std::ios::binary); - if (!in) throw std::runtime_error("failed to open wav: " + path); - in.seekg(0, std::ios::end); - std::streamoff size = in.tellg(); - if (size <= 44) throw std::runtime_error("wav file too small: " + path); - in.seekg(44, std::ios::beg); - std::vector pcm(static_cast(size - 44)); - in.read(reinterpret_cast(pcm.data()), static_cast(pcm.size())); - if (!in) throw std::runtime_error("failed to read wav pcm: " + path); - return pcm; -} - -int run_complete(const Args& args) { - cactus_model_t model = cactus_init(args.model.c_str(), nullptr, false); - if (!model) throw std::runtime_error("cactus_init failed for " + args.model); - std::vector response(kResponseBufferSize, 0); - std::string messages = build_messages(args); - std::string options = complete_options(args.max_tokens); - int rc = cactus_complete( - model, - messages.c_str(), - response.data(), - response.size(), - options.c_str(), - nullptr, - nullptr, - nullptr, - nullptr, - 0); - cactus_destroy(model); - std::cout << response.data() << "\n"; - return rc > 0 ? 0 : 2; -} - -int run_transcribe(const Args& args, bool pcm) { - if (args.audio.empty()) throw std::runtime_error("--audio is required"); - cactus_model_t model = cactus_init(args.model.c_str(), nullptr, false); - if (!model) throw std::runtime_error("cactus_init failed for " + args.model); - std::vector response(kResponseBufferSize, 0); - std::vector pcm_data; - const char* audio_path = args.audio.c_str(); - const uint8_t* pcm_buffer = nullptr; - size_t pcm_size = 0; - if (pcm) { - pcm_data = read_wav_pcm16(args.audio); - audio_path = nullptr; - pcm_buffer = pcm_data.data(); - pcm_size = pcm_data.size(); - } - int rc = cactus_transcribe( - model, - audio_path, - nullptr, - response.data(), - response.size(), - "{\"max_tokens\":200,\"telemetry_enabled\":false,\"auto_handoff\":false}", - nullptr, - nullptr, - pcm_buffer, - pcm_size); - cactus_destroy(model); - std::cout << response.data() << "\n"; - return rc > 0 ? 0 : 2; -} - -} // namespace - -int main(int argc, char** argv) { - try { - const char* level_env = std::getenv("CACTUS_LOG_LEVEL"); - if (level_env && level_env[0]) { - cactus_log_set_level(std::atoi(level_env)); - } - Args args = parse_args(argc, argv); - if (args.command == "e002-tokens") return run_e002_tokens(args); - if (args.command == "e002-preprocess") return run_e002_preprocess(args); - if (args.command == "e002-vision") return run_e002_vision(args); - if (args.command == "e002-vision-node-summary") return run_e002_vision_node_summary(args); - if (args.command == "e002-vision-selected-nodes") return run_e002_vision_selected_nodes(args); - if (args.command == "e002-audio-summary") return run_e002_audio_summary(args); - if (args.command == "e002-audio-node-summary") return run_e002_audio_node_summary(args); - if (args.command == "gemma-first-token") return run_gemma_first_token(args); - if (args.command == "gemma-first-token-media-step") return run_gemma_first_token(args); - if (args.command == "e003-preprocess-variants") return run_e003_preprocess_variants(args); - if (args.command == "complete") return run_complete(args); - if (args.command == "transcribe") return run_transcribe(args, false); - if (args.command == "transcribe-pcm") return run_transcribe(args, true); - throw std::runtime_error("unknown command: " + args.command); - } catch (const std::exception& exc) { - std::cerr << exc.what() << "\n"; - return 1; - } -} diff --git a/local_integration_tests/test_modalities.py b/local_integration_tests/test_modalities.py deleted file mode 100644 index 62251f3e2..000000000 --- a/local_integration_tests/test_modalities.py +++ /dev/null @@ -1,210 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from conftest import assert_completion, assert_transcript, model_path, run_runner - - -def long_prompt() -> str: - sentence = "Summarize this integration test context in one concise sentence. " - return sentence * 40 - - -@pytest.mark.smoke -def test_gemma4_e2b_text_image_and_audio(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - text_model = model_path( - weights_root, - "gemma-4-e2b-it-sharedkv", - "gemma-4-e2b-it-sharedkv-c128", - "gemma-4-e2b-it", - native_config=True, - ) - media_model = model_path( - weights_root, - "gemma-4-e2b-it-alt-fixed-rowtuned-mm", - "gemma-4-e2b-it", - native_config=True, - ) - cases = [ - [text_model, "--prompt", "Reply with one short sentence about local inference."], - [media_model, "--prompt", "Describe this image in one short sentence.", "--image", assets["image_monkey"]], - [media_model, "--prompt", "Transcribe or summarize this audio in one short sentence.", "--audio", assets["audio"]], - ] - for case in cases: - model, *args = case - result = run_runner(runner, "complete", "--model", model, "--max-tokens", 24, *args, timeout=240) - assert_completion(result) - - -@pytest.mark.smoke -def test_gemma4_e2b_combined_image_audio( - runner: Path, - weights_root: Path, - assets: dict[str, Path], -) -> None: - model = model_path( - weights_root, - "gemma-4-e2b-it-alt-fixed-rowtuned-mm", - "gemma-4-e2b-it", - native_config=True, - ) - result = run_runner( - runner, - "complete", - "--model", - model, - "--prompt", - "Describe the image and summarize the audio in one short sentence.", - "--image", - assets["image_thing"], - "--audio", - assets["audio"], - "--max-tokens", - 24, - timeout=240, - ) - assert_completion(result) - - -@pytest.mark.smoke -def test_gemma4_e2b_multiturn_text_reuse(runner: Path, weights_root: Path) -> None: - model = model_path( - weights_root, - "gemma-4-e2b-it-sharedkv", - "gemma-4-e2b-it-sharedkv-c128", - "gemma-4-e2b-it", - native_config=True, - ) - messages = [ - {"role": "system", "content": "Be brief."}, - {"role": "user", "content": "Remember the name Henry."}, - {"role": "assistant", "content": "I will remember the name Henry."}, - {"role": "user", "content": "What name should you remember?"}, - ] - result = run_runner( - runner, - "complete", - "--model", - model, - "--messages-json", - json.dumps(messages), - "--max-tokens", - 24, - timeout=240, - ) - assert_completion(result) - - -@pytest.mark.full -def test_gemma4_e4b_all_modalities_when_available(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - model = model_path(weights_root, "gemma-4-e4b-it", required=False, native_config=True) - result = run_runner( - runner, - "complete", - "--model", - model, - "--prompt", - "Describe the image and audio briefly.", - "--image", - assets["image_monkey"], - "--audio", - assets["audio"], - "--max-tokens", - 24, - timeout=360, - ) - assert_completion(result) - - -@pytest.mark.smoke -def test_qwen3_vl_text_and_chunked_prefill(runner: Path, weights_root: Path) -> None: - model = model_path( - weights_root, - "qwen3-vl-2b-instruct-reconvert", - "qwen3-vl-2b-instruct-reconvert-c256", - "qwen3-vl-2b-instruct-reconvert-chunks", - "qwen3-vl-2b-instruct", - "qwen3-vl-2b", - native_config=True, - ) - for prompt in ("Reply with one short sentence about cactus.", long_prompt()): - result = run_runner(runner, "complete", "--model", model, "--prompt", prompt, "--max-tokens", 16, timeout=300) - assert_completion(result) - - -@pytest.mark.smoke -def test_qwen3_vl_image(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - model = model_path( - weights_root, - "qwen3-vl-2b-instruct-reconvert", - "qwen3-vl-2b-instruct-reconvert-c256", - "qwen3-vl-2b-instruct-reconvert-c512", - native_config=True, - ) - result = run_runner( - runner, - "complete", - "--model", - model, - "--prompt", - "Describe this image in one short sentence.", - "--image", - assets["image_monkey"], - "--max-tokens", - 16, - timeout=300, - ) - assert_completion(result) - - -@pytest.mark.smoke -def test_lfm2_vl_text_image_and_chunked_prefill(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - model = model_path( - weights_root, - "lfm2.5-vl-1.6b-retranspile-4096", - "lfm2.5-vl-1.6b-rechunk-c256", - "lfm2.5-vl-1.6b", - "lfm2.5-vl-1.6b-rechunk-c128", - "lfm2-vl-450m", - native_config=True, - ) - for prompt, image in ( - ("Reply with one short sentence about local model testing.", None), - ("Describe this image in one short sentence.", assets["image_thing"]), - (long_prompt(), None), - ): - args: list[object] = ["complete", "--model", model, "--prompt", prompt, "--max-tokens", 16] - if image is not None: - args.extend(["--image", image]) - result = run_runner(runner, *args, timeout=300) - assert_completion(result) - - -@pytest.mark.smoke -def test_parakeet_tdt_file_and_pcm_transcription(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - model = model_path( - weights_root, - "parakeet-tdt-0.6b-v3-transpiled", - "parakeet-tdt-0.6b-v3", - native_config=True, - ) - file_result = run_runner(runner, "transcribe", "--model", model, "--audio", assets["audio"], timeout=240) - pcm_result = run_runner(runner, "transcribe-pcm", "--model", model, "--audio", assets["audio"], timeout=240) - assert_transcript(file_result) - assert_transcript(pcm_result) - - -@pytest.mark.full -def test_whisper_small_file_and_pcm_transcription_when_available( - runner: Path, - weights_root: Path, - assets: dict[str, Path], -) -> None: - model = model_path(weights_root, "whisper-small", required=False, native_config=True) - file_result = run_runner(runner, "transcribe", "--model", model, "--audio", assets["audio"], timeout=300) - pcm_result = run_runner(runner, "transcribe-pcm", "--model", model, "--audio", assets["audio"], timeout=300) - assert_transcript(file_result) - assert_transcript(pcm_result) diff --git a/local_investigations/gemma4_multimodal/BACKGROUND.md b/local_investigations/gemma4_multimodal/BACKGROUND.md deleted file mode 100644 index 8a7cf2031..000000000 --- a/local_investigations/gemma4_multimodal/BACKGROUND.md +++ /dev/null @@ -1,53 +0,0 @@ -# Background - -## Local artifact inventory - -Multimodal Gemma candidates: -- `gemma-4-e2b-it` -- `gemma-4-e2b-it-alt-fixed-rowtuned-mm` -- `gemma-4-e2b-it-fresh-mm` - -Text-only/shared-KV Gemma candidates: -- `gemma-4-e2b-it-sharedkv` -- `gemma-4-e2b-it-sharedkv-c128` -- `gemma-4-e2b-it-sharedkv-c256` -- `gemma-4-e2b-it-sharedkv-c512` - -The shared-KV bundles do not include `vision_encoder` or `audio_encoder`, so they are not valid vision/audio quality candidates. - -## Multimodal component shape facts - -Both local multimodal Gemma E2B bundles point at `google/gemma-4-E2B-it` and use the same local HF snapshot. - -The two older multimodal bundles point at missing HF snapshot `b324173c7d5721c2baba7f3b17b3b9b3d34ab1e9`. - -The fresh bundle points at current cached HF snapshot `905e84b50c4d2a365ebde34e685027578e6728db`. - -Component order: -- `audio_encoder` -- `vision_encoder` -- `lm_encoder` -- `decoder_prefill_chunk` -- `lm_encoder_step` -- `lm_encoder_media_step` -- `decoder_step` - -Important logical inputs: -- `vision_encoder`: `pixel_values`, `pixel_position_ids` -- `audio_encoder`: `input_features`, `input_features_mask` -- `lm_encoder`: `input_ids`, `image_features`, `audio_features` -- `lm_encoder_media_step`: `inputs_embeds`, `input_ids`, `position_ids` - -## Controls - -The paired audio created from `../cactus/what_is_in_this_image.mp3` was converted to RIFF/WAV because `cactus_complete` currently rejects MP3 with `Not RIFF`. - -Parakeet transcribes the converted WAV as: - -```text -What is in this image? -``` - -Qwen3-VL identifies `test_monkey.png` as a proboscis monkey, so the asset itself is a good vision control. - -HF reference on the current cached snapshot identifies the same image with the same general prompt as an orangutan. HF also works when using the Cactus-native expanded image prompt/tensors after renaming `pixel_position_ids` to HF's `image_position_ids`. diff --git a/local_investigations/gemma4_multimodal/EXPERIMENTS.md b/local_investigations/gemma4_multimodal/EXPERIMENTS.md deleted file mode 100644 index 0e5f36f6d..000000000 --- a/local_investigations/gemma4_multimodal/EXPERIMENTS.md +++ /dev/null @@ -1,80 +0,0 @@ -# Experiments - -## E1: Paired audio validation - -Purpose: -- Verify that the new audio fixture says what the user intended before using it for Gemma mixed testing. - -Inputs: -- `../cactus/what_is_in_this_image.mp3` -- Converted WAV: `local_integration_tests/.tmp/what_is_in_this_image.wav` -- Parakeet model: `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` - -Observed: - -```text -What is in this image? -``` - -Interpretation: -- The audio fixture is good. -- The native completion path currently requires WAV input, not MP3. - -## E2: Gemma and Qwen image controls - -Purpose: -- Confirm whether the monkey image is a valid vision test and whether the Gemma failure is model/path-specific. - -Observed: -- Qwen3-VL identifies the image as a proboscis monkey. -- Gemma multimodal artifacts fail to identify the monkey in image-only prompts. - -Interpretation: -- The asset is a valid vision control. -- The failure is specific to Gemma artifact/code path behavior, not the image itself. - -## E2b: HF reference with Cactus-native prompt style - -Purpose: -- Check whether Cactus-native expanded media prompts and native image tensors are themselves valid. - -Observed: -- HF current snapshot with normal processor chat template answered: `The animal in this image is an orangutan.` -- HF current snapshot with Cactus-native expanded prompt/tensors also answered: `The animal in this image is an orangutan.` - -Interpretation: -- Prompt style and native image tensor preparation are not the high-level cause. -- The remaining fault is in the transpiled/C++ component execution path or artifact lowering/runtime behavior. - -## E3: Force Gemma media through media-step path - -Purpose: -- Test whether Gemma image-only quality fails because arbitrary prompts are incorrectly routed through the static `lm_encoder` chunk path. - -Change tested: -- Temporarily disabled chunk media prefill for Gemma so `prefill_with_media` used the `lm_encoder_media_step` loop. - -Observed: -- Gemma image-only monkey prompt produced malformed/gibberish output instead of identifying the image. -- Repeating the same temporary routing change with `gemma-4-e2b-it-fresh-mm` also produced malformed/gibberish output. - -Interpretation: -- The simple routing fallback is not valid. -- The investigation should compare actual component contracts and traced prompt assumptions before changing routing further. - -## E4: Fresh reconvert artifact - -Purpose: -- Determine whether stale artifacts caused the Gemma vision failure. - -Artifact: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -Observed: -- Fresh artifact points at current cached snapshot `905e84b50c4d2a365ebde34e685027578e6728db`. -- Image-only monkey prompt produced: `The image shows a close-up of a human nose.` -- Similar prompt produced: `The image shows a close-up of a fawn's nose.` - -Interpretation: -- Reconvert was necessary to remove stale-artifact uncertainty. -- Reconvert alone does not solve Gemma native vision quality. diff --git a/local_investigations/gemma4_multimodal/FIXES.md b/local_investigations/gemma4_multimodal/FIXES.md deleted file mode 100644 index 854813740..000000000 --- a/local_investigations/gemma4_multimodal/FIXES.md +++ /dev/null @@ -1,52 +0,0 @@ -# Fixes - -## F1: Audio incremental decode prompt replay - -Status: applied before this investigation workspace was created - -Rationale: -- `cactus_complete` passed the entire processed prompt into `decode_with_audio` on each generated token. -- `decode_with_audio` currently delegates to incremental `decode`, so this replayed the prompt into KV cache repeatedly. - -Observed effect: -- Mixed-media generation speed improved. -- Gemma audio semantics remained poor, so this was not the full root cause. - -## F2: Audio encoder mask valid-frame handling - -Status: applied before this investigation workspace was created - -Rationale: -- The audio frontend pads feature buffers to graph capacity. -- The mask should mark only valid frames, not all padded frames. - -Observed effect: -- Build and smoke tests still pass. -- Gemma audio semantics remained poor, so this was not the full root cause. - -## F3: Force Gemma media-step fallback - -Status: reverted - -Rationale: -- Tested whether the static chunk `lm_encoder` path was the immediate cause of Gemma image-only failure. - -Observed effect: -- Output became gibberish. -- The change was reverted. -- The same experiment was repeated with the fresh artifact and was still gibberish. - -## F4: Fresh Gemma4 multimodal reconvert - -Status: created for investigation - -Artifact: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -Rationale: -- Existing multimodal manifests referenced a missing older HF snapshot. -- The current cached HF snapshot produces a valid HF reference answer. - -Observed effect: -- Fresh artifact no longer depends on the missing snapshot. -- Native Cactus vision quality is still wrong, so further runtime/component investigation is required. diff --git a/local_investigations/gemma4_multimodal/HYPOTHESES.md b/local_investigations/gemma4_multimodal/HYPOTHESES.md deleted file mode 100644 index c90541186..000000000 --- a/local_investigations/gemma4_multimodal/HYPOTHESES.md +++ /dev/null @@ -1,71 +0,0 @@ -# Hypotheses - -## H1: Current multimodal artifact is stale or malformed - -Status: supported but not sufficient - -Evidence so far: -- Both local multimodal Gemma E2B bundles have full multimodal components. -- Both reference the same HF snapshot. -- Shared-KV artifacts are text-only and should not be used for vision/audio quality. -- The older multimodal bundles reference a missing local HF snapshot. -- A fresh bundle was reconverted from the current cached HF snapshot. -- The fresh bundle improves the failure mode but still does not match HF vision quality. - -Next evidence needed: -- Keep using `gemma-4-e2b-it-fresh-mm` as the best Cactus artifact for further debugging. -- Do not assume reconversion alone fixes the native runtime path. - -## H2: Gemma native image preprocessing diverges from Python/HF preprocessing - -Status: open - -Evidence so far: -- C++ and Python both appear to use Gemma4-style resizing, patching, and position IDs. -- No tensor-level diff has been run yet. - -Next evidence needed: -- Compare `pixel_values` and `pixel_position_ids` between C++ and Python for the same image. - -## H3: Gemma multimodal chunk prefill uses a static traced merge plan that is invalid for arbitrary prompts - -Status: partially supported, simple fallback refuted - -Evidence so far: -- The `lm_encoder` component was traced with a fixed processor prompt embedded in the graph metadata. -- The Python adapter builds a native merge plan during transpile from the example `input_ids`. -- C++ currently sends arbitrary prompt tokens through this `lm_encoder` for media prefill. -- Gemma image-only fails even though Qwen identifies the same image. - -Experiment: -- Route Gemma media prefill through `lm_encoder_media_step` instead of the chunk `lm_encoder` path, then re-run image-only quality probes. - -Result: -- The simple dynamic fallback produced obvious gibberish for image-only monkey prompts. -- The same result reproduced with the fresh artifact. -- This suggests `lm_encoder_media_step` is not a drop-in prefill replacement as currently wired, or it needs additional exact contract handling. -- The static-merge concern remains plausible, but the fix is not simply disabling chunk prefill for all Gemma media. - -## H5: HF/native prompt and preprocessing are sound; the divergence is in transpiled/C++ execution - -Status: supported - -Evidence so far: -- HF with the normal processor chat template identifies the monkey image as an orangutan. -- HF with Cactus-native expanded prompt/tensors also identifies the image as an orangutan when `pixel_position_ids` are passed as HF `image_position_ids`. -- Fresh Cactus native with the same image says the image is a human/fawn nose. - -Next evidence needed: -- Compare component-level outputs or implement/test the intended dynamic Gemma media chunk path using `lm_encoder_text_chunk` and `lm_encoder_media_chunk`. - -## H4: Audio frontend or audio mask is wrong - -Status: open - -Evidence so far: -- The completion decode loop previously replayed the whole prompt for audio generation; this was fixed. -- The audio encoder mask was changed to mark only valid frames. -- Audio-only still does not show evidence of understanding the spoken content. - -Next evidence needed: -- Work on audio only after vision-only is explained or fixed. diff --git a/local_investigations/gemma4_multimodal/PLAN.md b/local_investigations/gemma4_multimodal/PLAN.md deleted file mode 100644 index 5fc3bdad7..000000000 --- a/local_investigations/gemma4_multimodal/PLAN.md +++ /dev/null @@ -1,29 +0,0 @@ -# Plan - -## Current phase - -Isolate Gemma vision-only correctness before treating mixed image+audio as a product behavior. - -## Method - -Use a hypothesis-driven loop: -1. Record a hypothesis in `HYPOTHESES.md`. -2. Run the smallest experiment that can support or refute it. -3. Record exact output and interpretation in `EXPERIMENTS.md`. -4. If supported, make the smallest fix and document it in `FIXES.md`. -5. Re-run the focused quality probe and the nearest regression suite. - -## Layer order - -1. Artifact validity and freshness. -2. Prompt/token construction. -3. Image preprocessing. -4. Vision encoder output. -5. LM media merge behavior. -6. Decoder/cache behavior. -7. Audio preprocessing and encoder behavior. -8. Mixed image+audio behavior. - -## Current decision - -Investigate whether Gemma multimodal prefill is incorrectly using the static chunk `lm_encoder` path for arbitrary prompts instead of the dynamic `lm_encoder_media_step` path. diff --git a/local_investigations/gemma4_multimodal/README.md b/local_investigations/gemma4_multimodal/README.md deleted file mode 100644 index 0cde142cb..000000000 --- a/local_investigations/gemma4_multimodal/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Gemma4 multimodal investigation - -Local, uncommitted workspace for investigating Gemma4 native vision/audio quality. - -Current status: -- Vision-only Gemma native does not reliably identify `cactus-engine/tests/assets/test_monkey.png`. -- Audio-only Gemma native does not yet show evidence that it is using the audio content. -- Mixed image+audio with `what_is_in_this_image.wav` currently fails because the underlying unimodal paths are not known-good. -- Qwen3-VL and Parakeet are useful controls: Qwen identifies the monkey, and Parakeet transcribes the paired audio. - -Primary artifacts: -- `../cactus/weights/gemma-4-e2b-it` -- `../cactus/weights/gemma-4-e2b-it-alt-fixed-rowtuned-mm` -- `../cactus/weights/qwen3-vl-2b-instruct-reconvert` -- `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` - -Read in order: -1. `BACKGROUND.md` -2. `PLAN.md` -3. `HYPOTHESES.md` -4. `EXPERIMENTS.md` -5. `FIXES.md` -6. `SUMMARY.md` diff --git a/local_investigations/gemma4_multimodal/SUMMARY.md b/local_investigations/gemma4_multimodal/SUMMARY.md deleted file mode 100644 index e39aa24e0..000000000 --- a/local_investigations/gemma4_multimodal/SUMMARY.md +++ /dev/null @@ -1,9 +0,0 @@ -# Summary - -Investigation is in progress. - -Current leading hypothesis: -- Gemma media prefill has a component-contract mismatch or lowering/runtime mismatch after HF-native prompt/preprocessing, likely around transpiled media encoder/LM merge execution. - -Current next step: -- Compare component-level outputs or implement/test the intended dynamic Gemma media chunk path using `lm_encoder_text_chunk` and `lm_encoder_media_chunk`. diff --git a/local_investigations/gemma4_multimodal_v2/00_INDEX.md b/local_investigations/gemma4_multimodal_v2/00_INDEX.md deleted file mode 100644 index 71ed21eb7..000000000 --- a/local_investigations/gemma4_multimodal_v2/00_INDEX.md +++ /dev/null @@ -1,48 +0,0 @@ -# Gemma4 Multimodal V2 Index - -Status: Gemma4 image-only, audio-only, and mixed image+audio now pass local end-to-end probes after the dynamic audio merge fix; Qwen, Parakeet, policy, and smoke controls pass - -Active phase: post-fix review / audit. Do not commit or push until the user reviews. - -Default artifact under investigation: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -Key controls: -- HF reference snapshot: `905e84b50c4d2a365ebde34e685027578e6728db` -- Qwen image control: `../cactus/weights/qwen3-vl-2b-instruct-reconvert` -- Parakeet audio control: `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` - -Read order: -1. `09_STATUS_FOR_NEXT_AGENT.md` -2. `10_CONTINUATION_PLAN.md` -3. `01_BACKGROUND.md` -4. `02_QUESTIONS.md` -5. `03_HYPOTHESES.md` -6. `04_EXPERIMENTS.md` -7. `05_DECISIONS.md` -8. `06_FIX_LOG.md` -9. `07_ACCEPTANCE.md` -10. `08_HANDOFF.md` - -Latest proven artifacts: - -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json`: image preprocessing parity recorded as `status: pass`, max abs diff `4.987010955859184e-07`. -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json`: latest recorded vision encoder output is FP16-close to the Python/HF reference for `pooler_output`, cosine `0.9999955754245847`, mean abs diff `0.0015996834738693023`. -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json`: image-only Cactus Gemma answered `The animal in the image is a golden monkey.` -- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json`: after the Gemma4 audio bias FP16 fix, first 64 audio rows are FP16-close to Python, cosine `0.9978210926`, mean abs diff `0.00645508`. -- `artifacts/E029_first_token_after_dynamic_merge/`: dynamic merge first-token probes now produce normal text starts, audio `Please`, mixed `This`. -- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json`: audio-only Cactus Gemma responds coherently: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json`: mixed Cactus Gemma answers: `This image is of a golden monkey.` -- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json`: image-only still answers: `The animal in the image is a golden monkey.` -- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json`: Qwen image control identifies a proboscis monkey. -- `artifacts/E034_parakeet_regression_after_dynamic_merge/`: Parakeet file and PCM controls both transcribe `What is in this image?` -- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json`: full emitted audio tensor matches the E023 post-bias tensor exactly, max diff `0`. - -Latest test commands: -- `source ../cactus/venv/bin/activate && ./cactus/build.sh` -- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q` -> `16 passed` -- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s` -> `7 passed, 2 deselected` - -Continuation contract: -- Follow `10_CONTINUATION_PLAN.md`. -- The acceptance probes now pass locally. Next work is review, cleanup, and any additional audit the user requests. diff --git a/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md b/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md deleted file mode 100644 index a4336c5f5..000000000 --- a/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md +++ /dev/null @@ -1,48 +0,0 @@ -# Background - -## Stable facts copied from V1 - -The older local Gemma multimodal bundles are: -- `../cactus/weights/gemma-4-e2b-it` -- `../cactus/weights/gemma-4-e2b-it-alt-fixed-rowtuned-mm` - -Both older bundles reference missing HF snapshot: -- `b324173c7d5721c2baba7f3b17b3b9b3d34ab1e9` - -The fresh investigation bundle is: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -The fresh bundle references current cached HF snapshot: -- `905e84b50c4d2a365ebde34e685027578e6728db` - -The shared-KV Gemma artifacts are text-only for this investigation because they do not include `vision_encoder` or `audio_encoder`. - -## Confirmed prior observations - -- HF reference identifies `test_monkey.png` as an orangutan/monkey-class animal. -- HF still works when using Cactus-native expanded image prompt/tensors with `pixel_position_ids` passed as HF `image_position_ids`. -- Fresh Cactus native Gemma originally failed image quality in E001, describing the monkey image as a human/fawn nose. -- Later current artifacts record image preprocessing parity and image-only generation success: - - `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` - - `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` -- Later artifacts record audio-only and mixed image+audio passing after the audio bias FP16 fix and dynamic Gemma4 audio merge: - - `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` - - `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` -- Qwen3-VL identifies the same image as a proboscis monkey. -- Parakeet transcribes the paired audio as: `What is in this image?` -- Direct MP3 input to native completion fails with `Not RIFF`; the paired audio must be converted to WAV. - -## Local paths - -Image: -- `cactus-engine/tests/assets/test_monkey.png` - -Audio: -- Source MP3: `../cactus/what_is_in_this_image.mp3` -- Converted WAV when needed: `local_integration_tests/.tmp/what_is_in_this_image.wav` - -Harness: -- `local_investigations/gemma4_multimodal_v2/scripts/run_probe.py` - -Artifacts: -- `local_investigations/gemma4_multimodal_v2/artifacts/` diff --git a/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md b/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md deleted file mode 100644 index 46965b759..000000000 --- a/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md +++ /dev/null @@ -1,41 +0,0 @@ -# Questions - -## Q1: Where is the first bad boundary? - -Status: answered for vision and audio in the current artifact set - -Candidate boundaries: -- Prompt/token IDs and media spans. -- Image preprocessing tensors. -- Vision encoder output. -- LM media merge output. -- Decoder first-token logits. -- Autoregressive decode loop. - -Resolution: -- For vision, E002/E003/E014/E015 have current artifacts and image-only is recorded as passing. -- For audio, E022/E023 localized and fixed the audio encoder bias issue, and E027/E029 localized and fixed the static media merge issue. - -## Q2: Is the fresh Gemma artifact good enough to be the default investigation artifact? - -Status: yes for current investigation - -Known evidence: -- It points at the current cached HF snapshot. -- E001 confirmed HF on the same snapshot worked while Cactus image-only originally did not. -- Later artifacts record Cactus image-only success on the fresh artifact. -- E030/E031 record audio-only and mixed image+audio passing on the fresh artifact. - -Next falsifier: -- If a fresh rerun contradicts E030/E031, inspect the changed artifact/runtime state before changing prompts. - -## Q3: Is the previous media-step fallback worth pursuing? - -Status: no as a wholesale fallback; dynamic media-step use inside chunked prefill is accepted. - -Known evidence: -- Forcing all Gemma media through the old step fallback produced gibberish on mixed prompts. -- The accepted fix uses `lm_encoder_media_step` only to assemble real prompt-position outputs before chunked decoder prefill. - -Next falsifier: -- Only revisit after proving the correct `lm_encoder_media_chunk`/step contract and how cache copies should work. diff --git a/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md b/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md deleted file mode 100644 index 46eacf99b..000000000 --- a/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md +++ /dev/null @@ -1,66 +0,0 @@ -# Hypotheses - -## H1: Artifact staleness caused the original failure - -Status: supported but not sufficient; not the active boundary - -Evidence: -- Older multimodal artifacts reference a missing HF snapshot. -- Fresh artifact was reconverted from current cached snapshot. -- Fresh artifact originally failed native Gemma vision quality in E001. -- Later artifacts show image-only now passes after targeted fixes. - -Decision: -- Use the fresh artifact for V2 unless a stronger artifact is produced for a specific experiment. - -## H2: Prompt style or native image preprocessing is invalid - -Status: refuted for current image-only path - -Evidence: -- HF works with the normal processor prompt. -- HF also works with Cactus-native expanded prompt/tensors. -- E001 stores both records under `artifacts/E001/`. -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records image preprocessing parity as `status: pass`. -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records image-only output `The animal in the image is a golden monkey.` - -Resolution: -- Audio and mixed image+audio now pass in E030/E031; do not reopen image preprocessing without contradictory fresh evidence. - -## H3: The transpiled/C++ component path diverges after preprocessing - -Status: historically true for vision and audio; currently fixed in local artifacts - -Evidence: -- Vision had later graph/runtime/conversion fixes recorded in `06_FIX_LOG.md`. -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` records FP16-close vision output agreement. -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` records a substantial emitted `audio_features` mismatch against the Python native-like reference. -- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` records audio features as FP16-close after the audio bias fix. - -Next test: -- Rerun E030/E031 if the artifact or runtime changes. - -## H4: The C++ media merge/cache path is using the wrong component contract - -Status: supported and fixed for Gemma4 prompts containing audio - -Evidence: -- Simple `lm_encoder_media_step` fallback produced gibberish. -- Fresh artifact includes `lm_encoder_text_chunk`, `lm_encoder_media_step`, and `lm_encoder_media_chunk`, but C++ currently uses the monolithic `lm_encoder` chunk path for media. -- E025/E026 previously returned empty responses after the audio encoder fix, which made media merge/cache the next boundary. -- After the audio encoder fix, E027 showed the static `lm_encoder` chunk path selected `` for audio/mixed first tokens. -- E029/E030/E031 show dynamic prompt-position media merge fixes first-token and generation behavior. - -Next test: -- Keep dynamic merge active for Gemma4 prompts with audio and rerun regressions after runtime changes. - -## H5: Audio should remain blocked until vision is explained - -Status: retired - -Evidence: -- Vision-only has a recorded passing artifact in `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json`. -- Audio-only and mixed image+audio pass in E030/E031. - -Next test: -- None unless a fresh rerun contradicts the passing artifacts. diff --git a/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md b/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md deleted file mode 100644 index 37ef0129d..000000000 --- a/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md +++ /dev/null @@ -1,356 +0,0 @@ -# Experiments - -## Experiment format - -Each experiment must include: -- ID -- Question -- Setup -- Command -- Raw result path -- Interpretation -- Next decision - -## E001: Baseline ledger - -Question: -- Can the harness reproduce the known HF/Cactus/control outputs and store them in a comparable format? - -Setup: -- Fresh Gemma artifact: `../cactus/weights/gemma-4-e2b-it-fresh-mm` -- Image: `cactus-engine/tests/assets/test_monkey.png` -- Audio: converted `what_is_in_this_image.wav` - -Command: - -```bash -python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case all-baseline --experiment E001 -``` - -Raw result path: -- `artifacts/E001/` - -Interpretation: -- Completed. -- HF Gemma normal prompt: `The animal in this image is an orangutan.` -- HF Gemma Cactus-native expanded prompt/tensors: `The animal in this image is an orangutan.` -- Cactus fresh Gemma native: `The image shows a close-up of a human nose.` -- Cactus Qwen control identifies the proboscis monkey. -- Parakeet file and PCM controls both transcribe: `What is in this image?` - -Next decision: -- Move to E002 component-boundary comparison. -- The next probe must localize the first bad boundary after HF-native prompt/preprocessing, not attempt another runtime fix. - -## E002: Component-boundary comparison - -Question: -- At which component boundary does fresh Cactus native first diverge from the HF/Python reference? - -Setup: -- Same artifact, prompt, and image as E001. -- Compare in this order: input IDs/media spans, image tensors, vision encoder output, LM merge output, decoder first-token logits. - -Command: -```bash -source ../cactus/venv/bin/activate -./cactus/build.sh -python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case e002-boundary --experiment E002 -``` - -Raw result path: -- `artifacts/E002/` -- `artifacts/E002/token_spans.json` -- `artifacts/E002/image_preprocess_compare.json` -- `artifacts/E002/hf_native_inputs.json` -- `artifacts/E002/cactus_token_probe.json` -- `artifacts/E002/cactus_preprocess.json` - -Interpretation: -- Completed 2026-05-20. -- Gate A passed: HF/Python native and C++ Cactus native `input_ids` match exactly. -- Gate A image span also matches exactly: token count `282`, assistant generation start `279`, image token id `258880`, image start `5`, image token count `256`. -- Gate B failed at `pixel_values`. -- `pixel_values` shape matches: `[1, 2520, 768]`. -- `pixel_position_ids` shape and values match, with valid position count `2304`. -- `pixel_values` differ immediately at element `0`; max absolute difference is `0.1298471341896057`, mean absolute difference is `0.002023158910580417`. -- HF/Python pixel range is `[0.0, 1.0]`; C++ pixel range is `[-0.0213023126, 1.00900519]`. -- The first divergent boundary is Gemma4 native image preprocessing, before the vision encoder. - -Next decision: -- Stop E002 at Gate B for now, then continue with E003. Do not run vision encoder, LM merge, or decoder comparison until preprocessing parity is fixed or a stricter preprocessing reference proves the current C++ tensor is acceptable. -- The next fix should be scoped to `preprocess_gemma4_image` parity with `_prepare_gemma4_native_image_tensors`, including resize/interpolation and numeric range behavior. -- This is not final acceptance; the mission continues until Gemma image-only, audio-only, and mixed image+audio pass end to end. - -Superseded by later artifacts below: -- E003 / `E002_after_E003_pillow_resize` records preprocessing parity as passing. -- E015 records image-only generation as passing. -- Audio-only and mixed image+audio are now the active failures. - -## Current status correction: later artifacts after E002 - -This section records the newer artifacts currently present in the worktree. It does not erase E001/E002 history; it updates the active boundary for future agents. - -## E003 / E002_after_E003_pillow_resize: Image preprocessing parity - -Question: -- Does the C++ Gemma4 image preprocessing output match the Python/HF native reference after the Pillow-compatible resize path? - -Raw result paths: -- `artifacts/E003/preprocess_operation_compare.json` -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` - -Interpretation: -- Completed in current artifacts. -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. -- `pixel_values` max abs diff is `4.987010955859184e-07`. -- `pixel_values` mean abs diff is `2.2151478535820782e-07`. -- HF/Python range is `[0.0, 1.0]`; C++ range is `[0, 1]`. - -Next decision: -- Treat image preprocessing as fixed unless a fresh rerun contradicts this artifact. -- Continue to the vision encoder boundary. - -## E014: Latest recorded vision encoder comparison - -Question: -- Does the Cactus `vision_encoder` output match the Python/HF reference closely enough after the recorded graph/runtime/conversion fixes? - -Raw result path: -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` - -Interpretation: -- The latest recorded `pooler_output` comparison has matching shape `[256, 1536]`. -- Cosine similarity is `0.9999955754245847`. -- Max abs diff is `0.03515437499999985`. -- Mean abs diff is `0.0015996834738693023`. -- This is recorded as FP16-close. The artifact does not by itself prove every later LM merge/logit boundary, so do not overstate it as full multimodal acceptance. - -Next decision: -- Use image-only generation quality as the next acceptance check for the vision path. - -## E015: Gemma image-only quality check after vision fixes - -Question: -- Does fresh Cactus Gemma image-only generation identify `test_monkey.png` as a monkey-class animal? - -Raw result path: -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` - -Interpretation: -- Cactus returned success. -- Response: `The animal in the image is a golden monkey.` -- `prefill_tokens: 282`. -- `decode_tokens: 10`. -- This satisfies the image-only semantic acceptance class recorded in `07_ACCEPTANCE.md`. - -Next decision: -- Treat Gemma image-only as passing in the current artifact set. -- Continue to Gemma audio-only and mixed image+audio. - -## E016 / E020: Gemma audio-only quality checks - -Question: -- Does fresh Cactus Gemma audio-only generation produce coherent text using the paired audio prompt? - -Raw result paths: -- `artifacts/E016_gemma_audio_after_vision_gate/cactus_gemma_audio.json` -- `artifacts/E020_gemma_audio_after_audio_retranspile/cactus_gemma_audio.json` - -Interpretation: -- Both recorded Cactus runs returned success but no generated text. -- E020 response is empty: `response: ""`. -- E020 has `prefill_tokens: 83` and `decode_tokens: 0`. -- Retranspiling the audio encoder in the recorded E019/E020 sequence did not resolve audio-only generation. - -Next decision: -- Audio-only remains an active failure. -- Continue with audio boundary comparison before changing prompts or decoder settings. - -## E017: Gemma mixed image+audio quality check - -Question: -- Does fresh Cactus Gemma mixed image+audio generation answer the spoken question about `test_monkey.png`? - -Raw result path: -- `artifacts/E017_gemma_mixed_after_vision_gate/cactus_gemma_mixed.json` - -Interpretation: -- Cactus returned success but no generated text. -- Response is empty: `response: ""`. -- `prefill_tokens: 349`. -- `decode_tokens: 0`. - -Next decision: -- Mixed image+audio remains an active failure. -- Do not treat image-only success as full acceptance. - -## E018 / E019: Audio summary before and after audio retranspile - -Question: -- Do the recorded Cactus audio preprocessing and emitted audio feature summaries change after retranspiling the audio encoder? - -Raw result paths: -- `artifacts/E018_audio_summary/audio_summary.json` -- `artifacts/E019_audio_summary_after_retranspile/audio_summary.json` - -Interpretation: -- Both artifacts record `num_frames: 255`, `num_soft_tokens: 64`, and `feature_count: 32640`. -- Both artifacts record emitted `audio_features` shape `[1, 753, 1536]` with precision `FP16`. -- The first emitted values are unchanged across E018 and E019. -- These summaries are not enough to prove audio preprocessing parity because they only include prefixes and aggregate statistics. - -Next decision: -- Use full-tensor comparison before selecting an audio fix. - -## E021: Full Cactus-vs-Python audio feature comparison - -Question: -- Do the Cactus emitted `audio_features` match the Python native-like audio feature reference? - -Raw result paths: -- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` - -Interpretation: -- `cactus_audio_summary.json` now includes full `audio_features.values`. -- Cactus and Python output shapes match: `[1, 753, 1536]`. -- Python prepared feature shape is `[1, 3012, 128]`; Python input feature mask sum is `255`. -- For the first 64 rows, cosine similarity is `0.36957991123199463`, max abs diff is `2.796870708465576`, and mean abs diff is `0.16708998382091522`. -- For all 753 rows, cosine similarity is `0.7203282713890076`, max abs diff is `2.796870708465576`, and mean abs diff is `0.10658295452594757`. -- The `python_audio_token_count` field in this artifact is not used for conclusions because the comparison script counted a token id that is not validated here. - -Next decision: -- The current best proven active boundary is the audio encoder output / emitted audio feature boundary. -- Next work should localize the first divergent internal audio encoder operation or weight. Do not infer a specific internal cause from E021 alone. - -## E022 / E023 / E024: Audio encoder localization and FP16 bias fix - -Question: -- What caused the emitted Gemma4 `audio_features` mismatch from E021? - -Raw result paths: -- `artifacts/E022_audio_node_localization/` -- `artifacts/E023_audio_after_fp16_bias/` -- `artifacts/E024_retranspile_audio_fp16_bias/` - -Interpretation: -- E022 localized the first important divergence to the audio output projection bias handling. -- The active artifact had stored `audio_output_proj.bias` as an INT8 fallback, which made Cactus add raw values like `[1, -2, 0, 70]` instead of HF FP16-like bias values. -- After changing Gemma4 audio bias policy to FP16 and updating the active artifact, E023 records first 64 audio rows as FP16-close to Python: cosine `0.9978210926`, mean abs diff `0.00645508`. - -Next decision: -- Keep Gemma4 audio bias tensors as FP16. -- Continue to generation and first-token probes because audio-only and mixed still returned empty strings after E023. - -## E025 / E026 / E027: Post-audio-encoder generation and first-token failure - -Question: -- After fixing the audio encoder boundary, why do audio-only and mixed generation still fail? - -Raw result paths: -- `artifacts/E025_gemma_audio_after_fp16_bias/cactus_gemma_audio.json` -- `artifacts/E026_gemma_mixed_after_fp16_bias/cactus_gemma_mixed.json` -- `artifacts/E027_first_token_after_audio_encoder_fix/` - -Interpretation: -- Audio-only and mixed still returned empty responses after the audio encoder fix. -- First-token probes showed the static chunk path selected `` for both audio-only and mixed prompts, unlike HF references (`The` / `This` starts). -- Forcing the old media-step path improved audio-only first token to `Please`, but mixed produced whitespace, so wholesale fallback was not acceptable. -- The static `lm_encoder` graph was captured with stale media span assumptions and could insert audio at the wrong positions/counts for the current prompt. - -Next decision: -- Use a Gemma4-audio-only dynamic media merge inside the chunked prefill path instead of relying on stale static `lm_encoder` merge captures. - -## E029: First-token probes after dynamic merge - -Question: -- Does the dynamic Gemma4 audio merge fix first-token behavior for audio-only and mixed prompts? - -Raw result path: -- `artifacts/E029_first_token_after_dynamic_merge/` - -Interpretation: -- Audio-only first token is `Please`, with normal text alternatives in top logits. -- Mixed image+audio first token is `This`, matching the HF-style first-token direction. -- Neither probe selects ``, ``, or whitespace-only as the top token. - -Next decision: -- Continue to full generation quality checks. - -## E030: Gemma audio-only after dynamic merge - -Question: -- Does audio-only Gemma use the paired spoken prompt coherently? - -Raw result path: -- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` - -Interpretation: -- Cactus generated: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- This is coherent with the audio prompt saying `what is in this image` and no image being supplied. -- `decode_tokens: 21`. - -Next decision: -- Audio-only quality gate passes. Continue to mixed image+audio. - -## E031: Gemma mixed image+audio after dynamic merge - -Question: -- Does mixed Gemma answer the spoken question about `test_monkey.png`? - -Raw result path: -- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` - -Interpretation: -- Cactus generated: `This image is of a golden monkey.` -- This satisfies the mixed acceptance class. -- `decode_tokens: 8`. - -Next decision: -- Mixed gate passes. Run regressions. - -## E032 / E033 / E034 / E035: Regression checks after dynamic merge - -Question: -- Did the dynamic merge fix regress image-only Gemma, Qwen, Parakeet, or the audio encoder boundary? - -Raw result paths: -- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` -- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` -- `artifacts/E034_parakeet_regression_after_dynamic_merge/` -- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` - -Interpretation: -- Gemma image-only still generated: `The animal in the image is a golden monkey.` -- Qwen image control generated a proboscis-monkey answer. -- Parakeet file and PCM controls both transcribed: `What is in this image?` -- E035 shows the emitted Cactus audio tensor is unchanged from E023, with full-tensor max diff `0`. -- Policy tests passed: `16 passed`. -- Smoke tests passed: `7 passed, 2 deselected`. - -Next decision: -- The planned local acceptance gates pass. Keep the working tree uncommitted for user audit. - -## E036 / E037 / E038 / E039 / E040: Cleanup verification probes - -Question: -- After ignoring/removing generated local outputs, do the required quality probes still regenerate needed files and pass? - -Raw result paths: -- `artifacts/E036_cleanup_gemma_image/cactus_gemma_image.json` -- `artifacts/E037_cleanup_gemma_audio/cactus_gemma_audio.json` -- `artifacts/E038_cleanup_gemma_mixed/cactus_gemma_mixed.json` -- `artifacts/E039_cleanup_qwen_image/cactus_qwen_image.json` -- `artifacts/E040_cleanup_parakeet/` - -Interpretation: -- Gemma image-only still generated: `The animal in the image is a golden monkey.` -- Gemma audio-only still generated: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- Gemma mixed still generated: `This image is of a golden monkey.` -- Qwen image still generated a proboscis-monkey answer. -- Parakeet file and PCM still transcribed: `What is in this image?` -- The ignored `.tmp` directory was recreated successfully by the harness. - -Next decision: -- Cleanup did not regress the acceptance probes or remove benchmark/test harness source. diff --git a/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md b/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md deleted file mode 100644 index 922eef32b..000000000 --- a/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md +++ /dev/null @@ -1,118 +0,0 @@ -# Decisions - -## D1: Do not use older multimodal Gemma artifacts as the default - -Decision: -- Use `gemma-4-e2b-it-fresh-mm` for V2 baseline work. - -Reason: -- Older artifacts reference a missing HF snapshot. - -## D2: Do not continue the simple media-step fallback - -Decision: -- Do not disable Gemma chunk prefill wholesale to force `lm_encoder_media_step`. - -Reason: -- It produced gibberish on old and fresh artifacts. - -## D3: Treat quality probes as acceptance, not root-cause tools - -Decision: -- Use quality probes to confirm behavior, but debug via component boundary comparison. - -Reason: -- Human text output identified the issue but did not localize the first bad boundary. - -## D4: First bad boundary is Gemma4 image preprocessing - -Decision: -- Treat E002 Gate B as the first proven divergence. -- Do not debug vision encoder, LM merge, or decoder logits until Gemma4 native preprocessing matches the HF/Python reference or a stronger reference refutes the HF/Python preprocessing tensor as the correct target. - -Reason: -- E002 Gate A proves prompt `input_ids` and image token spans match exactly between HF/Python native and C++ Cactus native. -- E002 Gate B proves `pixel_position_ids` match, but `pixel_values` differ immediately and outside tiny tolerance. -- The mismatch exists before any graph runtime, media merge, cache, or decoder behavior is involved. - -## D5: Localization is not acceptance - -Decision: -- Boundary localization is now treated only as a checkpoint. -- The investigation remains active until Gemma4 native works end to end across image-only, audio-only, and mixed image+audio, with controls and smoke still passing. - -Reason: -- The user explicitly raised the acceptance bar from identifying the cause to making the model fully work end to end. - -## D6: Image preprocessing is no longer the active boundary - -Decision: -- Treat Gemma4 image preprocessing parity as fixed in the current artifact set. - -Reason: -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. -- The recorded `pixel_values` max abs diff is `4.987010955859184e-07`, and mean abs diff is `2.2151478535820782e-07`. - -## D7: Image-only generation is currently accepted, but full multimodal is not - -Decision: -- Treat Gemma image-only as passing in the current artifact set. -- Do not treat the investigation as complete. - -Reason: -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records the response `The animal in the image is a golden monkey.` -- `07_ACCEPTANCE.md` allows monkey/orangutan/proboscis monkey class answers for image-only. -- Audio-only and mixed image+audio still return empty responses in the latest recorded artifacts. - -## D8: Superseded historical audio / mixed failure - -Decision: -- This was the correct decision at E021, but it is superseded by D10 and D11 after E023/E029/E030/E031. -- Do not return to prompt tweaks or decoder parameters unless a fresh rerun contradicts the passing artifacts. - -Reason: -- `artifacts/E020_gemma_audio_after_audio_retranspile/cactus_gemma_audio.json` records `response: ""` and `decode_tokens: 0`. -- `artifacts/E017_gemma_mixed_after_vision_gate/cactus_gemma_mixed.json` records `response: ""` and `decode_tokens: 0`. -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` records matching Cactus/Python audio feature output shapes but poor first-64-row agreement: cosine `0.36957991123199463`, mean abs diff `0.16708998382091522`. - -## D9: E021 localizes a boundary, not an internal cause - -Decision: -- Use E021 only to justify investigating inside the audio encoder path. -- Do not claim a specific audio op, weight, runtime kernel, or conversion policy is wrong until a smaller comparison identifies it. - -Reason: -- E021 compares full emitted audio features against a Python native-like reference and proves a mismatch at that boundary. -- It does not identify the first divergent node inside the audio encoder. - -## D10: Gemma4 audio bias tensors must stay FP16 - -Decision: -- Keep Gemma4 audio component bias tensors as FP16 fallback tensors. - -Reason: -- E022 localized the audio encoder divergence to the output projection bias handling. -- E023 shows the FP16 bias policy brings the emitted audio features close to the Python reference. -- Storing these biases as INT8 caused Cactus to add raw integer-like values instead of the expected FP16 bias values. - -## D11: Use dynamic Gemma4 merge for prompts with audio - -Decision: -- For Gemma4 prompts with audio, do not use the static full `lm_encoder` merge graph. -- Dynamically walk the actual prompt tokens, route text through `lm_encoder_step`, route image/audio token rows through `lm_encoder_media_step`, then feed the assembled tensors to `decoder_prefill_chunk`. - -Reason: -- E027 showed the static chunk path selected `` for audio-only and mixed prompts after the audio encoder boundary was fixed. -- The static `lm_encoder` graph can encode stale media span assumptions from capture time. -- E029/E030/E031 show the dynamic merge produces normal first tokens and passing end-to-end audio/mixed outputs. - -## D12: Do not carry vision changes from `justin/v2-cpp-fix` - -Decision: -- Treat `origin/justin/v2-cpp-fix` as already absorbed baseline infrastructure. -- Carry over only the later dynamic Gemma4 audio merge idea from `77f808ed`, adapted to the current branch structure. - -Reason: -- The latest `justin/v2-cpp-fix` commit is already in this branch line. -- Vision already has passing local artifacts. -- The relevant audio fix pattern is the dynamic media walk, not a vision-path change. diff --git a/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md b/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md deleted file mode 100644 index 5f2ec53d7..000000000 --- a/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md +++ /dev/null @@ -1,156 +0,0 @@ -# Fix Log - -Current status: V2 has recorded fixes in the worktree. Do not use the old `No V2 fixes yet` state. - -Rules: -- A code change must cite a supported hypothesis and experiment ID. -- A failed fix must be reverted or explicitly documented as intentionally retained. -- Each fix record must include before/after quality and nearest regression result. - -Historical fixes from before V2: -- Audio incremental decode changed to feed only the last token. -- Audio encoder mask changed to mark valid frames only. -- Fresh Gemma artifact reconverted as `gemma-4-e2b-it-fresh-mm`. -- Simple media-step fallback was tested and reverted. - -## Local investigation tooling - -2026-05-20: -- Added local-only E002 probe commands to `local_integration_tests/native_runner.cpp`: - - `e002-tokens` - - `e002-preprocess` -- Updated `local_integration_tests/CMakeLists.txt` so the local runner can include internal engine headers. -- Extended `scripts/run_probe.py` with `--case e002-boundary`. -- These are investigation artifacts only, not runtime fixes. - -## F1: Gemma4 image preprocessing parity - -Supported by: -- `artifacts/E003/preprocess_operation_compare.json` -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` - -Change: -- `cactus-engine/src/engine_image.cpp` now has a Pillow-compatible uint8 bilinear resize path used by `preprocess_gemma4_image`. - -Before: -- `artifacts/E002/image_preprocess_compare.json` recorded `pixel_values` max abs diff `0.1298471341896057` and mean abs diff `0.002023158910580417`. -- C++ pixel range extended outside `[0, 1]`. - -After: -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. -- `pixel_values` max abs diff is `4.987010955859184e-07`. -- `pixel_values` mean abs diff is `2.2151478535820782e-07`. - -## F2: Vision graph/runtime/transpile fixes recorded by later artifacts - -Supported by: -- `artifacts/E006_gateC_after_fp16_embedding_indices/vision_encoder_compare.json` -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` - -Changes visible in the worktree: -- `cactus-graph/src/ops_tensor.cpp` handles FP16 embedding indices in `compute_embedding_node`. -- `cactus-graph/src/ops_math.cpp` adds FP32 binary and reduction support used by scalar/math lowering paths. -- `python/cactus/transpile/lower.py` preserves tensor dtype in `_ensure_scalar_math_tensor`. -- `python/cactus/transpile/runtime_compat.py` preserves tensor dtype for scalar ops, `abs`, and `pow`. -- `python/cactus/transpile/model_adapters.py` updates Gemma4 native-like vision pooling/projection behavior. - -Evidence: -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` records FP16-close `pooler_output` agreement: cosine `0.9999955754245847`, mean abs diff `0.0015996834738693023`. -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records image-only output: `The animal in the image is a golden monkey.` - -Limit: -- This does not prove audio-only or mixed image+audio acceptance. - -## F3: Gemma4 vision projection conversion policy and active artifact adjustment - -Supported by: -- `artifacts/E013_gateC_after_fp16_projection_graph/vision_encoder_compare.json` -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` - -Changes visible in the worktree: -- `python/cactus/convert/model_adapters/policy.py` makes `model.embed_vision.embedding_projection.weight` an FP16 fallback with reason `vision embedding projection scale-sensitive`. -- `python/cactus/convert/model_adapters/adapters.py` no longer overrides that policy back to CQ. -- `python/cactus/convert/model_adapters/naming.py` and `python/cactus/convert/cactus_adapters/tensor_io.py` remove `embed_vision_proj` from Gemma4 divide-scale handling. -- `python/cactus/convert/tests/test_policy.py` now expects FP16 for the Gemma4 vision projection policy. - -Evidence: -- The latest recorded vision comparison improves to FP16-close agreement in `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json`. - -## F4: Audio full-tensor diagnostic instrumentation - -Supported by: -- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` - -Change: -- `local_integration_tests/native_runner.cpp` now emits full `audio_features.values` for `e002-audio-summary`. - -Evidence: -- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` contains full emitted `audio_features.values`. -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` compares those values against Python native-like audio features. - -Limit: -- This is diagnostic instrumentation only. -- It proves the emitted audio feature boundary mismatches, but not which internal audio encoder operation or weight is the first cause. - -## F5: Gemma4 audio bias FP16 policy and artifact adjustment - -Supported by: -- `artifacts/E022_audio_node_localization/` -- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` -- `artifacts/E024_retranspile_audio_fp16_bias/` - -Changes: -- `python/cactus/convert/model_adapters/policy.py` keeps Gemma4 audio component bias tensors in FP16. -- `python/cactus/convert/tests/test_policy.py` expects FP16 for Gemma4 audio bias policy. -- The active fresh Gemma artifact has an FP16 `audio_output_proj.bias` and regenerated `audio_encoder` graph artifacts. - -Before: -- E021 first 64 emitted audio rows had cosine `0.3695799112` and mean abs diff `0.16708998` versus Python. - -After: -- E023 first 64 emitted audio rows have cosine `0.9978210926` and mean abs diff `0.00645508` versus Python. - -## F6: Dynamic Gemma4 audio/media merge for chunked prefill - -Supported by: -- `artifacts/E027_first_token_after_audio_encoder_fix/` -- `artifacts/E029_first_token_after_dynamic_merge/` -- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` -- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` - -Changes: -- `cactus-engine/src/model.cpp` now builds Gemma4 LM encoder outputs dynamically for prompts containing audio. -- The dynamic path walks actual prompt tokens, uses `lm_encoder_step` for text, uses `lm_encoder_media_step` for image/audio token rows, and feeds the assembled outputs into `decoder_prefill_chunk`. -- `cactus-engine/src/engine.h` declares the dynamic helper. -- The existing static `lm_encoder` merge path remains for non-audio cases. - -Before: -- E025/E026 still returned empty audio-only and mixed responses after the audio encoder fix. -- E027 first-token probes selected `` for both audio-only and mixed prompts on the static chunk path. - -After: -- E029 first-token probes produce audio `Please` and mixed `This`. -- E030 audio-only produces coherent text responding to the spoken image question. -- E031 mixed image+audio produces `This image is of a golden monkey.` -- E032/E033/E034/E035 and smoke/policy tests record no local regression. - -## F7: Cleanup of generated local outputs - -Supported by: -- `artifacts/E036_cleanup_gemma_image/cactus_gemma_image.json` -- `artifacts/E037_cleanup_gemma_audio/cactus_gemma_audio.json` -- `artifacts/E038_cleanup_gemma_mixed/cactus_gemma_mixed.json` -- `artifacts/E039_cleanup_qwen_image/cactus_qwen_image.json` -- `artifacts/E040_cleanup_parakeet/` - -Changes: -- `.gitignore` now ignores local integration `.tmp` output, local Python bytecode, and local investigation artifact directories. -- Removed rebuildable local `.tmp` and `__pycache__` directories. -- Benchmark source/results under `benchmarks/` were not changed. - -Evidence: -- Policy tests passed after cleanup: `16 passed`. -- Smoke tests passed after cleanup: `7 passed, 2 deselected`. -- E036-E040 quality probes passed after cleanup. diff --git a/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md b/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md deleted file mode 100644 index 3e6887129..000000000 --- a/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md +++ /dev/null @@ -1,50 +0,0 @@ -# Acceptance - -## End-to-end gate - -Gemma4 native multimodal work is accepted only when the full user path works: -- Image-only Gemma correctly identifies `test_monkey.png` as a monkey/orangutan/proboscis monkey class animal. -- Audio-only Gemma shows it uses the paired spoken prompt coherently. -- Mixed image+audio Gemma answers the spoken question about `test_monkey.png` correctly. -- Any component boundary divergence found during debugging has a before/after artifact showing the fix. -- Qwen image control still works. -- Parakeet file and PCM controls still transcribe coherently. -- Smoke tests pass. - -Localization is not acceptance. It is a checkpoint that determines the next fix. - -## Vision gate - -Gemma native vision is accepted only when: -- HF and Cactus use the same image and semantically equivalent prompt. -- The first bad component boundary has been identified and fixed or ruled out. -- Cactus output identifies the monkey image as a monkey/orangutan/proboscis monkey class animal. -- Qwen image control still works. - -## Audio gate - -Gemma audio work starts after vision has a known-good path. A documented vision boundary failure is not enough to accept the mission. - -Audio is accepted only when: -- The paired WAV is verified by Parakeet. -- Gemma audio preprocessing and audio encoder behavior are compared against the Python/HF reference. -- Cactus output shows evidence of using the spoken content. - -## Mixed gate - -Mixed image+audio is accepted only after vision and audio have each passed their gates and the combined path works. - -Mixed image+audio must use the actual paired test assets: -- Image: `cactus-engine/tests/assets/test_monkey.png` -- Audio prompt: `what is in this image` -- Expected answer: a short coherent answer identifying the monkey/orangutan/proboscis monkey class. - -## Regression gate - -After any code fix: - -```bash -source ../cactus/venv/bin/activate -./cactus/build.sh -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s -``` diff --git a/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md b/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md deleted file mode 100644 index 9a27aa908..000000000 --- a/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md +++ /dev/null @@ -1,44 +0,0 @@ -# Handoff - -Status: local acceptance probes pass; leave uncommitted for user audit. - -Current proven state: -- Gemma4 image preprocessing parity is fixed. -- Gemma4 vision/image-only generation passes. -- Gemma4 audio encoder output is FP16-close to Python after the audio bias FP16 fix. -- Gemma4 audio-only generation is coherent. -- Gemma4 mixed image+audio answers the spoken question about `test_monkey.png` correctly. -- Qwen image, Parakeet file/PCM, policy tests, and smoke tests pass. - -## Key artifacts - -- Image preprocessing: `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` -- Vision encoder: `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` -- Audio encoder after bias fix: `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` -- First-token dynamic merge: `artifacts/E029_first_token_after_dynamic_merge/` -- Gemma audio-only: `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` -- Gemma mixed: `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` -- Gemma image regression: `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` -- Qwen regression: `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` -- Parakeet regression: `artifacts/E034_parakeet_regression_after_dynamic_merge/` -- Audio encoder unchanged after dynamic merge: `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` - -## Latest outputs - -- Gemma image-only: `The animal in the image is a golden monkey.` -- Gemma audio-only: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- Gemma mixed: `This image is of a golden monkey.` -- Qwen image: identifies a proboscis monkey. -- Parakeet file and PCM: `What is in this image?` - -## Tests - -- Build completed with `source ../cactus/venv/bin/activate && ./cactus/build.sh`. -- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q` -> `16 passed`. -- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s` -> `7 passed, 2 deselected`. - -## Review focus - -- Runtime: dynamic Gemma4 audio merge in `cactus-engine/src/model.cpp`. -- Policy: Gemma4 audio bias FP16 handling in `python/cactus/convert/model_adapters/policy.py`. -- Artifact changes: active fresh Gemma audio encoder graph/bias files were adjusted during the investigation. diff --git a/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md b/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md deleted file mode 100644 index da4b6605a..000000000 --- a/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md +++ /dev/null @@ -1,74 +0,0 @@ -# Status For Next Agent - -## One-line status - -Gemma4 image-only, audio-only, and mixed image+audio now pass local end-to-end probes after the audio bias FP16 fix and dynamic Gemma4 audio merge. Qwen, Parakeet, policy, and smoke controls also pass. - -## Current working state - -Repo: -- `/Users/noahcylich/Documents/Desert/cactus-v2-chunked-prefill` - -Branch: -- `v2-chunked-prefill` - -Do not commit or push unless the user explicitly asks. The worktree contains unrelated and related tracked changes plus local investigation artifacts. - -Default Gemma artifact: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -HF reference snapshot: -- `/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db` - -Assets: -- Image: `cactus-engine/tests/assets/test_monkey.png` -- Audio WAV: `local_integration_tests/.tmp/what_is_in_this_image.wav` -- Source MP3: `../cactus/what_is_in_this_image.mp3` - -## Fixes applied - -- Gemma4 audio component bias tensors are kept FP16. E023 shows the audio encoder boundary became FP16-close to Python after this. -- Gemma4 prompts containing audio now bypass the stale static full `lm_encoder` merge graph. The runtime dynamically walks the actual token sequence, routes text through `lm_encoder_step`, routes image/audio token rows through `lm_encoder_media_step`, assembles the LM encoder outputs, and feeds `decoder_prefill_chunk`. -- This keeps the existing static path for image-only and non-audio cases. - -## Latest proof artifacts - -- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json`: first 64 audio rows cosine `0.9978210926`, mean abs diff `0.00645508` versus Python. -- `artifacts/E029_first_token_after_dynamic_merge/`: audio first token `Please`, mixed first token `This`; no `` or immediate stop. -- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json`: audio-only response is coherent with the spoken prompt: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json`: mixed response: `This image is of a golden monkey.` -- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json`: image-only response: `The animal in the image is a golden monkey.` -- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json`: Qwen identifies a proboscis monkey. -- `artifacts/E034_parakeet_regression_after_dynamic_merge/`: Parakeet file and PCM both transcribe `What is in this image?` -- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json`: full emitted audio tensor matches E023 exactly, max diff `0`. - -## Latest commands run - -Required build: - -```bash -source ../cactus/venv/bin/activate -./cactus/build.sh -``` - -Policy: - -```bash -PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q -``` - -Result: -- `16 passed` - -Smoke: - -```bash -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s -``` - -Result: -- `7 passed, 2 deselected` - -## Remaining work - -No known local acceptance failure remains from the planned test matrix. Next step is user audit/review of the uncommitted changes. diff --git a/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md b/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md deleted file mode 100644 index 643082b7b..000000000 --- a/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md +++ /dev/null @@ -1,58 +0,0 @@ -# Continuation Plan - -## Goal - -Keep fresh Gemma4 Cactus native working end to end for image-only, audio-only, and mixed image+audio, while preserving Qwen, Parakeet, and smoke controls. - -The local acceptance plan now passes. This document is for review and any follow-up hardening, not for continuing a known active failure. - -## Current local result - -- Gemma image-only: passes, identifies `test_monkey.png` as a golden monkey. -- Gemma audio-only: passes, responds coherently to the spoken `what is in this image` prompt when no image is supplied. -- Gemma mixed image+audio: passes, answers `This image is of a golden monkey.` -- Qwen image control: passes, identifies a proboscis monkey. -- Parakeet file and PCM controls: pass, both transcribe `What is in this image?` -- Policy tests: pass, `16 passed`. -- Smoke tests: pass, `7 passed, 2 deselected`. - -## Important implementation decisions - -- Keep Gemma4 audio bias tensors FP16. -- For Gemma4 prompts with audio, use the dynamic media merge path rather than the static full `lm_encoder` merge graph. -- Do not carry additional vision behavior from `justin/v2-cpp-fix`; that branch is already in the ancestry and vision already passes. -- Keep non-audio/image-only behavior on the existing static chunk path. - -## Verification artifacts - -- Audio encoder boundary: `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` -- First-token dynamic merge: `artifacts/E029_first_token_after_dynamic_merge/` -- Gemma audio-only: `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` -- Gemma mixed: `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` -- Gemma image regression: `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` -- Qwen regression: `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` -- Parakeet regression: `artifacts/E034_parakeet_regression_after_dynamic_merge/` -- Audio encoder unchanged after dynamic merge: `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` - -## If further testing is requested - -Run the same sequence under the required build rule: - -```bash -source ../cactus/venv/bin/activate -./cactus/build.sh -``` - -Then rerun: - -```bash -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-audio --experiment -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-mixed --experiment -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-image --experiment -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-qwen-image --experiment -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case parakeet-audio --experiment -PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s -``` - -Do not commit or push until the user audits the changes. diff --git a/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py b/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py deleted file mode 100644 index cb12337b1..000000000 --- a/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py +++ /dev/null @@ -1,785 +0,0 @@ -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import subprocess -import sys -import time -from pathlib import Path -from typing import Any - - -REPO_ROOT = Path(__file__).resolve().parents[3] -INVESTIGATION_ROOT = Path(__file__).resolve().parents[1] -DEFAULT_WEIGHTS_ROOT = REPO_ROOT.parent / "cactus" / "weights" -ASSET_DIR = REPO_ROOT / "cactus-engine" / "tests" / "assets" -TMP_DIR = REPO_ROOT / "local_integration_tests" / ".tmp" - - -def run(cmd: list[str], *, timeout: int = 300) -> subprocess.CompletedProcess[str]: - env = os.environ.copy() - env["CACTUS_NO_CLOUD_TELE"] = "1" - return subprocess.run( - cmd, - cwd=REPO_ROOT, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - check=False, - ) - - -def write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - - -def command_record(case: str, command: list[str], completed: subprocess.CompletedProcess[str]) -> dict[str, Any]: - parsed: Any = None - if completed.stdout.strip().startswith("{"): - try: - parsed = json.loads(completed.stdout) - except json.JSONDecodeError: - parsed = None - return { - "case": case, - "command": command, - "returncode": completed.returncode, - "stdout": completed.stdout, - "stderr": completed.stderr, - "parsed_stdout": parsed, - } - - -def ensure_runner() -> Path: - build_dir = TMP_DIR / "build" - runner = build_dir / "integration_runner" - sources = [ - REPO_ROOT / "local_integration_tests" / "native_runner.cpp", - REPO_ROOT / "local_integration_tests" / "CMakeLists.txt", - REPO_ROOT / "cactus" / "build" / "libcactus.a", - ] - if runner.exists() and all(runner.stat().st_mtime >= path.stat().st_mtime for path in sources if path.exists()): - return runner - cmake = run( - [ - "cmake", - "-S", - str(REPO_ROOT / "local_integration_tests"), - "-B", - str(build_dir), - f"-DCACTUS_REPO_ROOT={REPO_ROOT}", - "-DCMAKE_BUILD_TYPE=Release", - ], - timeout=120, - ) - if cmake.returncode != 0: - raise RuntimeError(cmake.stderr or cmake.stdout) - build = run(["cmake", "--build", str(build_dir), "--target", "integration_runner", "-j", str(os.cpu_count() or 4)], timeout=120) - if build.returncode != 0: - raise RuntimeError(build.stderr or build.stdout) - if not runner.exists(): - raise RuntimeError(f"integration runner was not built: {runner}") - return runner - - -def ensure_question_wav() -> Path: - wav = TMP_DIR / "what_is_in_this_image.wav" - if wav.exists(): - return wav - mp3 = REPO_ROOT.parent / "cactus" / "what_is_in_this_image.mp3" - if not mp3.exists(): - raise RuntimeError(f"missing paired mp3: {mp3}") - wav.parent.mkdir(parents=True, exist_ok=True) - completed = run( - [ - "ffmpeg", - "-y", - "-hide_banner", - "-loglevel", - "error", - "-i", - str(mp3), - "-ac", - "1", - "-ar", - "16000", - "-sample_fmt", - "s16", - str(wav), - ], - timeout=120, - ) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - return wav - - -def cactus_complete(model: Path, prompt: str, image: Path | None = None, audio: Path | None = None, max_tokens: int = 60) -> dict[str, Any]: - runner = ensure_runner() - command: list[str] = [ - str(runner), - "complete", - "--model", - str(model), - "--prompt", - prompt, - "--max-tokens", - str(max_tokens), - ] - if image is not None: - command.extend(["--image", str(image)]) - if audio is not None: - command.extend(["--audio", str(audio)]) - return command_record("cactus_complete", command, run(command, timeout=360)) - - -def native_json(command: list[str], *, timeout: int = 360) -> dict[str, Any]: - completed = run(command, timeout=timeout) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - try: - return json.loads(completed.stdout) - except json.JSONDecodeError as exc: - raise RuntimeError(f"native command did not return JSON: {command}\n{completed.stdout[:1000]}") from exc - - -def parakeet_transcribe(model: Path, audio: Path, pcm: bool = False) -> dict[str, Any]: - runner = ensure_runner() - command = [str(runner), "transcribe-pcm" if pcm else "transcribe", "--model", str(model), "--audio", str(audio)] - return command_record("parakeet_transcribe_pcm" if pcm else "parakeet_transcribe", command, run(command, timeout=240)) - - -def hf_gemma_image(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: - script = f""" -import json -import torch -from pathlib import Path -from transformers import AutoProcessor, AutoModelForImageTextToText - -model_dir = Path({str(model_dir)!r}) -image_path = Path({str(image)!r}) -prompt = {prompt!r} -processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) -model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='auto') -messages = [{{'role': 'user', 'content': [{{'type': 'image', 'image': str(image_path)}}, {{'type': 'text', 'text': prompt}}]}}] -inputs = processor.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors='pt') -inputs = {{k: v.to(model.device) if hasattr(v, 'to') else v for k, v in inputs.items()}} -with torch.no_grad(): - out = model.generate(**inputs, max_new_tokens=40, do_sample=False) -text = processor.decode(out[0][inputs['input_ids'].shape[-1]:], skip_special_tokens=True) -print(json.dumps({{'response': text, 'input_tokens': int(inputs['input_ids'].shape[-1])}})) -""" - command = [sys.executable, "-c", script] - return command_record("hf_gemma_image", command, run(command, timeout=360)) - - -def hf_gemma_native_image(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: - script = f""" -import json -import torch -from pathlib import Path -from transformers import AutoProcessor, AutoModelForImageTextToText -from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs - -model_dir = Path({str(model_dir)!r}) -image_path = Path({str(image)!r}) -prompt = {prompt!r} -processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) -prepared = prepare_gemma4_multimodal_inputs( - processor, - prompt=prompt, - image_files=(str(image_path),), - audio_file=None, - torch_dtype=torch.float16, - use_gemma4_chat_template=True, -) -model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='auto') -inputs = {{}} -summaries = {{}} -for name, tensor in zip(prepared.names, prepared.tensors): - key = 'image_position_ids' if name == 'pixel_position_ids' else name - inputs[key] = tensor.to(model.device) - summaries[name] = {{ - 'shape': list(tensor.shape), - 'dtype': str(tensor.dtype), - 'min': float(tensor.float().min().item()) if tensor.numel() else 0.0, - 'max': float(tensor.float().max().item()) if tensor.numel() else 0.0, - 'mean': float(tensor.float().mean().item()) if tensor.numel() else 0.0, - }} -with torch.no_grad(): - out = model.generate(**inputs, max_new_tokens=40, do_sample=False) -text = processor.decode(out[0][inputs['input_ids'].shape[-1]:], skip_special_tokens=True) -input_ids = prepared.tensors[prepared.names.index('input_ids')] -print(json.dumps({{ - 'response': text, - 'input_tokens': int(input_ids.shape[-1]), - 'input_shapes': prepared.metadata.get('input_shapes'), - 'processor_prompt_prefix': str(prepared.metadata.get('processor_prompt'))[:240], - 'tensor_summaries': summaries, -}})) -""" - command = [sys.executable, "-c", script] - return command_record("hf_gemma_native_image", command, run(command, timeout=360)) - - -def hf_gemma_e002_inputs(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: - script = f""" -import hashlib -import json -import torch -from pathlib import Path -from transformers import AutoProcessor -from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs - -model_dir = Path({str(model_dir)!r}) -image_path = Path({str(image)!r}) -prompt = {prompt!r} -processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) -prepared = prepare_gemma4_multimodal_inputs( - processor, - prompt=prompt, - image_files=(str(image_path),), - audio_file=None, - torch_dtype=torch.float32, - use_gemma4_chat_template=True, -) -by_name = {{name: tensor.detach().cpu() for name, tensor in zip(prepared.names, prepared.tensors)}} -input_ids = by_name['input_ids'].reshape(-1).to(torch.int64) -pixel_values = by_name['pixel_values'].to(torch.float32).contiguous() -pixel_position_ids = by_name['pixel_position_ids'].reshape(-1).to(torch.int64).contiguous() -tokenizer = getattr(processor, 'tokenizer', processor) -image_token = getattr(processor, 'image_token', None) -image_token_id = tokenizer.convert_tokens_to_ids(image_token) if image_token is not None and hasattr(tokenizer, 'convert_tokens_to_ids') else -1 -ids = input_ids.tolist() -image_positions = [i for i, tok in enumerate(ids) if tok == image_token_id] -processor_prompt = str(prepared.metadata.get('processor_prompt')) -assistant_generation_start = len(ids) -if processor_prompt.endswith('<|turn>model\\n'): - no_generation_prompt = processor_prompt[:-len('<|turn>model\\n')] - no_gen = tokenizer(no_generation_prompt, return_tensors='pt', add_special_tokens=False)['input_ids'].reshape(-1) - assistant_generation_start = int(no_gen.numel()) -valid_positions = int(((by_name['pixel_position_ids'] != -1).any(dim=-1)).sum().item()) -pixel_np = pixel_values.numpy() -pos_np = by_name['pixel_position_ids'].to(torch.int64).contiguous().numpy() -payload = {{ - 'token_count': len(ids), - 'assistant_generation_start': assistant_generation_start, - 'image_token_id': int(image_token_id), - 'image_token_start': int(image_positions[0]) if image_positions else -1, - 'image_token_count': len(image_positions), - 'first_20': ids[:20], - 'last_20': ids[-20:], - 'input_ids': ids, - 'prompt_prefix': processor_prompt[:320], - 'pixel_values_shape': list(pixel_values.shape), - 'pixel_position_ids_shape': list(by_name['pixel_position_ids'].shape), - 'pixel_values_min': float(pixel_values.min().item()), - 'pixel_values_max': float(pixel_values.max().item()), - 'pixel_values_mean': float(pixel_values.mean().item()), - 'pixel_values_sha256': hashlib.sha256(pixel_np.tobytes()).hexdigest(), - 'pixel_position_ids_sha256': hashlib.sha256(pos_np.tobytes()).hexdigest(), - 'valid_position_count': valid_positions, - 'first_pixel_values': pixel_values.reshape(-1)[:32].tolist(), - 'first_position_rows_flat': pixel_position_ids[:32].tolist(), - 'pixel_values': pixel_values.reshape(-1).tolist(), - 'pixel_position_ids': pixel_position_ids.tolist(), -}} -print(json.dumps(payload)) -""" - command = [sys.executable, "-c", script] - completed = run(command, timeout=360) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - return json.loads(completed.stdout) - - -def e002_cactus_tokens(model: Path, image: Path, prompt: str) -> dict[str, Any]: - runner = ensure_runner() - command = [ - str(runner), - "e002-tokens", - "--model", - str(model), - "--prompt", - prompt, - "--image", - str(image), - ] - return native_json(command, timeout=360) - - -def e002_cactus_preprocess(model: Path, image: Path) -> dict[str, Any]: - runner = ensure_runner() - command = [ - str(runner), - "e002-preprocess", - "--model", - str(model), - "--image", - str(image), - ] - return native_json(command, timeout=360) - - -def e003_cactus_preprocess_variants(model: Path, image: Path) -> dict[str, Any]: - runner = ensure_runner() - command = [ - str(runner), - "e003-preprocess-variants", - "--model", - str(model), - "--image", - str(image), - ] - return native_json(command, timeout=360) - - -def e002_cactus_vision(model: Path, image: Path) -> dict[str, Any]: - runner = ensure_runner() - command = [ - str(runner), - "e002-vision", - "--model", - str(model), - "--image", - str(image), - "--prompt", - "What animal is in this image? Reply in one complete short sentence.", - ] - return native_json(command, timeout=360) - - -def hf_gemma_vision_features(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: - script = f""" -import json -import torch -from pathlib import Path -from transformers import AutoProcessor, AutoModelForImageTextToText -from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs - -model_dir = Path({str(model_dir)!r}) -image_path = Path({str(image)!r}) -prompt = {prompt!r} -processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) -prepared = prepare_gemma4_multimodal_inputs( - processor, - prompt=prompt, - image_files=(str(image_path),), - audio_file=None, - torch_dtype=torch.float16, - use_gemma4_chat_template=True, -) -by_name = {{name: tensor for name, tensor in zip(prepared.names, prepared.tensors)}} -model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='cpu') -pixel_values = by_name['pixel_values'].to(dtype=torch.float16) -position_ids = by_name['pixel_position_ids'].to(dtype=torch.long) -with torch.no_grad(): - outputs = model.get_image_features(pixel_values=pixel_values, image_position_ids=position_ids) -candidates = {{}} -for name, tensor in ( - ('last_hidden_state', getattr(outputs, 'last_hidden_state', None)), - ('pooler_output', getattr(outputs, 'pooler_output', None)), -): - if tensor is None: - continue - flat = tensor.detach().float().cpu().reshape(-1) - candidates[name] = {{ - 'shape': list(tensor.shape), - 'dtype': str(tensor.dtype), - 'count': int(flat.numel()), - 'min': float(flat.min().item()) if flat.numel() else 0.0, - 'max': float(flat.max().item()) if flat.numel() else 0.0, - 'mean': float(flat.mean().item()) if flat.numel() else 0.0, - 'l2': float(torch.linalg.vector_norm(flat).item()) if flat.numel() else 0.0, - 'first_values': flat[:32].tolist(), - 'values': flat.tolist(), - }} -print(json.dumps({{'features': candidates}})) -""" - completed = run([sys.executable, "-c", script], timeout=420) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - return json.loads(completed.stdout) - - -def first_mismatch(left: list[Any], right: list[Any]) -> int | None: - for idx, (lval, rval) in enumerate(zip(left, right)): - if lval != rval: - return idx - if len(left) != len(right): - return min(len(left), len(right)) - return None - - -def sha256_float_list(values: list[float]) -> str: - import array - - arr = array.array("f", values) - return hashlib.sha256(arr.tobytes()).hexdigest() - - -def run_e002_boundary(out_dir: Path, weights_root: Path) -> None: - image = ASSET_DIR / "test_monkey.png" - prompt = "What animal is in this image? Reply in one complete short sentence." - fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" - hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") - - hf = hf_gemma_e002_inputs(hf_snapshot, image, prompt) - cactus_tokens = e002_cactus_tokens(fresh_gemma, image, prompt) - write_json(out_dir / "hf_native_inputs.json", hf) - write_json(out_dir / "cactus_token_probe.json", cactus_tokens) - - token_mismatch = first_mismatch(hf["input_ids"], cactus_tokens["input_ids"]) - token_compare = { - "gate": "prompt_and_token_spans", - "status": "pass" if token_mismatch is None else "fail", - "input_ids_match": token_mismatch is None, - "first_mismatch_index": token_mismatch, - "hf": {key: hf[key] for key in ("token_count", "assistant_generation_start", "image_token_id", "image_token_start", "image_token_count", "first_20", "last_20", "prompt_prefix")}, - "cactus": {key: cactus_tokens[key] for key in ("token_count", "assistant_generation_start", "image_token_id", "image_token_start", "image_token_count", "first_20", "last_20", "prompt_prefix")}, - } - if token_mismatch is not None: - lo = max(0, token_mismatch - 8) - hi = token_mismatch + 9 - token_compare["mismatch_window"] = { - "hf": hf["input_ids"][lo:hi], - "cactus": cactus_tokens["input_ids"][lo:hi], - "start": lo, - } - write_json(out_dir / "token_spans.json", token_compare) - if token_mismatch is not None: - return - - cactus_preprocess = e002_cactus_preprocess(fresh_gemma, image) - write_json(out_dir / "cactus_preprocess.json", cactus_preprocess) - hf_values = hf["pixel_values"] - cactus_values = cactus_preprocess["pixel_values"] - hf_positions = hf["pixel_position_ids"] - cactus_positions = cactus_preprocess["pixel_position_ids"] - if len(hf_values) != len(cactus_values): - max_abs_diff = None - mean_abs_diff = None - first_pixel_mismatch = min(len(hf_values), len(cactus_values)) - else: - diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, cactus_values)] - max_abs_diff = max(diffs) if diffs else 0.0 - mean_abs_diff = sum(diffs) / len(diffs) if diffs else 0.0 - first_pixel_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-5), None) - position_mismatch = first_mismatch(hf_positions, cactus_positions) - preprocess_pass = ( - hf["pixel_values_shape"] == cactus_preprocess["pixel_values_shape"] - and hf["pixel_position_ids_shape"] == cactus_preprocess["pixel_position_ids_shape"] - and position_mismatch is None - and max_abs_diff is not None - and max_abs_diff <= 1e-5 - ) - compare = { - "gate": "image_preprocessing_tensors", - "status": "pass" if preprocess_pass else "fail", - "pixel_values_shape_match": hf["pixel_values_shape"] == cactus_preprocess["pixel_values_shape"], - "pixel_position_ids_shape_match": hf["pixel_position_ids_shape"] == cactus_preprocess["pixel_position_ids_shape"], - "pixel_position_ids_match": position_mismatch is None, - "first_position_mismatch_index": position_mismatch, - "pixel_values_max_abs_diff": max_abs_diff, - "pixel_values_mean_abs_diff": mean_abs_diff, - "first_pixel_value_mismatch_index_at_1e-5": first_pixel_mismatch, - "hf_summary": {key: hf[key] for key in ("pixel_values_shape", "pixel_position_ids_shape", "pixel_values_min", "pixel_values_max", "pixel_values_mean", "pixel_values_sha256", "pixel_position_ids_sha256", "valid_position_count", "first_pixel_values", "first_position_rows_flat")}, - "cactus_summary": {key: cactus_preprocess[key] for key in ("pixel_values_shape", "pixel_position_ids_shape", "pixel_values_min", "pixel_values_max", "pixel_values_mean", "pixel_values_hash", "pixel_position_ids_hash", "valid_position_count", "first_pixel_values", "first_position_rows_flat")}, - } - if first_pixel_mismatch is not None: - lo = max(0, first_pixel_mismatch - 8) - hi = first_pixel_mismatch + 9 - compare["pixel_mismatch_window"] = { - "start": lo, - "hf": hf_values[lo:hi], - "cactus": cactus_values[lo:hi], - } - if position_mismatch is not None: - lo = max(0, position_mismatch - 8) - hi = position_mismatch + 9 - compare["position_mismatch_window"] = { - "start": lo, - "hf": hf_positions[lo:hi], - "cactus": cactus_positions[lo:hi], - } - write_json(out_dir / "image_preprocess_compare.json", compare) - - -def run_e003_preprocess_isolation(out_dir: Path, weights_root: Path) -> None: - image = ASSET_DIR / "test_monkey.png" - prompt = "What animal is in this image? Reply in one complete short sentence." - fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" - hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") - - hf = hf_gemma_e002_inputs(hf_snapshot, image, prompt) - variants = e003_cactus_preprocess_variants(fresh_gemma, image) - write_json(out_dir / "hf_native_inputs.json", hf) - write_json(out_dir / "cactus_resize_variants.json", variants) - - hf_values = hf["pixel_values"] - comparisons: dict[str, Any] = {} - best_name: str | None = None - best_max: float | None = None - for name, variant in variants["variants"].items(): - values = variant["pixel_values"] - if len(values) != len(hf_values): - comparison = { - "status": "fail", - "length_match": False, - "hf_length": len(hf_values), - "variant_length": len(values), - } - else: - diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, values)] - max_abs = max(diffs) if diffs else 0.0 - mean_abs = sum(diffs) / len(diffs) if diffs else 0.0 - first_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-5), None) - comparison = { - "status": "pass" if max_abs <= 1e-5 else "fail", - "length_match": True, - "max_abs_diff": max_abs, - "mean_abs_diff": mean_abs, - "first_mismatch_index_at_1e-5": first_mismatch, - "variant_summary": { - key: variant[key] - for key in ( - "resized_min", - "resized_max", - "first_resized_values", - "pixel_values_min", - "pixel_values_max", - "pixel_values_mean", - "first_pixel_values", - ) - }, - } - if first_mismatch is not None: - lo = max(0, first_mismatch - 8) - hi = first_mismatch + 9 - comparison["mismatch_window"] = { - "start": lo, - "hf": hf_values[lo:hi], - "variant": values[lo:hi], - } - if best_max is None or max_abs < best_max: - best_name = name - best_max = max_abs - comparisons[name] = comparison - - write_json( - out_dir / "preprocess_operation_compare.json", - { - "gate": "e003_preprocess_operation_isolation", - "source_width": variants["source_width"], - "source_height": variants["source_height"], - "target_width": variants["target_width"], - "target_height": variants["target_height"], - "patch_size": variants["patch_size"], - "pooling_kernel_size": variants["pooling_kernel_size"], - "max_patches": variants["max_patches"], - "patch_dim": variants["patch_dim"], - "rescale_factor": variants["rescale_factor"], - "hf_summary": { - key: hf[key] - for key in ( - "pixel_values_shape", - "pixel_values_min", - "pixel_values_max", - "pixel_values_mean", - "first_pixel_values", - ) - }, - "best_variant": best_name, - "best_variant_max_abs_diff": best_max, - "comparisons": comparisons, - }, - ) - - -def compare_float_vectors(hf_values: list[float], cactus_values: list[float]) -> dict[str, Any]: - if len(hf_values) != len(cactus_values): - return { - "status": "fail", - "length_match": False, - "hf_length": len(hf_values), - "cactus_length": len(cactus_values), - } - cactus_nonfinite = sum(1 for value in cactus_values if value is None) - hf_nonfinite = sum(1 for value in hf_values if value is None) - if cactus_nonfinite or hf_nonfinite: - return { - "status": "fail", - "length_match": True, - "hf_nonfinite_count": hf_nonfinite, - "cactus_nonfinite_count": cactus_nonfinite, - "first_nonfinite_index": next( - (i for i, (a, b) in enumerate(zip(hf_values, cactus_values)) if a is None or b is None), - None, - ), - } - diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, cactus_values)] - dot = sum(float(a) * float(b) for a, b in zip(hf_values, cactus_values)) - hf_norm = sum(float(a) * float(a) for a in hf_values) ** 0.5 - cactus_norm = sum(float(b) * float(b) for b in cactus_values) ** 0.5 - max_abs = max(diffs) if diffs else 0.0 - mean_abs = sum(diffs) / len(diffs) if diffs else 0.0 - first_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-3), None) - result: dict[str, Any] = { - "status": "pass" if max_abs <= 1e-2 and mean_abs <= 1e-3 else "fail", - "length_match": True, - "max_abs_diff": max_abs, - "mean_abs_diff": mean_abs, - "cosine_similarity": dot / (hf_norm * cactus_norm) if hf_norm and cactus_norm else 0.0, - "first_mismatch_index_at_1e-3": first_mismatch, - } - if first_mismatch is not None: - lo = max(0, first_mismatch - 8) - hi = first_mismatch + 9 - result["mismatch_window"] = { - "start": lo, - "hf": hf_values[lo:hi], - "cactus": cactus_values[lo:hi], - } - return result - - -def run_e002_vision_boundary(out_dir: Path, weights_root: Path) -> None: - image = ASSET_DIR / "test_monkey.png" - prompt = "What animal is in this image? Reply in one complete short sentence." - fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" - hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") - - hf = hf_gemma_vision_features(hf_snapshot, image, prompt) - cactus = e002_cactus_vision(fresh_gemma, image) - write_json(out_dir / "hf_vision_features.json", hf) - write_json(out_dir / "cactus_vision_features.json", cactus) - - comparisons: dict[str, Any] = {} - for cactus_name, cactus_feature in cactus["features"].items(): - comparisons[cactus_name] = {} - for hf_name, hf_feature in hf["features"].items(): - comparison = compare_float_vectors(hf_feature["values"], cactus_feature["values"]) - comparison["hf_shape"] = hf_feature["shape"] - comparison["cactus_shape"] = cactus_feature["shape"] - comparison["hf_summary"] = {key: hf_feature[key] for key in ("dtype", "count", "min", "max", "mean", "l2", "first_values")} - comparison["cactus_summary"] = {key: cactus_feature[key] for key in ("precision", "count", "finite_count", "nonfinite_count", "min", "max", "mean", "l2", "first_values")} - comparisons[cactus_name][hf_name] = comparison - - write_json( - out_dir / "vision_encoder_compare.json", - { - "gate": "vision_encoder_output", - "comparisons": comparisons, - }, - ) - - -def manifest_summary(path: Path) -> dict[str, Any]: - manifest = path / "components" / "manifest.json" - config = path / "config.txt" - result: dict[str, Any] = {"path": str(path), "manifest_exists": manifest.exists(), "config_exists": config.exists()} - if manifest.exists(): - data = json.loads(manifest.read_text()) - result.update( - { - "family": data.get("family"), - "task": data.get("task"), - "model_source": data.get("model_source"), - "component_order": data.get("component_order"), - "components": [ - { - "component": component.get("component"), - "logical_inputs": component.get("logical_inputs"), - "logical_outputs": component.get("logical_outputs"), - } - for component in data.get("components", []) - ], - } - ) - return result - - -def run_case(case: str, out_dir: Path, weights_root: Path) -> None: - image = ASSET_DIR / "test_monkey.png" - prompt = "What animal is in this image? Reply in one complete short sentence." - fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" - qwen = weights_root / "qwen3-vl-2b-instruct-reconvert" - parakeet = weights_root / "parakeet-tdt-0.6b-v3-transpiled" - hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") - - if case == "inventory": - write_json(out_dir / "inventory.json", {"gemma_fresh": manifest_summary(fresh_gemma)}) - return - if case == "hf-gemma-image": - write_json(out_dir / "hf_gemma_image.json", hf_gemma_image(hf_snapshot, image, prompt)) - return - if case == "hf-gemma-native-image": - write_json(out_dir / "hf_gemma_native_image.json", hf_gemma_native_image(hf_snapshot, image, prompt)) - return - if case == "cactus-gemma-image": - write_json(out_dir / "cactus_gemma_image.json", cactus_complete(fresh_gemma, prompt, image=image)) - return - if case == "cactus-gemma-audio": - wav = ensure_question_wav() - audio_prompt = "Listen to the audio and answer briefly." - write_json(out_dir / "cactus_gemma_audio.json", cactus_complete(fresh_gemma, audio_prompt, audio=wav)) - return - if case == "cactus-gemma-mixed": - wav = ensure_question_wav() - mixed_prompt = "Answer the spoken question using the provided image. Reply in one short sentence." - write_json(out_dir / "cactus_gemma_mixed.json", cactus_complete(fresh_gemma, mixed_prompt, image=image, audio=wav)) - return - if case == "cactus-qwen-image": - write_json(out_dir / "cactus_qwen_image.json", cactus_complete(qwen, prompt, image=image, max_tokens=40)) - return - if case == "parakeet-audio": - wav = ensure_question_wav() - write_json(out_dir / "parakeet_audio.json", parakeet_transcribe(parakeet, wav)) - write_json(out_dir / "parakeet_audio_pcm.json", parakeet_transcribe(parakeet, wav, pcm=True)) - return - if case == "all-baseline": - for subcase in ("inventory", "hf-gemma-image", "hf-gemma-native-image", "cactus-gemma-image", "cactus-qwen-image", "parakeet-audio"): - run_case(subcase, out_dir, weights_root) - return - if case == "e002-boundary": - run_e002_boundary(out_dir, weights_root) - return - if case == "e003-preprocess-isolation": - run_e003_preprocess_isolation(out_dir, weights_root) - return - if case == "e002-vision-boundary": - run_e002_vision_boundary(out_dir, weights_root) - return - raise ValueError(f"unknown case: {case}") - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--case", required=True) - parser.add_argument("--experiment", required=True) - parser.add_argument("--weights-root", default=str(DEFAULT_WEIGHTS_ROOT)) - args = parser.parse_args() - - out_dir = INVESTIGATION_ROOT / "artifacts" / args.experiment - out_dir.mkdir(parents=True, exist_ok=True) - write_json( - out_dir / "run_metadata.json", - { - "case": args.case, - "experiment": args.experiment, - "repo_root": str(REPO_ROOT), - "weights_root": str(Path(args.weights_root).resolve()), - "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"), - }, - ) - run_case(args.case, out_dir, Path(args.weights_root).resolve()) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 0f14f03af70cad74ea59c87ddb7699fb703a4c77 Mon Sep 17 00:00:00 2001 From: justinl66 Date: Fri, 29 May 2026 08:22:47 -0700 Subject: [PATCH 7/8] remove comments --- cactus-engine/src/model.cpp | 4 - cactus-engine/src/model_npu.cpp | 6 - python/cactus/transpile/hf_model.py | 10 -- python/cactus/transpile/npu/__init__.py | 10 -- python/cactus/transpile/npu/audio.py | 30 ----- .../transpile/npu/coremltools_patches.py | 107 ------------------ python/cactus/transpile/npu/pipeline.py | 47 +------- python/cactus/transpile/npu/vision.py | 30 ----- 8 files changed, 4 insertions(+), 240 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 40a3007ce..d26aff2cd 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2177,10 +2177,6 @@ std::vector Model::transcribe_parakeet_tdt(const std::vector& a write_typed_buffer(feat_buf, feat_desc.precision, transposed.data(), transposed.size() * sizeof(float), Precision::FP32); - // Try NPU audio encoder first; fall back to CPU graph on failure. - // The NPU mlpackage has a fixed input window (e.g. 500 frames -> 63 hidden frames), - // so we chunk the input and concatenate chunk outputs into a buffer sized to the - // CPU graph's expected T (subsampling preserves the chunk:hidden ratio). std::vector<__fp16> npu_hidden_storage; bool used_npu = false; size_t npu_hidden_T = 0; diff --git a/cactus-engine/src/model_npu.cpp b/cactus-engine/src/model_npu.cpp index dfa35afd6..0661f055c 100644 --- a/cactus-engine/src/model_npu.cpp +++ b/cactus-engine/src/model_npu.cpp @@ -64,8 +64,6 @@ bool Model::audio_encode_via_npu(const std::vector& audio_features) { input_fp16.data(), output_fp16.data(), input_shape, "x", "encoded"); if (written == 0) return false; - // Copy NPU output into the CPU audio_encoder's primary output buffer so - // downstream code (dynamic walker / media_features_) sees the same path. for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { const std::string& name = audio_encoder_->logical_outputs[i]; @@ -130,9 +128,6 @@ bool Model::vision_encode_via_npu(const std::vector& pixel_values) { input_fp16.data(), output_fp16.data(), input_shape, "x", "encoded"); if (written == 0) return false; - // Append NPU output into the media_features_ slot, matching the CPU - // vision_encoder path which appends per-image features (the chunk-prefill - // loop calls this once per image; subsequent calls grow the slot). for (size_t i = 0; i < vision_encoder_->output_node_ids.size() && i < vision_encoder_->logical_outputs.size(); ++i) { const std::string& name = vision_encoder_->logical_outputs[i]; @@ -163,7 +158,6 @@ bool Model::vision_encode_via_npu(const std::vector& pixel_values) { if (shape_it == media_feature_shapes_.end() || shape_it->second.empty()) { media_feature_shapes_[name] = npu_shape; } else if (npu_shape.size() >= 2 && shape_it->second.size() == npu_shape.size()) { - // Extend the row-count dim (second-to-last), mirroring CPU path. shape_it->second[shape_it->second.size() - 2] += npu_shape[npu_shape.size() - 2]; } media_feature_precisions_[name] = cpu_prec; diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index 7b381b7ba..3e3adbb36 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -545,9 +545,6 @@ def _run_component_pipeline_transpile( if weights_dir: print(f"weights_dir={weights_dir}") - # NPU emit runs BEFORE cactus capture so peak memory is just HF model + - # per-encoder coremltools temp, not HF model + all captured graphs. - # Encoders only — text prefill is intentionally not on NPU. npu_enabled = bool(getattr(args, "npu", False)) npu_quantize = getattr(args, "npu_quantize", None) # legacy override npu_audio_quantize = getattr(args, "npu_audio_quantize", None) # default 8 if neither set @@ -563,8 +560,6 @@ def _run_component_pipeline_transpile( audio_quantize_bits=npu_audio_quantize, vision_quantize_bits=npu_vision_quantize, ) - # Drop coremltools-side intermediates before the heavy cactus capture - # below — both phases compete for the same working set on a 16 GB host. import gc as _gc _gc.collect() @@ -699,8 +694,6 @@ def _run_component_pipeline_transpile( f"saved_optimized_component_ir_{component}=" f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" ) - # NPU mlpackages were already emitted at the top of this function, - # before cactus capture, to keep peak memory bounded. component_manifest_path = _write_component_bundle( artifact_dir=artifact_dir, @@ -3382,9 +3375,6 @@ def main() -> int: f"saved_optimized_component_ir_{component}=" f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" ) - # NPU emit (encoders only — text prefill is intentionally not on NPU) - # is handled in `_run_component_pipeline_transpile`; this legacy - # single-graph path does not exercise the multimodal adapters. component_manifest_path = _write_component_bundle( artifact_dir=artifact_dir, diff --git a/python/cactus/transpile/npu/__init__.py b/python/cactus/transpile/npu/__init__.py index a1ee1c1e2..e2fc7da08 100644 --- a/python/cactus/transpile/npu/__init__.py +++ b/python/cactus/transpile/npu/__init__.py @@ -1,13 +1,3 @@ -"""NPU transpiler pipeline. - -Emits CoreML `.mlpackage`s for Apple Neural Engine **audio + vision encoders** -alongside the cactus graph transpiler's output. Text-decoder prefill is -intentionally not on NPU — CPU prefill is the supported path. - -Entry point: `run_encoder_pipeline`, invoked from -`hf_model._run_component_pipeline_transpile` when `--npu` is passed. -""" - from .pipeline import run_encoder_pipeline __all__ = ["run_encoder_pipeline"] diff --git a/python/cactus/transpile/npu/audio.py b/python/cactus/transpile/npu/audio.py index c62a1149e..dab7d67cc 100644 --- a/python/cactus/transpile/npu/audio.py +++ b/python/cactus/transpile/npu/audio.py @@ -1,13 +1,3 @@ -"""CoreML audio encoder emit. - -Wraps an audio encoder PyTorch module (Conformer / Whisper-style) into an -ANE-targeted `.mlpackage`. The runtime side (`cactus-engine/src/npu_ane.mm`) -loads this with a single named input "x" and a single named output, matching -the `NPUEncoder::encode` interface. - -Targets the audio encoder NPU path — historically the highest-impact use -of the ANE for cactus (CPU was faster for text prefill in most cases). -""" from __future__ import annotations from pathlib import Path @@ -17,17 +7,6 @@ class AudioEncoderWrapper(torch.nn.Module): - """Single-input single-output wrapper around an audio encoder module. - - Matches the runtime interface in `ANEEncoder::encode(input, output, ...)`: - one fp16 input tensor, one fp16 output tensor. - - Audio encoder adapters that take auxiliary inputs (e.g. Gemma 4's - ``input_features_mask``) supply them via ``baked_inputs`` — these are - registered as constant buffers so the exported model keeps a static - single-input signature, as the ANE requires. - """ - def __init__( self, audio_module: torch.nn.Module, @@ -85,14 +64,6 @@ def emit_audio_encoder_mlpackage( minimum_deployment_target: str = "iOS18", quantize_bits: int | None = None, ) -> str | None: - """Trace + convert + save. Returns the filename or None on failure. - - `example_input` should be the shape the runtime will use at inference - time (e.g., for Parakeet: ``[1, num_frames, mel_bins]``). The ANE - requires a fully-specified shape; dynamic dims are not supported here. - `baked_inputs` carries any auxiliary encoder inputs (masks etc.) that - get frozen into the model as constants. - """ ct = _import_coremltools() if ct is None: print("npu.audio: coremltools not installed; skipping mlpackage emit") @@ -109,7 +80,6 @@ def emit_audio_encoder_mlpackage( print(f"npu.audio: torch.export failed ({type(exc).__name__}: {exc}); skipping mlpackage emit") return None - # Free the live module — ExportedProgram now owns what it needs. del wrapper import gc as _gc _gc.collect() diff --git a/python/cactus/transpile/npu/coremltools_patches.py b/python/cactus/transpile/npu/coremltools_patches.py index f48003071..f8812b345 100644 --- a/python/cactus/transpile/npu/coremltools_patches.py +++ b/python/cactus/transpile/npu/coremltools_patches.py @@ -1,16 +1,3 @@ -"""Runtime patches for coremltools 9.0 to support Gemma 4 + torch.export. - -Each patch is idempotent and guarded so reimporting the module is safe. -Call `apply_all_coremltools_patches()` once before invoking `ct.convert`. - -Patches: -- register_func: allow dunder-suffixed op names (e.g. ``__and__``) past the - inplace-op rejection check. -- new_ones: register a translation that maps to ``mb.fill(shape=..., value=1.0)``. -- SDPA scalar-cast: ``mb.sub/mul/add`` auto-cast python scalars to the dtype - of the tensor input (avoids fp16/fp32 mismatch in the SDPA mask path). -- Pipeline: remove ``common::fuse_prelu`` (KeyError on float64 in 9.0). -""" from __future__ import annotations _APPLIED = False @@ -65,12 +52,6 @@ def register_func(self, func=None, torch_alias=None, override=False): def _register_new_ones_op() -> None: - """Translate Tensor.new_ones(size, ...) -> mb.fill(shape=size, value=1). - - Mirrors coremltools' built-in `ones` translator (shift by 1 to skip the - `self` tensor argument). Direct registration via set_func_by_name avoids - the decorator's redundant name lookups. - """ from coremltools.converters.mil.frontend.torch.torch_op_registry import ( _TORCH_OPS_REGISTRY, ) @@ -100,9 +81,6 @@ def new_ones(context, node): def _register_logical_and_op() -> None: - """Register __and__ / __or__ translations. Gemma 4's attention mask path emits - `__and__.Tensor` nodes for combining causal + padding masks; coremltools 9.0 - doesn't ship a translator for those names by default.""" from coremltools.converters.mil.frontend.torch.torch_op_registry import ( _TORCH_OPS_REGISTRY, register_torch_op, @@ -114,7 +92,6 @@ def _force_bool(v): name = getattr(dtype, "__name__", str(dtype) if dtype else "") if "bool" in name: return v - # cast through fp32 -> bool to handle exotic source dtypes (e.g. fp64) if "fp16" in name or "fp32" in name or "fp64" in name or "double" in name or "float" in name: non_zero = mb.not_equal(x=v, y=0.0) return non_zero @@ -137,21 +114,12 @@ def _impl(context, node): context.add(out, node.name) return _impl - # Register all aliases that sanitize_op_kind + unify_inplace_and_functional - # may resolve fx node `__and__.Tensor` / `__or__.Tensor` to. The lookup - # chain for `__and__.Tensor` ends at `__and_` (one trailing underscore), - # for `aten::__and__` ends at `__and_`, and the raw key `__and__` ends at - # `and`. Cover all three so the bool-casting handler always wins. for tag, op_kind in [("__and_", "and"), ("__or_", "or"), ("__xor_", "xor")]: if tag not in _TORCH_OPS_REGISTRY.name_to_func_mapping: _TORCH_OPS_REGISTRY.set_func_by_name(make_logical(op_kind), tag) for tag, op_kind in [("__and__", "and"), ("__or__", "or"), ("__xor__", "xor")]: if tag not in _TORCH_OPS_REGISTRY.name_to_func_mapping: _TORCH_OPS_REGISTRY.set_func_by_name(make_logical(op_kind), tag) - # The default `bitwise_and` translator chokes on mixed bool+float inputs - # that Gemma 4's masking emits; override the lookup keys with the - # bool-casting variant. `and` and `or` are the sanitize_op_kind results - # for many aten op kinds, so override those too. for tag in ("bitwise_and", "and"): _TORCH_OPS_REGISTRY.set_func_by_name(make_logical("and"), tag) for tag in ("bitwise_or", "or"): @@ -159,13 +127,6 @@ def _impl(context, node): def _override_layer_norm_translator() -> None: - """Override the torch frontend's ``layer_norm`` translator so gamma/beta/ - epsilon share one dtype before reaching ``mb.layer_norm``. - - coremltools 9.0's iOS17/iOS18 layer_norm op enforces this at validation - time. FP16-traced models hit it because the python-scalar ``eps`` arg - survives as fp32 while gamma/beta come through as fp16. - """ import numpy as np from coremltools.converters.mil.frontend.torch.torch_op_registry import ( _TORCH_OPS_REGISTRY, @@ -178,8 +139,6 @@ def _dtype_name(v): return getattr(d, "__name__", str(d) if d else "") def _to_typed_const(v, target): - """Force v to a const Var of dtype `target`. Eps stays a const so - later MIL passes (add_fp16_cast) don't promote it back to fp32.""" if v is None: return v np_dtype = np.float16 if target == "fp16" else np.float32 @@ -211,7 +170,6 @@ def layer_norm(context, node): "fp32" if ("fp32" in ref_name or "float32" in ref_name) else "fp16" ) - # x is a runtime tensor — keep as cast Var; gamma/beta/eps are const-like. if x is not None and hasattr(x, "dtype") and _dtype_name(x) != target: x = mb.cast(x=x, dtype=target) weight = _to_typed_const(weight, target) @@ -237,27 +195,15 @@ def layer_norm(context, node): def _patch_pipeline_remove_fuse_prelu() -> None: - """No-op placeholder. ``PassPipeline.DEFAULT`` is a classproperty that - returns a fresh instance each time, so mutating it in-place doesn't - persist. We instead build a trimmed pipeline at convert time via - :func:`build_cactus_pass_pipeline` and pass it to ``ct.convert``. - """ return _PASSES_TO_DROP = [ - # fuse_prelu crashes on fp64 const tables in coremltools 9.0. "common::fuse_prelu", ] def build_cactus_pass_pipeline(): - """Construct a fresh PassPipeline with our problem passes removed. - - Pass this to ``ct.convert(..., pass_pipeline=pipeline)`` — mutating - ``PassPipeline.DEFAULT`` doesn't work because that's a classproperty - returning a new instance every time. - """ from coremltools.converters.mil.mil.passes.pass_pipeline import PassPipeline pipeline = PassPipeline.DEFAULT for pass_name in _PASSES_TO_DROP: @@ -269,17 +215,6 @@ def build_cactus_pass_pipeline(): def _patch_mb_binops_scalar_cast() -> None: - """Auto-align dtypes for mb.{sub,mul,add,div} so SDPA mask paths don't - crash on mixed fp16/fp32 inputs. - - Critical: only Var operands get a ``mb.cast`` injected — numpy - arrays/scalars are retyped via ``numpy.astype`` because injecting a - fresh ``mb.cast`` Var from inside MIL passes lands it in the wrong - block scope (e.g. ``divide_to_multiply`` calling ``mb.mul`` with - ``before_op=`` on an op nested in ``block0``). That mis-scoping fails - the "var visibility" validator and trips a hard ValueError mid-pass. - Python scalars are materialized as typed ``mb.const`` for the same - reason.""" import numpy as np from coremltools.converters.mil import Builder as mb @@ -300,20 +235,14 @@ def _np_for(dtype_name): return np.float32 def _is_var(v): - # MIL Vars carry sym_type / op fields; numpy arrays don't. return hasattr(v, "op") and hasattr(v, "sym_type") def _retype_in_place(v, target_dtype_name): - """Convert a *non-Var* operand (numpy array/scalar/python scalar) to - ``target_dtype_name``. Returning a fresh numpy value (not a new Var) - keeps the operand inline so MIL passes don't see a stray Var in the - wrong block.""" np_dtype = _np_for(target_dtype_name) if isinstance(v, np.ndarray): return v.astype(np_dtype) if isinstance(v, (int, float)) and not isinstance(v, bool): return np_dtype(v) - # Best-effort fallback (lists, etc.) try: return np.asarray(v, dtype=np_dtype) except Exception: @@ -333,13 +262,11 @@ def wrapper(**kwargs): y = kwargs.get("y") xn = _dtype_name(x) yn = _dtype_name(y) - # If only one is a tensor, materialize scalars as typed consts. if xn and not yn and _is_fp(xn) and isinstance(y, (int, float)) and not isinstance(y, bool): kwargs["y"] = _scalar_const(y, xn) elif yn and not xn and _is_fp(yn) and isinstance(x, (int, float)) and not isinstance(x, bool): kwargs["x"] = _scalar_const(x, yn) elif xn and yn and xn != yn: - # Both have a dtype but mismatched. Prefer fp16. target = "fp16" if (xn == "fp16" or yn == "fp16") else xn if xn != target: kwargs["x"] = mb.cast(x=x, dtype=target) if _is_var(x) else _retype_in_place(x, target) @@ -352,9 +279,6 @@ def wrapper(**kwargs): setattr(mb, op_name, make_wrapper(original)) - # mb.select(cond, a, b) requires a and b to share a dtype. Gemma 4 vision - # hits this via `torch.where(cond, scalar, tensor)` — the scalar becomes - # fp32 while the tensor is fp16. Materialize scalars as typed consts. select_op = getattr(mb, "select", None) if select_op is not None and not getattr(select_op, "_cactus_scalar_cast_patched", False): def _select_wrap(orig): @@ -379,8 +303,6 @@ def wrapper(**kwargs): return wrapper setattr(mb, "select", _select_wrap(select_op)) - # layer_norm: epsilon (scalar) must match gamma/beta dtype. Parakeet's - # Conformer norms produce fp16 gamma but coremltools emits fp32 epsilon. orig_ln = getattr(mb, "layer_norm", None) if orig_ln is not None and not getattr(orig_ln, "_cactus_scalar_cast_patched", False): def _ln_wrap(orig): @@ -399,14 +321,12 @@ def wrapper(**kwargs): return wrapper setattr(mb, "layer_norm", _ln_wrap(orig_ln)) - # batch_norm / instance_norm have similar epsilon shapes for op_name in ("batch_norm", "instance_norm", "rms_norm"): orig_op = getattr(mb, op_name, None) if orig_op is None or getattr(orig_op, "_cactus_scalar_cast_patched", False): continue def _bn_wrap(orig): def wrapper(**kwargs): - # Reference dtype: gamma if present, else x ref = kwargs.get("gamma") or kwargs.get("x") rn = _dtype_name(ref) if _is_fp(rn): @@ -423,13 +343,6 @@ def wrapper(**kwargs): def _patch_fp16_cast_skip_layer_norm() -> None: - """Tell the ``common::add_fp16_cast`` MIL pass to skip ``layer_norm`` and - ``batch_norm``-family ops. The pass casts gamma to fp16 but leaves - epsilon as fp32, which violates the iOS17+ ``layer_norm`` constraint that - gamma, beta, and epsilon share one dtype. - - By marking them unsupported, we keep them at their original (fp32) dtype. - """ try: from coremltools.converters.mil.mil.passes.defs.quantization import ( FP16ComputePrecision, @@ -444,13 +357,6 @@ def _patch_fp16_cast_skip_layer_norm() -> None: def _override_one_hot_translator() -> None: - """Override ``one_hot`` to cast indices to int32 before calling ``mb.one_hot``. - - The MIL ``one_hot`` op requires int32 indices, but Gemma 4 vision feeds - ``pixel_position_ids`` as fp32 through ``torch.export``. The default - coremltools translator passes the labels through unchanged, which trips - the type domain check. - """ from coremltools.converters.mil.frontend.torch.torch_op_registry import ( _TORCH_OPS_REGISTRY, ) @@ -479,17 +385,6 @@ def one_hot(context, node): def _register_unfold_op() -> None: - """Translate ``Tensor.unfold(dimension, size, step)`` to ``mb.sliding_windows``. - - Layout note: PyTorch's ``unfold`` *appends* the window-size dim at the - end of the output (rank N+1). MIL's ``mb.sliding_windows`` *inserts* - the size dim at ``axis+1`` instead. We translate to ``sliding_windows`` - then ``mb.transpose`` the size dim to the trailing position so downstream - permutes/movedims see the shape PyTorch produced. - - Gemma 4 audio's ``_extract_block_context`` is the canonical caller and - deeply depends on this layout (size at last axis, then ``movedim(-1, 2)``). - """ from coremltools.converters.mil.frontend.torch.torch_op_registry import ( _TORCH_OPS_REGISTRY, ) @@ -515,8 +410,6 @@ def unfold(context, node): stride=int(step), ) - # sliding_windows inserts the window-size dim at axis+1; PyTorch - # unfold appends it at the end. Permute (size_axis) to the last axis. out_rank = rank + 1 perm = list(range(out_rank)) size_axis = dim + 1 diff --git a/python/cactus/transpile/npu/pipeline.py b/python/cactus/transpile/npu/pipeline.py index 3e6d4946a..017226368 100644 --- a/python/cactus/transpile/npu/pipeline.py +++ b/python/cactus/transpile/npu/pipeline.py @@ -1,14 +1,3 @@ -"""NPU transpiler orchestrator. - -Emits CoreML `.mlpackage`s for **audio + vision encoders** from the same -``ComponentModuleSpec`` adapter modules the graph transpiler captures. -Invoked from ``hf_model._run_component_pipeline_transpile`` when ``--npu`` -is passed on the CLI. - -Text-decoder prefill is intentionally not on NPU — CPU prefill is the -supported path. (Historically ``origin/main`` shipped pre-built NPU prefill -mlpackages for Gemma/Qwen; v2 does not regenerate them.) -""" from __future__ import annotations from pathlib import Path @@ -17,7 +6,6 @@ from .vision import emit_vision_encoder_mlpackage -# Component name -> (emit function, output mlpackage filename). _ENCODER_COMPONENTS = { "audio_encoder": (emit_audio_encoder_mlpackage, "audio_encoder.mlpackage"), "vision_encoder": (emit_vision_encoder_mlpackage, "vision_encoder.mlpackage"), @@ -29,35 +17,10 @@ def run_encoder_pipeline( artifact_dir: Path, *, enabled: bool = True, - quantize_bits: int | None = None, # legacy override: forces both - audio_quantize_bits: int | None = None, # per-component, defaults to int8 - vision_quantize_bits: int | None = None, # per-component, defaults to fp16 + quantize_bits: int | None = None, + audio_quantize_bits: int | None = None, + vision_quantize_bits: int | None = None, ) -> dict[str, str]: - """Emit audio / vision encoder `.mlpackage`s from captured component specs. - - Per-component quantization defaults — set independently because the two - encoder families have very different sensitivity to weight quant: - - - **Audio** (Conformer-style: Parakeet, Gemma 4 audio): wide hidden - dims, conv subsampling that averages noise across many input frames, - long sequences. int8 holds up cleanly (verified cos_sim 0.9963 vs HF - on Gemma 4); even int4 generally works. - - **Vision** (ViT-style: LFM-VL SigLIP2, Gemma 4 vision): narrow - attention heads, one patch-embedding conv is a single point of - failure, short sequence (256-1024 tokens). int4 visibly degrades - outputs; int8 is fine but fp16 is the rock-solid default since the - vision tower is small (one-shot per request, bandwidth not critical). - - Quantization knob is per-component (``audio_quantize_bits`` / - ``vision_quantize_bits``). The legacy ``quantize_bits`` arg, when set, - overrides BOTH per-component defaults — preserves prior behavior where - one ``--npu-quantize 4`` flag controlled the whole conversion. - - Defaults: audio → int8, vision → fp16. ``0`` means fp16 (no quant). - - Returns a dict mapping ``manifest.json`` key (``npu_audio_encoder`` / - ``npu_vision_encoder``) to the manifest-relative `.mlpackage` path. - """ results: dict[str, str] = {} if not enabled or not component_specs: return results @@ -65,8 +28,6 @@ def run_encoder_pipeline( bundle_root = artifact_dir / "components" bundle_root.mkdir(parents=True, exist_ok=True) - # Resolve final quant per component. Per-component arg wins if given; - # else the legacy `quantize_bits` override; else per-component defaults. def _resolve(per_component: int | None, default: int | None) -> int | None: chosen = per_component if per_component is not None else ( quantize_bits if quantize_bits is not None else default @@ -75,7 +36,7 @@ def _resolve(per_component: int | None, default: int | None) -> int | None: component_quants = { "audio_encoder": _resolve(audio_quantize_bits, default=8), - "vision_encoder": _resolve(vision_quantize_bits, default=None), # default fp16 + "vision_encoder": _resolve(vision_quantize_bits, default=None), } for spec in component_specs: diff --git a/python/cactus/transpile/npu/vision.py b/python/cactus/transpile/npu/vision.py index 8f1ea6019..322cbd24d 100644 --- a/python/cactus/transpile/npu/vision.py +++ b/python/cactus/transpile/npu/vision.py @@ -1,13 +1,3 @@ -"""CoreML vision encoder emit. - -Wraps a vision encoder PyTorch module (SigLIP2 / Gemma 4 vision tower) -into an ANE-targeted `.mlpackage`. The runtime side -(`cactus-engine/src/npu_ane.mm`) loads this with a single named input "x" -and a single named output, matching the `NPUEncoder::encode` interface. - -Mirrors `audio.py`. Historically (`origin/main`) the NPU vision targets -were SigLIP2 and the Gemma 4 vision tower; LFM-VL was never NPU-accelerated. -""" from __future__ import annotations from pathlib import Path @@ -17,17 +7,6 @@ class VisionEncoderWrapper(torch.nn.Module): - """Single-input single-output wrapper around a vision encoder module. - - Matches the runtime interface in `ANEEncoder::encode(input, output, ...)`: - one fp16 input tensor, one fp16 output tensor. - - Auxiliary inputs that a vision tower needs (position ids, spatial - shapes, attention masks) are supplied via ``baked_inputs`` and frozen - as constant buffers so the exported model keeps a static single-input - signature, as the ANE requires. - """ - def __init__( self, vision_module: torch.nn.Module, @@ -85,14 +64,6 @@ def emit_vision_encoder_mlpackage( minimum_deployment_target: str = "iOS18", quantize_bits: int | None = None, ) -> str | None: - """Trace + convert + save. Returns the filename or None on failure. - - `example_input` should be the shape the runtime will use at inference - time (e.g. ``[1, num_patches, patch_dim]``). The ANE requires a - fully-specified shape; dynamic dims are not supported here. - `baked_inputs` carries any auxiliary encoder inputs (position ids, - masks) that get frozen into the model as constants. - """ ct = _import_coremltools() if ct is None: print("npu.vision: coremltools not installed; skipping mlpackage emit") @@ -109,7 +80,6 @@ def emit_vision_encoder_mlpackage( print(f"npu.vision: torch.export failed ({type(exc).__name__}: {exc}); skipping mlpackage emit") return None - # Free the live module — ExportedProgram now owns what it needs. del wrapper import gc as _gc _gc.collect() From 423d9dec9c15aaf589d5be6fcb39fa5f2d0841c8 Mon Sep 17 00:00:00 2001 From: justinl66 Date: Fri, 29 May 2026 08:27:49 -0700 Subject: [PATCH 8/8] Remove more commnets --- cactus-engine/src/model_npu.cpp | 8 ++++---- python/cactus/transpile/hf_model.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cactus-engine/src/model_npu.cpp b/cactus-engine/src/model_npu.cpp index 0661f055c..b5be8fb72 100644 --- a/cactus-engine/src/model_npu.cpp +++ b/cactus-engine/src/model_npu.cpp @@ -89,7 +89,7 @@ bool Model::audio_encode_via_npu(const std::vector& audio_features) { media_feature_shapes_[name] = std::move(shape); } media_feature_precisions_[name] = desc.precision; - break; // audio encoder only has one logical output + break; } return true; } @@ -161,11 +161,11 @@ bool Model::vision_encode_via_npu(const std::vector& pixel_values) { shape_it->second[shape_it->second.size() - 2] += npu_shape[npu_shape.size() - 2]; } media_feature_precisions_[name] = cpu_prec; - break; // vision encoder has one logical output + break; } return true; } -} // namespace engine -} // namespace cactus +} +} diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index 3e3adbb36..f9648e0fc 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -546,9 +546,9 @@ def _run_component_pipeline_transpile( print(f"weights_dir={weights_dir}") npu_enabled = bool(getattr(args, "npu", False)) - npu_quantize = getattr(args, "npu_quantize", None) # legacy override - npu_audio_quantize = getattr(args, "npu_audio_quantize", None) # default 8 if neither set - npu_vision_quantize = getattr(args, "npu_vision_quantize", None) # default 0 (fp16) if neither set + npu_quantize = getattr(args, "npu_quantize", None) + npu_audio_quantize = getattr(args, "npu_audio_quantize", None) + npu_vision_quantize = getattr(args, "npu_vision_quantize", None) npu_encoder_mlpackages: dict[str, str] = {} if npu_enabled and artifact_dir is not None: from .npu import run_encoder_pipeline