From ef082029b51e93288247fd78b4b8e32141f4832d Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 20 May 2026 12:40:33 -0700 Subject: [PATCH 1/8] Add native chunked prefill for transpiled LLM bundles Implement native chunked prefill support for transpiled multimodal LLM bundles and align the generated component pipelines with the C++ runtime loader. Runtime changes: - Load only the required component graphs for each supported decode route. - Add chunked text prefill routing through lm_encoder_text_chunk and decoder_prefill_chunk components. - Copy chunked prefill cache state into step decode components when compatible. - Support direct decoder-step and full-context text decode routes for bundles that do not expose the full chunked path. - Preserve cache metadata on reset and shift cache rows when thinking-token ranges are removed. - Keep FP16 KV cache configuration stable after process startup. Transpiler changes: - Emit chunk prefill components for Gemma4, Qwen3-VL, and LFM2-VL component pipelines. - Lower shared internal KV-cache attention paths for prefill/query reuse. - Add Qwen3-VL model-type/profile handling and safer Qwen layer-type mask selection. - Preserve Gemma4 vision projection precision and adjust Gemma4 audio bias handling. - Prefer 128-token chunk defaults for supported multimodal profiles. Build and platform changes: - Make local Cactus builds less racy by serializing build directory updates. - Improve Android NDK discovery for Homebrew and SDK installs. - Respect existing Android CPU affinity masks when pinning worker threads. Tests: - Update transpile default expectations for multimodal component pipelines. - Update graph serialization tests for persisted node ids. - Add KV-cache full-window eviction coverage. Signed-off-by: Noah Cylich --- android/build.sh | 9 +- cactus-engine/src/engine.h | 32 +- cactus-engine/src/model.cpp | 632 +++++++++++++++--- cactus-graph/src/ops_cache.cpp | 133 ++-- cactus-graph/tests/test_cache.cpp | 27 + cactus-graph/tests/test_io.cpp | 8 +- cactus-graph/tests/test_utils.h | 2 +- cactus-kernels/src/threading.h | 9 +- cactus/build.sh | 11 +- python/cactus/cli/compile.py | 8 + .../cactus/convert/model_adapters/policy.py | 12 +- python/cactus/transpile/hf_model.py | 17 +- python/cactus/transpile/lower.py | 89 +++ python/cactus/transpile/model_adapters.py | 345 ++++++++-- python/cactus/transpile/model_profiles.py | 19 +- python/tests/test_cli_transpile_defaults.py | 25 +- 16 files changed, 1148 insertions(+), 230 deletions(-) diff --git a/android/build.sh b/android/build.sh index 9d6f2d1e2..370a3fd83 100755 --- a/android/build.sh +++ b/android/build.sh @@ -12,9 +12,16 @@ CACTUS_CURL_ROOT=${CACTUS_CURL_ROOT:-"$PROJECT_ROOT/cactus-engine/libs/curl"} if [ -z "$ANDROID_NDK_HOME" ]; then if [ -n "$ANDROID_HOME" ]; then ANDROID_NDK_HOME=$(ls -d "$ANDROID_HOME/ndk/"* 2>/dev/null | sort -V | tail -1) - elif [ -d "$HOME/Library/Android/sdk" ]; then + fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "$HOME/Library/Android/sdk" ]; then ANDROID_NDK_HOME=$(ls -d "$HOME/Library/Android/sdk/ndk/"* 2>/dev/null | sort -V | tail -1) fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/Caskroom/android-ndk" ]; then + ANDROID_NDK_HOME=$(ls -d /opt/homebrew/Caskroom/android-ndk/*/AndroidNDK*.app/Contents/NDK 2>/dev/null | sort -V | tail -1) + fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/share/android-ndk" ]; then + ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" + fi fi if [ -z "$ANDROID_NDK_HOME" ] || [ ! -d "$ANDROID_NDK_HOME" ]; then diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 6cce3a484..2656a0898 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -580,7 +580,8 @@ class Model { size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f); - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + void prefill(const std::vector& tokens, size_t chunk_size = 128, const std::string& profile_file = "", + bool prepare_decode = true); void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, const std::string& profile_file = ""); @@ -629,7 +630,7 @@ class Model { bool load_npu_prefill(const std::string& model_path); bool has_npu_prefill() const { return false; } - size_t get_prefill_chunk_size() const { return 256; } + size_t get_prefill_chunk_size() const { return 128; } void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} @@ -649,7 +650,7 @@ class Model { std::string path; }; - struct CacheStateEntry { + struct CacheStateBinding { std::string layer_key; int key_node_id = -1; int value_node_id = -1; @@ -663,22 +664,33 @@ class Model { std::vector output_node_ids; std::vector logical_outputs; std::vector bindings; + std::vector cache_states; std::unique_ptr graph; std::vector> input_buffers; - std::vector cache_states; }; - void copy_cache_state(const Component& src, Component& dst); - bool load_manifest(); bool setup_tokenizer(); - bool load_components(); + bool load_components(const std::unordered_set& required_components); bool bind_runtime_buffers(Component& comp); void run_step(uint32_t token_id, size_t position, bool read_logits); + void run_encoder_step(uint32_t token_id, size_t position); void run_media_step(size_t position, const uint8_t* feature_row, size_t feature_row_bytes, Precision feature_precision); + void copy_component_outputs_to_inputs(const Component& source, Component& target); void copy_encoder_outputs_to_decoder(const Component& enc); + void copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index); + void copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset); + bool cache_states_compatible(const Component& source, const Component& target) const; + void copy_cache_states(const Component& source, Component& target); + void reset_component_cache_states(Component& comp); + size_t component_chunk_tokens(const Component& comp, const std::string& input_name) const; + size_t component_output_tokens(const Component& comp, const std::string& output_name) const; + size_t run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, + bool prepare_decode); + void run_full_context_text(); void write_int_input(Component& comp, const std::string& name, int64_t value); + void write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value); void write_bytes_input(Component& comp, const std::string& name, const void* data, size_t byte_size); int input_index(const Component& comp, const std::string& name) const; int output_index(const Component& comp, const std::string& name) const; @@ -693,6 +705,10 @@ class Model { std::map components_; Component* encoder_ = nullptr; Component* decoder_ = nullptr; + Component* decoder_prefill_ = nullptr; + Component* prefill_encoder_ = nullptr; + enum class DecodeRoute { CACHED_STEP, DIRECT_DECODER_STEP, FULL_CONTEXT_TEXT }; + DecodeRoute decode_route_ = DecodeRoute::CACHED_STEP; Component* vision_encoder_ = nullptr; Component* audio_encoder_ = nullptr; Component* lm_encoder_media_step_ = nullptr; @@ -710,6 +726,8 @@ class Model { bool initialized_ = false; size_t cache_total_seq_len_ = 0; size_t cache_max_seq_len_ = 4096; + size_t last_logit_position_ = 0; + std::vector context_tokens_; static constexpr size_t MAX_TOKEN_HISTORY = 128; std::vector token_history_; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 4efc957f2..64a5854f1 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -19,10 +19,70 @@ #include #include #include +#include namespace cactus { namespace engine { +float read_scalar_value(Precision precision, const uint8_t* data, size_t index) { + const uint8_t* ptr = data + PrecisionTraits::byte_offset_of(precision, index); + switch (precision) { + case Precision::FP32: + return *reinterpret_cast(ptr); + case Precision::FP16: + return static_cast(*reinterpret_cast(ptr)); + case Precision::INT8: + return static_cast(*reinterpret_cast(ptr)); + default: + return 0.0f; + } +} + +void write_scalar_value(Precision precision, uint8_t* data, size_t index, float value) { + uint8_t* ptr = data + PrecisionTraits::byte_offset_of(precision, index); + switch (precision) { + case Precision::FP32: + *reinterpret_cast(ptr) = value; + break; + case Precision::FP16: + *reinterpret_cast<__fp16*>(ptr) = static_cast<__fp16>(value); + break; + case Precision::INT8: + *reinterpret_cast(ptr) = static_cast(value); + break; + default: + break; + } +} + +bool copy_component_tensor(CactusGraph& source_graph, + const BufferDesc& src_desc, + size_t src_node, + const BufferDesc& dst_desc, + std::vector& dst_buffer, + size_t dst_element_offset, + size_t element_count, + const std::string& name) { + const auto* src_ptr = static_cast(source_graph.get_output(src_node)); + if (src_desc.precision == dst_desc.precision) { + size_t dst_offset = PrecisionTraits::byte_offset_of(dst_desc.precision, dst_element_offset); + std::memcpy( + dst_buffer.data() + dst_offset, + src_ptr, + PrecisionTraits::packed_size_of(src_desc.precision, element_count)); + return true; + } + if (name != "position_ids" && name != "attention_mask") return false; + for (size_t i = 0; i < element_count; ++i) { + write_scalar_value( + dst_desc.precision, + dst_buffer.data(), + dst_element_offset + i, + read_scalar_value(src_desc.precision, src_ptr, i)); + } + return true; +} + void ConvCache::init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision) { num_layers = layers; hidden_size = hidden_dim; @@ -144,30 +204,95 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_ERROR("model", "Tokenizer init failed for bundle: " << bundle_dir); return false; } - if (!load_components()) return false; - - encoder_ = components_.count("lm_encoder_step") ? &components_.at("lm_encoder_step") : nullptr; - decoder_ = components_.count("decoder_step") ? &components_.at("decoder_step") : nullptr; - const bool is_lm = (encoder_ != nullptr && decoder_ != nullptr); - const bool is_transcription = components_.count("audio_encoder") && - (components_.count("decoder") || components_.count("decoder_joint")); - if (!is_lm && !is_transcription) { - CACTUS_LOG_ERROR("model", "Bundle missing required components: need lm_encoder_step+decoder_step (LM) or audio_encoder+decoder (transcription)"); + std::string encoder_name; + std::string decoder_name; + std::unordered_set required_components; + bool has_chunked_prefill = components_.count("lm_encoder_step") + && components_.count("decoder_media_step") + && components_.count("lm_encoder_text_chunk") + && components_.count("decoder_prefill_chunk"); + if (has_chunked_prefill) { + encoder_name = "lm_encoder_step"; + decoder_name = "decoder_media_step"; + decode_route_ = DecodeRoute::CACHED_STEP; + required_components = { + encoder_name, + decoder_name, + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + }; + } else if (components_.count("decoder_step") + && input_index(components_.at("decoder_step"), "input_ids") >= 0 + && input_index(components_.at("decoder_step"), "position_ids") >= 0) { + decoder_name = "decoder_step"; + decode_route_ = DecodeRoute::DIRECT_DECODER_STEP; + required_components = {decoder_name}; + } else if (components_.count("lm_encoder_step") && components_.count("decoder_step")) { + encoder_name = "lm_encoder_step"; + decoder_name = "decoder_step"; + decode_route_ = DecodeRoute::CACHED_STEP; + required_components = {encoder_name, decoder_name}; + if (components_.count("decoder_prefill_chunk")) { + required_components.insert("decoder_prefill_chunk"); + } + if (components_.count("lm_encoder_text_chunk")) { + required_components.insert("lm_encoder_text_chunk"); + } + } else if (components_.count("text_lm_encoder") && components_.count("decoder")) { + encoder_name = "text_lm_encoder"; + decoder_name = "decoder"; + decode_route_ = DecodeRoute::FULL_CONTEXT_TEXT; + required_components = {encoder_name, decoder_name}; + } else if (components_.count("audio_encoder") && + (components_.count("decoder") || components_.count("decoder_joint"))) { + decoder_name = components_.count("decoder") ? "decoder" : "decoder_joint"; + decode_route_ = DecodeRoute::DIRECT_DECODER_STEP; + required_components = {"audio_encoder", decoder_name}; + } else { + CACTUS_LOG_ERROR("model", "Bundle missing lm_encoder_step+decoder_step or text_lm_encoder+decoder components"); return false; } - if (encoder_ && !bind_runtime_buffers(*encoder_)) return false; - if (decoder_ && !bind_runtime_buffers(*decoder_)) return false; - + for (const auto& optional : { + "vision_encoder", + "audio_encoder", + "lm_encoder_media_step", + "decoder_prefill_chunk", + "lm_encoder", + }) { + if (components_.count(optional)) { + required_components.insert(optional); + } + } + if (!load_components(required_components)) return false; + if (!encoder_name.empty()) encoder_ = &components_.at(encoder_name); + if (!decoder_name.empty()) decoder_ = &components_.at(decoder_name); + if (components_.count("decoder_prefill_chunk") && components_.at("decoder_prefill_chunk").graph) { + decoder_prefill_ = &components_.at("decoder_prefill_chunk"); + decoder_prefill_chunk_ = decoder_prefill_; + } + if (components_.count("lm_encoder_text_chunk") && components_.at("lm_encoder_text_chunk").graph) { + prefill_encoder_ = &components_.at("lm_encoder_text_chunk"); + } vision_encoder_ = components_.count("vision_encoder") ? &components_.at("vision_encoder") : nullptr; audio_encoder_ = components_.count("audio_encoder") ? &components_.at("audio_encoder") : nullptr; lm_encoder_media_step_ = components_.count("lm_encoder_media_step") ? &components_.at("lm_encoder_media_step") : nullptr; - decoder_prefill_chunk_ = components_.count("decoder_prefill_chunk") ? &components_.at("decoder_prefill_chunk") : nullptr; lm_encoder_ = components_.count("lm_encoder") ? &components_.at("lm_encoder") : nullptr; - if (vision_encoder_ && !bind_runtime_buffers(*vision_encoder_)) return false; - if (audio_encoder_ && !bind_runtime_buffers(*audio_encoder_)) return false; - if (lm_encoder_media_step_ && !bind_runtime_buffers(*lm_encoder_media_step_)) return false; - if (decoder_prefill_chunk_ && !bind_runtime_buffers(*decoder_prefill_chunk_)) return false; - if (lm_encoder_ && !bind_runtime_buffers(*lm_encoder_)) return false; + std::vector to_bind = { + encoder_, + prefill_encoder_, + decoder_, + decoder_prefill_, + vision_encoder_, + audio_encoder_, + lm_encoder_media_step_, + lm_encoder_, + }; + std::unordered_set bound; + for (Component* comp : to_bind) { + if (!comp || !comp->graph || bound.count(comp)) continue; + if (!bind_runtime_buffers(*comp)) return false; + bound.insert(comp); + } if (vision_encoder_ && tokenizer_ && !vision_encoder_->output_node_ids.empty()) { size_t out_node = static_cast(vision_encoder_->output_node_ids[0]); @@ -228,15 +353,15 @@ bool Model::load_manifest() { } } if (c.count("cache_state_node_ids")) { - for (const auto& ev : c.at("cache_state_node_ids").get()) { - if (!ev.is()) continue; - const auto& e = ev.get(); - CacheStateEntry cs; - if (e.count("layer_key")) cs.layer_key = e.at("layer_key").get(); - if (e.count("key") && e.at("key").is()) - cs.key_node_id = static_cast(e.at("key").get()); - if (e.count("value") && e.at("value").is()) - cs.value_node_id = static_cast(e.at("value").get()); + for (const auto& sv : c.at("cache_state_node_ids").get()) { + if (!sv.is()) continue; + const auto& s = sv.get(); + CacheStateBinding cs; + if (s.count("layer_key")) cs.layer_key = s.at("layer_key").get(); + if (s.count("key") && s.at("key").is()) + cs.key_node_id = static_cast(s.at("key").get()); + if (s.count("value") && s.at("value").is()) + cs.value_node_id = static_cast(s.at("value").get()); if (cs.key_node_id >= 0 && cs.value_node_id >= 0) { comp.cache_states.push_back(std::move(cs)); } @@ -261,8 +386,9 @@ bool Model::setup_tokenizer() { return tokenizer_->load_vocabulary_with_config(vocab, merges, cfg); } -bool Model::load_components() { +bool Model::load_components(const std::unordered_set& required_components) { for (auto& [name, comp] : components_) { + if (!required_components.empty() && !required_components.count(name)) continue; if (comp.graph_path.empty()) continue; fs::path full = fs::path(bundle_dir_) / comp.graph_path; try { @@ -274,8 +400,14 @@ bool Model::load_components() { for (const auto& b : comp.bindings) { if (b.node_id < 0) continue; try { - comp.graph->bind_mmap_weights(static_cast(b.node_id), - (fs::path(bundle_dir_) / b.path).string()); + fs::path weight_path(b.path); + if (weight_path.is_absolute()) { + fs::path local = fs::path(bundle_dir_) / weight_path.filename(); + if (fs::exists(local)) weight_path = local; + } else { + weight_path = fs::path(bundle_dir_) / weight_path; + } + comp.graph->bind_mmap_weights(static_cast(b.node_id), weight_path.string()); } catch (const std::exception& e) { CACTUS_LOG_ERROR("model", "bind " << b.path << ": " << e.what()); return false; @@ -304,23 +436,30 @@ int Model::input_index(const Component& comp, const std::string& name) const { } void Model::write_int_input(Component& comp, const std::string& name, int64_t value) { + write_int_input_at(comp, name, 0, value); +} + +void Model::write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value) { int idx = input_index(comp, name); if (idx < 0) return; size_t node_id = static_cast(comp.runtime_input_node_ids[idx]); const auto& desc = comp.graph->get_output_buffer(node_id); auto& buf = comp.input_buffers[idx]; + if (index >= desc.total_size) return; + size_t offset = PrecisionTraits::byte_offset_of(desc.precision, index); + auto* dst = buf.data() + offset; switch (desc.precision) { case Precision::FP32: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; case Precision::FP16: - *reinterpret_cast<__fp16*>(buf.data()) = static_cast<__fp16>(value); + *reinterpret_cast<__fp16*>(dst) = static_cast<__fp16>(value); break; case Precision::INT8: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; default: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; } } @@ -358,10 +497,365 @@ void Model::copy_encoder_outputs_to_decoder(const Component& enc) { } void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { + if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { + write_int_input(*decoder_, "input_ids", static_cast(token_id)); + write_int_input(*decoder_, "position_ids", static_cast(position)); + decoder_->graph->execute(); + return; + } + run_encoder_step(token_id, position); + copy_component_outputs_to_inputs(*encoder_, *decoder_); + decoder_->graph->execute(); +} + +void Model::run_encoder_step(uint32_t token_id, size_t position) { write_int_input(*encoder_, "input_ids", static_cast(token_id)); write_int_input(*encoder_, "position_ids", static_cast(position)); encoder_->graph->execute(); - copy_encoder_outputs_to_decoder(*encoder_); +} + +void Model::copy_component_outputs_to_inputs(const Component& source, Component& target) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + std::fill(target.input_buffers[dst_idx].begin(), target.input_buffers[dst_idx].end(), 0); + size_t elements = std::min(src_desc.total_size, dst_desc.total_size); + if (!copy_component_tensor(*source.graph, src_desc, src_node, dst_desc, target.input_buffers[dst_idx], 0, elements, out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +void Model::copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + size_t chunk_tokens = component_chunk_tokens(target, out_name); + if (chunk_tokens <= token_index || chunk_tokens == 0) { + throw std::runtime_error("chunk prefill token index exceeds input capacity for " + out_name); + } + if (dst_desc.total_size % chunk_tokens != 0) { + throw std::runtime_error("chunk prefill input shape is not token-aligned for " + out_name); + } + size_t elements_per_token = dst_desc.total_size / chunk_tokens; + if (src_desc.total_size != elements_per_token) { + throw std::runtime_error("component output/input token shape mismatch for " + out_name); + } + if (!copy_component_tensor( + *source.graph, + src_desc, + src_node, + dst_desc, + target.input_buffers[dst_idx], + token_index * elements_per_token, + src_desc.total_size, + out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +void Model::copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + size_t src_tokens = component_output_tokens(source, out_name); + size_t dst_tokens = component_chunk_tokens(target, out_name); + if (src_tokens == 0 || dst_tokens == 0 || token_offset + src_tokens > dst_tokens) { + throw std::runtime_error("chunk prefill output range exceeds input capacity for " + out_name); + } + if (src_desc.total_size % src_tokens != 0 || dst_desc.total_size % dst_tokens != 0) { + throw std::runtime_error("chunk prefill output/input shape is not token-aligned for " + out_name); + } + size_t src_elements_per_token = src_desc.total_size / src_tokens; + size_t dst_elements_per_token = dst_desc.total_size / dst_tokens; + if (src_elements_per_token != dst_elements_per_token) { + throw std::runtime_error("component output/input token shape mismatch for " + out_name); + } + if (!copy_component_tensor( + *source.graph, + src_desc, + src_node, + dst_desc, + target.input_buffers[dst_idx], + token_offset * dst_elements_per_token, + src_desc.total_size, + out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +bool Model::cache_states_compatible(const Component& source, const Component& target) const { + if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) return false; + for (size_t i = 0; i < source.cache_states.size(); ++i) { + const auto& src = source.cache_states[i]; + const auto& dst = target.cache_states[i]; + if (src.layer_key != dst.layer_key) return false; + if (src.key_node_id < 0 || src.value_node_id < 0 || dst.key_node_id < 0 || dst.value_node_id < 0) return false; + } + return true; +} + +void Model::copy_cache_states(const Component& source, Component& target) { + if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) { + throw std::runtime_error("prefill and step cache states are not compatible"); + } + for (size_t i = 0; i < source.cache_states.size(); ++i) { + const auto& src = source.cache_states[i]; + const auto& dst = target.cache_states[i]; + if (src.layer_key != dst.layer_key) { + throw std::runtime_error("prefill and step cache layer mismatch: " + src.layer_key + " != " + dst.layer_key); + } + for (auto [src_node, dst_node] : {std::pair{src.key_node_id, dst.key_node_id}, std::pair{src.value_node_id, dst.value_node_id}}) { + const auto& src_desc = source.graph->get_output_buffer(static_cast(src_node)); + const auto& dst_desc = target.graph->get_output_buffer(static_cast(dst_node)); + if (src_desc.precision != dst_desc.precision) { + std::ostringstream oss; + oss << "prefill and step cache precision mismatch at layer " << src.layer_key + << ": " << static_cast(src_desc.precision) + << " vs " << static_cast(dst_desc.precision); + throw std::runtime_error(oss.str()); + } + void* src_ptr = source.graph->get_output(static_cast(src_node)); + void* dst_ptr = target.graph->get_output(static_cast(dst_node)); + if (src_desc.byte_size == dst_desc.byte_size) { + std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); + continue; + } + if (src_desc.precision == Precision::INT8) { + auto* src_meta = static_cast(src_ptr); + auto* dst_meta = static_cast(dst_ptr); + const size_t src_current = static_cast(src_meta[0]); + const size_t src_max = static_cast(src_meta[1]); + const size_t kv_heads = static_cast(src_meta[2]); + const size_t head_dim = static_cast(src_meta[3]); + const size_t sink = static_cast(src_meta[4]); + if (src_max == 0 || kv_heads == 0 || head_dim == 0) { + throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); + } + const size_t groups = (head_dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + const size_t int8_stride = kv_heads * head_dim; + const size_t scale_stride = kv_heads * groups; + const size_t row_bytes = int8_stride + scale_stride * sizeof(float); + const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; + if (dst_max == 0) { + throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); + } + const size_t dst_current = std::min(src_current, dst_max); + dst_meta[0] = dst_current; + dst_meta[1] = dst_max; + dst_meta[2] = kv_heads; + dst_meta[3] = head_dim; + dst_meta[4] = std::min(sink, dst_current); + std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); + + const auto* src_i8 = static_cast(src_ptr) + 64; + const auto* src_scales = reinterpret_cast( + static_cast(src_ptr) + 64 + src_max * int8_stride); + auto* dst_i8 = static_cast(dst_ptr) + 64; + auto* dst_scales = reinterpret_cast( + static_cast(dst_ptr) + 64 + dst_max * int8_stride); + auto copy_rows = [&](size_t dst_row, size_t src_row, size_t rows) { + if (rows == 0) return; + std::memcpy( + dst_i8 + dst_row * int8_stride, + src_i8 + src_row * int8_stride, + rows * int8_stride); + std::memcpy( + dst_scales + dst_row * scale_stride, + src_scales + src_row * scale_stride, + rows * scale_stride * sizeof(float)); + }; + if (src_current <= dst_max) { + copy_rows(0, 0, src_current); + } else { + const size_t copied_sink = std::min(sink, dst_max); + const size_t tail_rows = dst_max - copied_sink; + copy_rows(0, 0, copied_sink); + if (tail_rows > 0) copy_rows(copied_sink, src_current - tail_rows, tail_rows); + } + continue; + } + if (PrecisionTraits::is_cq(src_desc.precision) || src_desc.byte_size < 64 || dst_desc.byte_size < 64) { + std::ostringstream oss; + oss << "prefill and step cache buffer mismatch at layer " << src.layer_key + << ": " << src_desc.byte_size << " bytes vs " << dst_desc.byte_size << " bytes"; + throw std::runtime_error(oss.str()); + } + + auto* src_meta = static_cast(src_ptr); + auto* dst_meta = static_cast(dst_ptr); + const size_t src_current = static_cast(src_meta[0]); + const size_t src_max = static_cast(src_meta[1]); + const size_t kv_heads = static_cast(src_meta[2]); + const size_t head_dim = static_cast(src_meta[3]); + const size_t sink = static_cast(src_meta[4]); + if (src_max == 0 || kv_heads == 0 || head_dim == 0) { + throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); + } + const size_t row_bytes = kv_heads * head_dim * PrecisionTraits::size_of(src_desc.precision); + const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; + if (dst_max == 0) { + throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); + } + const size_t dst_current = std::min(src_current, dst_max); + dst_meta[0] = dst_current; + dst_meta[1] = dst_max; + dst_meta[2] = kv_heads; + dst_meta[3] = head_dim; + dst_meta[4] = std::min(sink, dst_current); + std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); + const auto* src_rows = static_cast(src_ptr) + 64; + auto* dst_rows = static_cast(dst_ptr) + 64; + if (src_current <= dst_max) { + std::memcpy(dst_rows, src_rows, src_current * row_bytes); + } else { + const size_t copied_sink = std::min(sink, dst_max); + const size_t tail_rows = dst_max - copied_sink; + if (copied_sink > 0) { + std::memcpy(dst_rows, src_rows, copied_sink * row_bytes); + } + if (tail_rows > 0) { + std::memcpy( + dst_rows + copied_sink * row_bytes, + src_rows + (src_current - tail_rows) * row_bytes, + tail_rows * row_bytes); + } + } + } + } +} + +void Model::reset_component_cache_states(Component& comp) { + for (const auto& state : comp.cache_states) { + for (int node_id : {state.key_node_id, state.value_node_id}) { + if (node_id < 0) continue; + const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); + if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue; + void* ptr = comp.graph->get_output(static_cast(node_id)); + if (!ptr) continue; + auto* metadata = static_cast(ptr); + metadata[0] = 0; + } + } +} + +size_t Model::component_chunk_tokens(const Component& comp, const std::string& input_name) const { + int idx = input_index(comp, input_name); + if (idx < 0) return 0; + const auto& desc = comp.graph->get_output_buffer(static_cast(comp.runtime_input_node_ids[idx])); + if (desc.shape.size() >= 2 && desc.shape[0] == 1) return desc.shape[1]; + return desc.shape.empty() ? 0 : desc.shape[0]; +} + +size_t Model::component_output_tokens(const Component& comp, const std::string& output_name) const { + for (size_t i = 0; i < comp.logical_outputs.size() && i < comp.output_node_ids.size(); ++i) { + if (comp.logical_outputs[i] != output_name) continue; + const auto& desc = comp.graph->get_output_buffer(static_cast(comp.output_node_ids[i])); + if (desc.shape.size() >= 2 && desc.shape[0] == 1) return desc.shape[1]; + return desc.shape.empty() ? 0 : desc.shape[0]; + } + return 0; +} + +size_t Model::run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, bool prepare_decode) { + if (decode_route_ != DecodeRoute::CACHED_STEP || !encoder_ || !decoder_ || !decoder_prefill_) return 0; + if (start_position != 0) return 0; + if (!cache_states_compatible(*decoder_prefill_, *decoder_)) return 0; + size_t component_tokens = component_chunk_tokens(*decoder_prefill_, "inputs_embeds"); + if (component_tokens <= 1) return 0; + size_t effective_chunk = chunk_size > 0 ? std::min(chunk_size, component_tokens) : component_tokens; + if (effective_chunk != component_tokens) effective_chunk = component_tokens; + if (tokens.size() < effective_chunk) return 0; + + size_t encoder_chunk = 0; + if (prefill_encoder_ && input_index(*prefill_encoder_, "input_ids") >= 0 && input_index(*prefill_encoder_, "position_ids") >= 0) { + encoder_chunk = component_chunk_tokens(*prefill_encoder_, "input_ids"); + if (encoder_chunk == 0 || effective_chunk % encoder_chunk != 0) { + encoder_chunk = 0; + } + } + + size_t processed = 0; + while (processed + effective_chunk <= tokens.size()) { + for (size_t i = 0; i < decoder_prefill_->input_buffers.size(); ++i) { + std::fill(decoder_prefill_->input_buffers[i].begin(), decoder_prefill_->input_buffers[i].end(), 0); + } + if (encoder_chunk > 0) { + for (size_t chunk_offset = 0; chunk_offset < effective_chunk; chunk_offset += encoder_chunk) { + for (size_t i = 0; i < prefill_encoder_->input_buffers.size(); ++i) { + std::fill(prefill_encoder_->input_buffers[i].begin(), prefill_encoder_->input_buffers[i].end(), 0); + } + for (size_t i = 0; i < encoder_chunk; ++i) { + write_int_input_at(*prefill_encoder_, "input_ids", i, static_cast(tokens[processed + chunk_offset + i])); + write_int_input_at(*prefill_encoder_, "position_ids", i, static_cast(start_position + processed + chunk_offset + i)); + } + prefill_encoder_->graph->execute(); + copy_component_outputs_to_chunk_inputs_range(*prefill_encoder_, *decoder_prefill_, chunk_offset); + } + } else { + for (size_t i = 0; i < effective_chunk; ++i) { + run_encoder_step(tokens[processed + i], start_position + processed + i); + copy_component_outputs_to_chunk_inputs(*encoder_, *decoder_prefill_, i); + } + } + decoder_prefill_->graph->execute(); + processed += effective_chunk; + } + if (processed > 0 && prepare_decode) { + for (size_t i = 0; i < decoder_->input_buffers.size(); ++i) { + std::fill(decoder_->input_buffers[i].begin(), decoder_->input_buffers[i].end(), 0); + } + copy_cache_states(*decoder_prefill_, *decoder_); + } + return processed; +} + +void Model::run_full_context_text() { + if (!encoder_ || !decoder_ || context_tokens_.empty()) return; + int input_ids_idx = input_index(*encoder_, "input_ids"); + int attention_mask_idx = input_index(*encoder_, "attention_mask"); + if (input_ids_idx < 0 || attention_mask_idx < 0) { + throw std::runtime_error("text_lm_encoder requires input_ids and attention_mask inputs"); + } + size_t input_node = static_cast(encoder_->runtime_input_node_ids[input_ids_idx]); + const auto& input_desc = encoder_->graph->get_output_buffer(input_node); + if (context_tokens_.size() > input_desc.total_size) { + throw std::runtime_error("context exceeds transpiled text_lm_encoder capacity"); + } + std::fill(encoder_->input_buffers[input_ids_idx].begin(), encoder_->input_buffers[input_ids_idx].end(), 0); + std::fill(encoder_->input_buffers[attention_mask_idx].begin(), encoder_->input_buffers[attention_mask_idx].end(), 0); + for (size_t i = 0; i < context_tokens_.size(); ++i) { + write_int_input_at(*encoder_, "input_ids", i, static_cast(context_tokens_[i])); + write_int_input_at(*encoder_, "attention_mask", i, 1); + } + encoder_->graph->execute(); + for (size_t i = 0; i < encoder_->output_node_ids.size() && i < encoder_->logical_outputs.size(); ++i) { + const std::string& out_name = encoder_->logical_outputs[i]; + int dst_idx = input_index(*decoder_, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(encoder_->output_node_ids[i]); + const auto& src_desc = encoder_->graph->get_output_buffer(src_node); + void* src_ptr = encoder_->graph->get_output(src_node); + std::memcpy(decoder_->input_buffers[dst_idx].data(), src_ptr, src_desc.byte_size); + } + last_logit_position_ = context_tokens_.empty() ? 0 : context_tokens_.size() - 1; decoder_->graph->execute(); } @@ -493,7 +987,8 @@ uint32_t Model::argmax_last_logits() { void* ptr = decoder_->graph->get_output(out_node); size_t vocab = desc.shape.empty() ? 0 : desc.shape.back(); size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; - size_t row_off = (seq > 0 ? (seq - 1) * vocab : 0); + size_t row = decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT ? std::min(last_logit_position_, seq > 0 ? seq - 1 : 0) : (seq > 0 ? seq - 1 : 0); + size_t row_off = row * vocab; uint32_t best = 0; float best_v = -std::numeric_limits::infinity(); if (desc.precision == Precision::FP32) { @@ -512,11 +1007,19 @@ uint32_t Model::argmax_last_logits() { return best; } -void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& /*profile_file*/) { - for (size_t i = 0; i < tokens.size(); ++i) { - run_step(tokens[i], cache_total_seq_len_ + i, /*read_logits=*/false); +void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& /*profile_file*/, bool prepare_decode) { + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + if (!context_tokens_.empty()) run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + return; + } + size_t processed = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), prepare_decode); + cache_total_seq_len_ += processed; + for (size_t i = processed; i < tokens.size(); ++i) { + run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false); + ++cache_total_seq_len_; } - cache_total_seq_len_ += tokens.size(); } void Model::prefill_with_images(const std::vector& tokens, @@ -601,31 +1104,6 @@ void write_tokens_buffer(std::vector& buf, Precision prec, } // namespace -void Model::copy_cache_state(const Component& src, Component& dst) { - if (src.cache_states.empty() || dst.cache_states.empty()) return; - if (src.cache_states.size() != dst.cache_states.size()) { - throw std::runtime_error("cache state count mismatch between " + src.name + " and " + dst.name); - } - for (size_t i = 0; i < src.cache_states.size(); ++i) { - const auto& s = src.cache_states[i]; - const auto& d = dst.cache_states[i]; - if (s.layer_key != d.layer_key) { - throw std::runtime_error("cache layer mismatch: " + s.layer_key + " vs " + d.layer_key); - } - for (auto pair : {std::pair{s.key_node_id, d.key_node_id}, - std::pair{s.value_node_id, d.value_node_id}}) { - const auto& sd = src.graph->get_output_buffer(static_cast(pair.first)); - const auto& dd = dst.graph->get_output_buffer(static_cast(pair.second)); - if (sd.byte_size != dd.byte_size) { - throw std::runtime_error("cache byte size mismatch for layer " + s.layer_key); - } - std::memcpy(dst.graph->get_output(static_cast(pair.second)), - src.graph->get_output(static_cast(pair.first)), - sd.byte_size); - } - } -} - bool Model::run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, const std::vector& audio_features) { @@ -842,7 +1320,7 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, decoder_prefill_chunk_->graph->execute(); } if (whole_chunks_end > 0 && decoder_ != nullptr) { - copy_cache_state(*decoder_prefill_chunk_, *decoder_); + copy_cache_states(*decoder_prefill_chunk_, *decoder_); } for (size_t pos = whole_chunks_end; pos < valid_seq; ++pos) { for (const auto& kv : store_bytes) { @@ -985,6 +1463,15 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* size_t /*top_k*/, const std::string& /*profile_file*/, float* out_entropy, float /*min_p*/, float /*repetition_penalty*/) { if (tokens.empty()) return 0; + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + if (out_entropy) *out_entropy = 0.0f; + uint32_t result = argmax_last_logits(); + record_sampled_token(result); + return result; + } for (size_t i = 0; i + 1 < tokens.size(); ++i) { run_step(tokens[i], cache_total_seq_len_ + i, /*read_logits=*/false); } @@ -1201,20 +1688,13 @@ std::vector Model::get_audio_embeddings(const std::vector& /*mel_b void Model::reset_cache() { cache_total_seq_len_ = 0; + last_logit_position_ = 0; + context_tokens_.clear(); token_history_.clear(); for (auto& kv : components_) { Component& comp = kv.second; if (!comp.graph) continue; - for (const auto& cs : comp.cache_states) { - for (int node_id : {cs.key_node_id, cs.value_node_id}) { - if (node_id < 0) continue; - const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); - if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue; - void* data = comp.graph->get_output(static_cast(node_id)); - if (!data) continue; - *static_cast(data) = 0; - } - } + reset_component_cache_states(comp); } } diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 04fa56207..ad722f7ca 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -146,24 +146,33 @@ void compute_kv_cache_append_node( size_t new_total = current_len + new_seq_len; bool needs_eviction = new_total > window; if (needs_eviction) { - size_t sink_eff = std::min(sink, window); - if (new_seq_len >= window) { - size_t src_skip = new_seq_len - window; - std::memcpy(fp16_base, source + src_skip * stride, window * stride * sizeof(__fp16)); - } else { - size_t remaining = (new_seq_len + sink_eff < window) - ? window - sink_eff - new_seq_len : 0; - size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; - if (remaining > 0 && shift_src > sink_eff) { - std::memmove( - fp16_base + sink_eff * stride, - fp16_base + shift_src * stride, - remaining * stride * sizeof(__fp16)); + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) { + if (tail_capacity > 0) { + size_t source_offset = new_seq_len - tail_capacity; + std::memcpy( + fp16_base + keep_sink * stride, + source + source_offset * stride, + tail_capacity * stride * sizeof(__fp16)); } - size_t append_offset = window - new_seq_len; - std::memcpy(fp16_base + append_offset * stride, source, new_seq_len * stride * sizeof(__fp16)); + meta->current_seq_len = keep_sink + tail_capacity; + *node.output_buffer.data_as() = static_cast(meta->current_seq_len); + return; } - meta->current_seq_len = window; + + size_t remaining = tail_capacity - new_seq_len; + remaining = std::min(remaining, current_len - keep_sink); + size_t shift_src = current_len - remaining; + if (remaining > 0 && shift_src > keep_sink) { + std::memmove( + fp16_base + keep_sink * stride, + fp16_base + shift_src * stride, + remaining * stride * sizeof(__fp16)); + } + size_t append_offset = keep_sink + remaining; + std::memcpy(fp16_base + append_offset * stride, source, new_seq_len * stride * sizeof(__fp16)); + meta->current_seq_len = append_offset + new_seq_len; } else { std::memcpy(fp16_base + current_len * stride, source, new_seq_len * stride * sizeof(__fp16)); meta->current_seq_len = new_total; @@ -183,37 +192,43 @@ void compute_kv_cache_append_node( bool needs_eviction = new_total > window; if (needs_eviction) { - size_t sink_eff = std::min(sink, window); - if (new_seq_len >= window) { - size_t src_skip = new_seq_len - window; - cactus_quantize_kv_fp16_to_int8( - new_kv.data_as<__fp16>() + src_skip * (kv_heads * hdim), - int8_base, - scale_base, - window, kv_heads, hdim); - } else { - size_t remaining = (new_seq_len + sink_eff < window) - ? window - sink_eff - new_seq_len : 0; - size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; - - if (remaining > 0 && shift_src > sink_eff) { - std::memmove(int8_base + sink_eff * int8_stride, - int8_base + shift_src * int8_stride, - remaining * int8_stride); - std::memmove(scale_base + sink_eff * scale_stride, - scale_base + shift_src * scale_stride, - remaining * scale_stride * sizeof(float)); + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) { + if (tail_capacity > 0) { + size_t source_offset = new_seq_len - tail_capacity; + cactus_quantize_kv_fp16_to_int8( + new_kv.data_as<__fp16>() + source_offset * int8_stride, + int8_base + keep_sink * int8_stride, + scale_base + keep_sink * scale_stride, + tail_capacity, kv_heads, hdim); } + meta->current_seq_len = keep_sink + tail_capacity; + *node.output_buffer.data_as() = static_cast(meta->current_seq_len); + return; + } - size_t append_offset = window - new_seq_len; - cactus_quantize_kv_fp16_to_int8( - new_kv.data_as<__fp16>(), - int8_base + append_offset * int8_stride, - scale_base + append_offset * scale_stride, - new_seq_len, kv_heads, hdim); + size_t remaining = tail_capacity - new_seq_len; + remaining = std::min(remaining, current_len - keep_sink); + size_t shift_src = current_len - remaining; + + if (remaining > 0 && shift_src > keep_sink) { + std::memmove(int8_base + keep_sink * int8_stride, + int8_base + shift_src * int8_stride, + remaining * int8_stride); + std::memmove(scale_base + keep_sink * scale_stride, + scale_base + shift_src * scale_stride, + remaining * scale_stride * sizeof(float)); } - meta->current_seq_len = window; + size_t append_offset = keep_sink + remaining; + cactus_quantize_kv_fp16_to_int8( + new_kv.data_as<__fp16>(), + int8_base + append_offset * int8_stride, + scale_base + append_offset * scale_stride, + new_seq_len, kv_heads, hdim); + + meta->current_seq_len = append_offset + new_seq_len; } else { cactus_quantize_kv_fp16_to_int8( new_kv.data_as<__fp16>(), @@ -260,9 +275,15 @@ void compute_attention_cached_node( size_t new_seq_len = key_new_buf.total_size / (kv_heads * hdim); size_t history_len = (cache_len >= new_seq_len) ? cache_len - new_seq_len : 0; + bool cache_only_attention = false; size_t position_offset = node.params.position_offset; if (position_offset == std::numeric_limits::max()) { position_offset = history_len; + } else if (position_offset == std::numeric_limits::max() - 1) { + position_offset = (cache_len >= seq_len) ? cache_len - seq_len : 0; + history_len = cache_len; + new_seq_len = 0; + cache_only_attention = true; } if (k_cache_buf.precision == Precision::FP16 || v_cache_buf.precision == Precision::FP16) { @@ -284,19 +305,19 @@ void compute_attention_cached_node( return; } - cactus_attention_hybrid_int8_fp16( - query_buf.data_as<__fp16>(), - cached_keys, - cached_values, - k_scales, - v_scales, - key_new_buf.data_as<__fp16>(), - val_new_buf.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, history_len, seq_len, - num_q_heads, kv_heads, hdim, - node.params.scale, - position_offset, + cactus_attention_hybrid_int8_fp16( + query_buf.data_as<__fp16>(), + cached_keys, + cached_values, + k_scales, + v_scales, + key_new_buf.data_as<__fp16>(), + val_new_buf.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, history_len, cache_only_attention ? 0 : seq_len, + num_q_heads, kv_heads, hdim, + node.params.scale, + position_offset, true, node.params.window_size, KV_QUANT_GROUP_SIZE, diff --git a/cactus-graph/tests/test_cache.cpp b/cactus-graph/tests/test_cache.cpp index 38947597f..21cbdca7d 100644 --- a/cactus-graph/tests/test_cache.cpp +++ b/cactus-graph/tests/test_cache.cpp @@ -163,6 +163,32 @@ bool test_kv_cache_append_eviction() { return true; } +bool test_kv_cache_append_full_window_eviction() { + CactusGraph g; + + const size_t kv_heads = 1, head_dim = 16; + const size_t window = 8, sink = 2; + + size_t cache_node = g.kv_cache_state(window, kv_heads, head_dim, window, sink); + + for (int pass = 0; pass < 2; pass++) { + if (pass > 0) g.soft_reset(); + const size_t tokens = window; + const size_t elements = tokens * kv_heads * head_dim; + size_t kv_input = g.input({elements}, Precision::FP16); + std::vector<__fp16> data(elements, static_cast<__fp16>(static_cast(pass + 1))); + g.set_input(kv_input, data.data(), Precision::FP16); + g.kv_cache_append(kv_input, cache_node, window, sink); + g.execute(); + + auto* raw = static_cast(g.get_output(cache_node)); + uint64_t seq = *reinterpret_cast(raw); + if (seq != window) return false; + } + + return true; +} + bool test_attention_cached_basic() { const size_t b = 1, s = 1, h = 2, kv = 2, d = 16; const size_t max_seq = 64; @@ -493,6 +519,7 @@ int main() { runner.run_test("KV Cache Append Basic", test_kv_cache_append_basic()); runner.run_test("KV Cache Append Multiple", test_kv_cache_append_multiple()); runner.run_test("KV Cache Append Eviction", test_kv_cache_append_eviction()); + runner.run_test("KV Cache Append Full Window Eviction", test_kv_cache_append_full_window_eviction()); runner.run_test("Attention Cached Basic", test_attention_cached_basic()); runner.run_test("Attention Cached Multistep", test_attention_cached_multistep()); runner.run_test("KV Cache Invalidate", test_kv_cache_invalidate()); diff --git a/cactus-graph/tests/test_io.cpp b/cactus-graph/tests/test_io.cpp index 826bf54dc..298e6c440 100644 --- a/cactus-graph/tests/test_io.cpp +++ b/cactus-graph/tests/test_io.cpp @@ -75,7 +75,7 @@ bool test_graph_save_load() { size_t input_a = graph.input({2, 3}, Precision::FP16); size_t input_b = graph.input({2, 3}, Precision::FP16); size_t sum_id = graph.add(input_a, input_b); - graph.pow(sum_id, 2.0f); + size_t pow_id = graph.pow(sum_id, 2.0f); graph.save(filename); @@ -88,8 +88,8 @@ bool test_graph_save_load() { return false; } - if (sg.graph_inputs.size() != 2 || sg.graph_inputs[0] != 0 || - sg.graph_inputs[1] != 1) { + if (sg.graph_inputs.size() != 2 || sg.graph_inputs[0] != input_a || + sg.graph_inputs[1] != input_b) { std::cout << "[graph_save_load] unexpected graph_inputs:"; for (uint32_t idx : sg.graph_inputs) { std::cout << " " << idx; @@ -99,7 +99,7 @@ bool test_graph_save_load() { return false; } - if (sg.graph_outputs.size() != 1 || sg.graph_outputs[0] != 3) { + if (sg.graph_outputs.size() != 1 || sg.graph_outputs[0] != pow_id) { std::cout << "[graph_save_load] unexpected graph_outputs:"; for (uint32_t idx : sg.graph_outputs) { std::cout << " " << idx; diff --git a/cactus-graph/tests/test_utils.h b/cactus-graph/tests/test_utils.h index 31bd14945..511fd446e 100644 --- a/cactus-graph/tests/test_utils.h +++ b/cactus-graph/tests/test_utils.h @@ -135,7 +135,7 @@ inline void fill_random_fp16(std::vector<__fp16>& data) { } inline size_t runtime_id_from_serialized_index(uint32_t serialized_index) { - return static_cast(serialized_index) + 1; + return static_cast(serialized_index); } inline bool test_basic_operation(const std::string& op_name, diff --git a/cactus-kernels/src/threading.h b/cactus-kernels/src/threading.h index 665fecc69..8d5c7cdcc 100644 --- a/cactus-kernels/src/threading.h +++ b/cactus-kernels/src/threading.h @@ -240,11 +240,18 @@ namespace CactusThreading { inline bool pin_current_thread_to_cores(const std::vector& cores) { if (cores.empty()) return false; + cpu_set_t current_mask; + const bool has_current_mask = sched_getaffinity(0, sizeof(current_mask), ¤t_mask) == 0; cpu_set_t mask; CPU_ZERO(&mask); + bool selected = false; for (int core : cores) { - CPU_SET(core, &mask); + if (!has_current_mask || CPU_ISSET(core, ¤t_mask)) { + CPU_SET(core, &mask); + selected = true; + } } + if (!selected && has_current_mask) return true; return sched_setaffinity(0, sizeof(mask), &mask) == 0; } #endif diff --git a/cactus/build.sh b/cactus/build.sh index b139b23e0..dc41a98d3 100755 --- a/cactus/build.sh +++ b/cactus/build.sh @@ -32,7 +32,16 @@ echo "Building Cactus library..." cd "$(dirname "$0")/../cactus" -rm -rf build +LOCK_DIR="${TMPDIR:-/tmp}/cactus-build.lock" +while ! mkdir "$LOCK_DIR" 2>/dev/null; do + if [ -f "$LOCK_DIR/pid" ] && ! kill -0 "$(cat "$LOCK_DIR/pid" 2>/dev/null)" 2>/dev/null; then + rm -rf "$LOCK_DIR" + continue + fi + sleep 2 +done +echo "$$" > "$LOCK_DIR/pid" +trap 'rm -rf "$LOCK_DIR"' EXIT mkdir -p build cd build diff --git a/python/cactus/cli/compile.py b/python/cactus/cli/compile.py index 9681600ab..423f8ab2b 100644 --- a/python/cactus/cli/compile.py +++ b/python/cactus/cli/compile.py @@ -75,6 +75,14 @@ def cmd_build(args): if result.returncode != 0: print_color(RED, "Failed to build cactus library") return 1 + for _ in range(50): + if lib_path.exists(): + break + import time + time.sleep(0.1) + if not lib_path.exists(): + print_color(RED, f"Build did not produce {lib_path}") + return 1 tests_dir = PROJECT_ROOT / "cactus-engine" / "tests" build_dir = tests_dir / "build" build_dir.mkdir(parents=True, exist_ok=True) diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index 118683495..6c0aebd71 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -24,6 +24,8 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("ignored", "none", None, component, False, "none", "no output filename") if "norm" in out.lower(): return TensorPolicy("fallback", "FP16", None, component, False, "none", "norm tensor") + if family == "gemma4" and name == "model.embed_vision.embedding_projection.weight": + return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, False, "hadamard", "vision embedding projection has no stable Hessian target") if family in {"parakeet", "parakeet_tdt"} and out.endswith(".bias") and ( "conv_" in out or out.startswith("subsampling_") or out.startswith("ctc_head_") ): @@ -32,18 +34,20 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("fallback", "FP16", None, component, False, "none", "lstm recurrent tensor") if family == "parakeet_tdt" and out.startswith("tdt_"): return TensorPolicy("fallback", "FP16", None, component, False, "none", "tdt decoder tensor") + if family in {"parakeet", "parakeet_tdt"} and "conv_pointwise" in out and len(shape) == 3 and shape[2] == 1: + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "pointwise conv tensor") + if "conv_depthwise.weights" in out and len(shape) == 3 and shape[1] == 1: + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "depthwise conv tensor") if family in {"parakeet", "parakeet_tdt"} and out.startswith("layer_") and ( "conv_pointwise" in out or "conv_depthwise" in out ): return TensorPolicy("fallback", "FP16", None, component, False, "none", "conformer conv tensor") if family == "whisper" and out.startswith("encoder.layer_"): return TensorPolicy("fallback", "FP16", None, component, False, "none", "whisper encoder tensor") + if family == "gemma4" and component == "audio" and (name.endswith(".bias") or out.endswith(".bias") or ".bias." in out): + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "bias tensor") if name.endswith(".bias") or out.endswith(".bias") or ".bias." in out: return TensorPolicy("fallback", "FP16", None, component, False, "none", "bias tensor") - if family in {"parakeet", "parakeet_tdt"} and "conv_pointwise" in out and len(shape) == 3 and shape[2] == 1: - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "pointwise conv tensor") - if "conv_depthwise.weights" in out and len(shape) == 3 and shape[1] == 1: - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "depthwise conv tensor") if len(shape) != 2: return TensorPolicy("fallback", "FP16", None, component, False, "none", "non-2d tensor") if "position_embedding" in out.lower() or "pos_embed" in out.lower() or "embed_positions" in out.lower(): diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index d3daf70f3..0402ee77d 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -303,6 +303,7 @@ def _write_component_bundle( "audio_encoder", "vision_encoder", "lm_encoder", + "lm_encoder_text_chunk", "decoder_prefill_chunk", "decoder", "lm_encoder_step", @@ -2471,6 +2472,14 @@ def _load_transformers_bundle( ): config = _load_config_json(model_id) config_model_type = str(config.get("model_type", "") or "").lower() + if not config_model_type: + normalized_model_id = model_id.lower().replace("_", "-") + if "qwen3-vl" in normalized_model_id: + config_model_type = "qwen3_vl" + elif "lfm2.5-vl" in normalized_model_id or "lfm2-vl" in normalized_model_id: + config_model_type = "lfm2_vl" + if config_model_type: + config = {"model_type": config_model_type} external_transformers_site_packages = _ensure_transformers_supports_model_type(config_model_type) patch_note = _patch_transformers_torchvision_probe() if patch_note: @@ -2524,7 +2533,7 @@ def _load_transformers_bundle( f"Could not load tokenizer for {model_id}:\n" + "\n".join(tokenizer_errors) ) - if config_model_type in {"lfm2_vl", "qwen3_5"}: + if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}: model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -2535,7 +2544,7 @@ def _load_transformers_bundle( tie_note = _tie_lfm2_vl_lm_head_if_needed(model) if tie_note: print(f"note={tie_note}") - elif config_model_type == "qwen3_5" and isinstance(config.get("text_config"), dict): + elif config_model_type in {"qwen3_5", "qwen3_vl"} and isinstance(config.get("text_config"), dict): model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -2584,7 +2593,7 @@ def _load_transformers_bundle( + processor_config_hint ) - if config_model_type in {"lfm2_vl", "qwen3_5"}: + if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}: model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -3017,7 +3026,7 @@ def main() -> int: system_prompt=args.system_prompt, enable_thinking_if_supported=args.enable_thinking, ) - elif config_model_type == "qwen3_5": + elif config_model_type in {"qwen3_5", "qwen3_vl"}: prepared = _prepare_qwen3_5_multimodal_inputs( processor_or_tokenizer, prompt=args.prompt, diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index 4925e8e47..c6639295a 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -23,6 +23,16 @@ _DYNAMIC_KV_POSITION_OFFSET = (1 << 64) - 1 +_DYNAMIC_KV_QUERY_POSITION_OFFSET = (1 << 64) - 2 +_KV_CACHE_TRANSPARENT_OPS = { + "expand", + "precision_cast", + "reshape", + "slice", + "transpose", + "unsqueeze", + "view", +} @dataclass @@ -254,6 +264,13 @@ def _lower_attention_with_internal_kv_cache( sink_size = int(ir.meta.get("cache_sink_size", 4) or 4) g.kv_cache_append(key, k_cache, window_size=window_size, sink_size=sink_size) g.kv_cache_append(value, v_cache, window_size=window_size, sink_size=sink_size) + kv_input_cache_states: dict[tuple[str, str], tuple[Tensor, Tensor, int, int]] = env.setdefault( + "__internal_kv_cache_by_kv_inputs", + {}, + ) # type: ignore[assignment] + cache_entry = (k_cache, v_cache, window_size, int(value_shape[3])) + kv_input_cache_states[(node.inputs[1], node.inputs[2])] = cache_entry + kv_input_cache_states[_canonical_kv_cache_input_key(ir, node)] = cache_entry out = g.attention_cached( query, key, @@ -270,6 +287,66 @@ def _lower_attention_with_internal_kv_cache( return [g.permute(out, (0, 2, 1, 3))] +def _lower_attention_with_shared_internal_kv_cache( + g: Graph, + ir: IRGraph, + env: dict[str, Any], + node: IRNode, + *, + query: Tensor, + key: Tensor, + value: Tensor, + output_layout: str, +) -> list[Tensor] | None: + kv_input_cache_states: dict[tuple[str, str], tuple[Tensor, Tensor, int, int]] = env.get( + "__internal_kv_cache_by_kv_inputs", + {}, + ) + cache_state = kv_input_cache_states.get((node.inputs[1], node.inputs[2])) + if cache_state is None: + cache_state = kv_input_cache_states.get(_canonical_kv_cache_input_key(ir, node)) + if cache_state is None: + return None + k_cache, v_cache, window_size, v_head_dim = cache_state + out = g.attention_cached( + query, + key, + value, + k_cache, + v_cache, + scale=_resolve_attention_scale(node, query), + position_offset=_DYNAMIC_KV_QUERY_POSITION_OFFSET, + window_size=window_size, + v_head_dim=v_head_dim, + ) + if output_layout == "bthd": + return [out] + return [g.permute(out, (0, 2, 1, 3))] + + +def _canonical_kv_cache_input_key(ir: IRGraph, node: IRNode) -> tuple[str, str]: + return ( + _canonical_kv_cache_value_id(ir, node.inputs[1]), + _canonical_kv_cache_value_id(ir, node.inputs[2]), + ) + + +def _canonical_kv_cache_value_id(ir: IRGraph, value_id: str) -> str: + current = value_id + seen: set[str] = set() + while current not in seen: + seen.add(current) + value = ir.values.get(current) + producer_id = None if value is None else value.producer + if producer_id is None: + break + producer = ir.nodes.get(producer_id) + if producer is None or producer.op not in _KV_CACHE_TRANSPARENT_OPS or len(producer.inputs) != 1: + break + current = producer.inputs[0] + return current + + def _should_lower_conv1d_with_internal_conv_cache(ir: IRGraph, node: IRNode, x: Tensor, weight: Tensor) -> bool: if not bool(ir.meta.get("use_internal_conv_cache", False)): return False @@ -1090,6 +1167,18 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> if mask_tensor is not None: mask_tensor = _ensure_fp16_tensor(g, mask_tensor) mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + shared_cached = _lower_attention_with_shared_internal_kv_cache( + g, + ir, + env, + node, + query=query, + key=key, + value=value, + output_layout=output_layout, + ) + if shared_cached is not None: + return shared_cached if _should_lower_attention_with_internal_kv_cache(ir, node): return _lower_attention_with_internal_kv_cache( g, diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 62be8a529..914f693f5 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -18,6 +18,7 @@ _GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE = 1.0 / 64.0 +_UNSET = object() try: from transformers.models.gemma4.modeling_gemma4 import create_bidirectional_mask as _GEMMA4_CREATE_BIDIRECTIONAL_MASK # type: ignore except Exception: @@ -330,6 +331,10 @@ def _torch_is_compiling() -> bool: def _gemma4_cpu_safe_text_mlp_enabled(layer: torch.nn.Module, hidden_states: torch.Tensor) -> bool: if hidden_states.device.type != "cpu" or hidden_states.dtype != torch.float16: return False + return _gemma4_text_mlp_manual_available(layer) + + +def _gemma4_text_mlp_manual_available(layer: torch.nn.Module) -> bool: if bool(getattr(layer, "enable_moe_block", False)): return False mlp = getattr(layer, "mlp", None) @@ -417,8 +422,12 @@ def _gemma4_text_attention_forward( value_states = attn.v_norm(value_states) value_states = value_states.transpose(1, 2) - if bool(getattr(attn, "store_full_length_kv", False)): - shared_kv_states[int(attn.layer_idx)] = (key_states, value_states) + layer_idx = getattr(attn, "layer_idx", None) + if layer_idx is not None and ( + bool(getattr(attn, "store_full_length_kv", False)) + or not bool(getattr(attn, "is_kv_shared_layer", False)) + ): + shared_kv_states[int(layer_idx)] = (key_states, value_states) attention_interface = eager_attention_forward if getattr(attn.config, "_attn_implementation", "eager") != "eager": @@ -450,7 +459,8 @@ def _gemma4_text_decoder_layer_forward( use_cache: bool = False, shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, ) -> torch.Tensor: - if not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): + manual_attention_required = shared_kv_states is not None + if not manual_attention_required and not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): return layer( hidden_states, position_embeddings=position_embeddings, @@ -460,6 +470,8 @@ def _gemma4_text_decoder_layer_forward( past_key_values=past_key_values, use_cache=use_cache, ) + if not _gemma4_text_mlp_manual_available(layer): + raise RuntimeError("Gemma4 KV sharing requires a supported text MLP structure") residual = hidden_states @@ -506,6 +518,10 @@ def _gemma4_text_backbone_forward( per_layer_inputs: torch.Tensor | None, causal_mask_mapping: dict[str, torch.Tensor], position_ids: torch.LongTensor, + layer_start: int = 0, + layer_end: int | None = None, + apply_norm: bool = True, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None | object = _UNSET, ) -> torch.Tensor: hidden_states = inputs_embeds layer_types = tuple(dict.fromkeys(getattr(backbone.config, "layer_types", ()))) @@ -513,10 +529,14 @@ def _gemma4_text_backbone_forward( layer_type: backbone.rotary_emb(hidden_states, position_ids, layer_type) for layer_type in layer_types } - shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + if shared_kv_states is _UNSET: + num_shared_layers = int(getattr(backbone.config, "num_kv_shared_layers", 0) or 0) + shared_kv_states = {} if num_shared_layers > 0 else None config_layer_types = tuple(getattr(backbone.config, "layer_types", ())) - for layer_index, decoder_layer in enumerate(backbone.layers[: backbone.config.num_hidden_layers]): + end = backbone.config.num_hidden_layers if layer_end is None else min(int(layer_end), int(backbone.config.num_hidden_layers)) + for layer_index in range(max(0, int(layer_start)), end): + decoder_layer = backbone.layers[layer_index] layer_per_input = None if per_layer_inputs is not None: layer_per_input = per_layer_inputs[:, :, decoder_layer.layer_idx, :] @@ -537,7 +557,7 @@ def _gemma4_text_backbone_forward( shared_kv_states=shared_kv_states, ) - return backbone.norm(hidden_states) + return backbone.norm(hidden_states) if apply_norm else hidden_states def _gemma4_strip_audio_padding(audio_output: object) -> torch.Tensor: @@ -837,6 +857,7 @@ def _gemma4_compute_native_like_image_features( projection = getattr(embed_vision, "embedding_projection", None) if not isinstance(projection, torch.nn.Linear): raise TypeError("Gemma4 vision embedder is missing embedding_projection") + pooled_hidden = pooled_hidden.to(dtype=projection.weight.dtype) projected = F.linear( pooled_hidden, projection.weight, @@ -1688,6 +1709,10 @@ def get_transpile_metadata(self): } +class Lfm2VlLMEncoderTextChunkAdapter(Lfm2VlLMEncoderStepAdapter): + pass + + class Lfm2VlDecoderAdapter(torch.nn.Module): def __init__( self, @@ -3040,6 +3065,75 @@ class Gemma4DecoderStepAdapter(Gemma4DecoderAdapter): pass +class Gemma4DecoderPrefillChunkAdapter(Gemma4DecoderAdapter): + def forward( + self, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor, + position_ids: torch.LongTensor, + ) -> torch.Tensor: + normalized_per_layer_inputs = per_layer_inputs + if normalized_per_layer_inputs.numel() == 0: + normalized_per_layer_inputs = None + attention_mask = torch.ones( + position_ids.shape, + dtype=torch.long, + device=position_ids.device, + ) + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + num_layers = int(getattr(self.backbone.config, "num_hidden_layers", len(self.backbone.layers))) + num_shared_layers = int(getattr(self.backbone.config, "num_kv_shared_layers", 0) or 0) + first_shared_layer = max(0, num_layers - num_shared_layers) + if first_shared_layer <= 0 or first_shared_layer >= num_layers or int(inputs_embeds.shape[1]) <= 1: + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + ) + else: + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + layer_end=first_shared_layer, + apply_norm=False, + shared_kv_states=shared_kv_states, + ) + tail_masks = { + key: value[:, :, -1:, :] if value.ndim == 4 else value + for key, value in causal_mask_mapping.items() + } + tail_per_layer_inputs = ( + None + if normalized_per_layer_inputs is None + else normalized_per_layer_inputs[:, -1:, :, :] + ) + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=hidden_states[:, -1:, :], + per_layer_inputs=tail_per_layer_inputs, + causal_mask_mapping=tail_masks, + position_ids=position_ids[:, -1:], + layer_start=first_shared_layer, + apply_norm=True, + shared_kv_states=shared_kv_states, + ) + logits = self.model.lm_head(hidden_states[:, -1:, :]) + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def _build_gemma4_multimodal_component_specs( model: torch.nn.Module, *, @@ -3076,32 +3170,27 @@ def _require(component: str) -> None: _require("lm_encoder_step") _require("lm_encoder_media_step") _require("decoder_step") - if "decoder_prefill_chunk" in requested_set: + if "lm_encoder" in requested_set: _require("vision_encoder") _require("audio_encoder") _require("lm_encoder") + if "decoder_prefill_chunk" in requested_set: _require("decoder_prefill_chunk") - elif "lm_encoder" in requested_set: + if "vision_encoder" in requested_set: _require("vision_encoder") + if "audio_encoder" in requested_set: _require("audio_encoder") - _require("lm_encoder") - else: - if "vision_encoder" in requested_set: - _require("vision_encoder") - if "audio_encoder" in requested_set: - _require("audio_encoder") - if "lm_encoder_step" in requested_set: - _require("lm_encoder_step") - if "lm_encoder_text_chunk" in requested_set: - _require("lm_encoder_text_chunk") - if "lm_encoder_media_step" in requested_set: - _require("lm_encoder_media_step") - if "lm_encoder_media_chunk" in requested_set: - _require("lm_encoder_media_chunk") - if "decoder_step" in requested_set: - _require("lm_encoder_step") - _require("lm_encoder_media_step") - _require("decoder_step") + if "lm_encoder_step" in requested_set: + _require("lm_encoder_step") + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "lm_encoder_media_step" in requested_set: + _require("lm_encoder_media_step") + if "lm_encoder_media_chunk" in requested_set: + _require("lm_encoder_media_chunk") + if "decoder_step" in requested_set: + _require("lm_encoder_step") + _require("decoder_step") vision_tower = getattr(getattr(model, "model", model), "vision_tower", None) pooling_kernel_size = int(_module_or_config_attr(vision_tower, "pooling_kernel_size", 3) or 3) @@ -3125,6 +3214,7 @@ def _require(component: str) -> None: image_features: torch.Tensor | None = None audio_features: torch.Tensor | None = None decoder_inputs: tuple[torch.Tensor, ...] | None = None + prefill_decoder_inputs: tuple[torch.Tensor, ...] | None = None decoder_step_inputs: tuple[torch.Tensor, ...] | None = None step_input_ids: torch.Tensor | None = None step_position_ids: torch.Tensor | None = None @@ -3135,16 +3225,14 @@ def _require(component: str) -> None: if "vision_encoder" in expanded_components and ( "lm_encoder" in expanded_components or "decoder" in expanded_components - or "decoder_prefill_chunk" in expanded_components ): image_features = vision_encoder(pixel_values, pixel_position_ids) if "audio_encoder" in expanded_components and ( "lm_encoder" in expanded_components or "decoder" in expanded_components - or "decoder_prefill_chunk" in expanded_components ): audio_features = audio_encoder(input_features, input_features_mask) - if "lm_encoder" in expanded_components or "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + if "lm_encoder" in expanded_components or "decoder" in expanded_components: if image_features is None: image_features = vision_encoder(pixel_values, pixel_position_ids) if audio_features is None: @@ -3160,17 +3248,18 @@ def _require(component: str) -> None: weights_dir=weights_dir, native_merge_plan=native_merge_plan, ).eval() - encoder_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_ENCODER_CHUNK", "32") or "32")) + encoder_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_ENCODER_CHUNK", "128") or "128")) encoder_chunk_size = min(encoder_chunk_size, int(input_ids.shape[1])) lm_encoder_step = Gemma4LMEncoderStepAdapter(model, weights_dir=weights_dir).eval() lm_encoder_text_chunk = Gemma4LMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() lm_encoder_media_step = Gemma4LMEncoderMediaStepAdapter(model, weights_dir=weights_dir).eval() lm_encoder_media_chunk = Gemma4LMEncoderMediaChunkAdapter(model, weights_dir=weights_dir).eval() decoder = Gemma4DecoderAdapter(model, weights_dir=weights_dir).eval() + decoder_prefill = Gemma4DecoderPrefillChunkAdapter(model, weights_dir=weights_dir).eval() decoder_step = Gemma4DecoderStepAdapter(model, weights_dir=weights_dir).eval() with torch.no_grad(): - if "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + if "decoder" in expanded_components: decoder_inputs = lm_encoder( input_ids, attention_mask, @@ -3209,6 +3298,8 @@ def _require(component: str) -> None: dtype=torch.long, device=input_ids.device, ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + if "decoder_prefill_chunk" in expanded_components and decoder_inputs is None: + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) if media_step_embeds is not None: hidden_size = int(media_step_embeds.shape[-1]) media_chunk_embeds = torch.zeros( @@ -3268,7 +3359,7 @@ def _require(component: str) -> None: metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) cache_seq_len = max(1024, int(input_ids.shape[1]) + 256) - prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "64") or "64")) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "128") or "128")) prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) if "decoder" in expanded_components: if decoder_inputs is None: @@ -3283,17 +3374,21 @@ def _require(component: str) -> None: metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) if "decoder_prefill_chunk" in expanded_components: - if decoder_inputs is None: + if decoder_inputs is not None: + chunk_inputs = tuple( + value[:, :prefill_chunk_size, ...].contiguous() + if value.ndim >= 2 + else value + for value in decoder_inputs + ) + elif prefill_decoder_inputs is not None: + prefill_chunk_size = int(prefill_decoder_inputs[0].shape[1]) + chunk_inputs = tuple(prefill_decoder_inputs) + else: raise RuntimeError("Gemma4 decoder_prefill_chunk spec requires precomputed decoder inputs") - chunk_inputs = tuple( - value[:, :prefill_chunk_size, ...].contiguous() - if value.ndim >= 2 - else value - for value in decoder_inputs - ) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", - module=decoder, + module=decoder_prefill, example_inputs=chunk_inputs, input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, output_keys=("logits",), @@ -3422,11 +3517,15 @@ def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[tor position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=linear_attn_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3443,7 +3542,7 @@ def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[tor return self.model.lm_head(hidden_states), checkpoints def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3488,11 +3587,15 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch. hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=linear_attn_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3502,7 +3605,7 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch. return self.model.lm_head(hidden_states[:, -1:, :]) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3534,6 +3637,26 @@ def _resolve_qwen35_multimodal_backbone(model: torch.nn.Module) -> torch.nn.Modu raise AttributeError(f"{type(model).__name__} does not expose a Qwen3.5 multimodal backbone") +def _qwen35_layer_types(config) -> tuple[str, ...]: + layer_types = tuple(str(value) for value in (getattr(config, "layer_types", ()) or ())) + if layer_types: + return layer_types + num_layers = int(getattr(config, "num_hidden_layers", 0) or 0) + return tuple("full_attention" for _ in range(num_layers)) + + +def _qwen35_attention_mask_for_layer( + config, + layer_index: int, + *, + causal_mask, + linear_attention_mask, +): + layer_types = _qwen35_layer_types(config) + layer_type = layer_types[layer_index] if layer_index < len(layer_types) else "full_attention" + return linear_attention_mask if layer_type == "linear_attention" else causal_mask + + class Qwen35VisionEncoderAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module, *, image_grid_thw: torch.Tensor): super().__init__() @@ -3677,11 +3800,15 @@ def forward( hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = attention_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=attention_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3690,7 +3817,7 @@ def forward( return self.model.lm_head(hidden_states) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3732,6 +3859,10 @@ def get_transpile_metadata(self): } +class Qwen35LMEncoderTextChunkAdapter(Qwen35LMEncoderStepAdapter): + pass + + class Qwen35EmbedsCausalLMStepAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module): super().__init__() @@ -3762,11 +3893,15 @@ def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> to hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = None if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=None, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3775,7 +3910,7 @@ def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> to return self.model.lm_head(hidden_states[:, -1:, :]) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3790,6 +3925,10 @@ def get_transpile_metadata(self): } +class Qwen35EmbedsCausalLMPrefillChunkAdapter(Qwen35EmbedsCausalLMStepAdapter): + pass + + class Qwen3CausalLMLogitsAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): super().__init__() @@ -4011,6 +4150,8 @@ def _build_qwen3_5_multimodal_component_specs( "vision_encoder", "lm_encoder", "decoder", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", "lm_encoder_step", "decoder_media_step", "decoder_step", @@ -4026,9 +4167,16 @@ def _require(component: str) -> None: _require("vision_encoder") _require("lm_encoder") _require("decoder") + _require("lm_encoder_text_chunk") + _require("decoder_prefill_chunk") if "lm_encoder" in requested_set: _require("vision_encoder") _require("lm_encoder") + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "decoder_prefill_chunk" in requested_set: + _require("lm_encoder_text_chunk") + _require("decoder_prefill_chunk") if "vision_encoder" in requested_set: _require("vision_encoder") if "decoder_step" in requested_set: @@ -4042,11 +4190,14 @@ def _require(component: str) -> None: lm_encoder = Qwen35LMEncoderAdapter(model, input_ids=input_ids).eval() decoder = Qwen35EmbedsCausalLMLogitsAdapter(model).eval() lm_encoder_step = Qwen35LMEncoderStepAdapter(model).eval() + lm_encoder_text_chunk = Qwen35LMEncoderTextChunkAdapter(model).eval() decoder_media_step = Qwen35EmbedsCausalLMStepAdapter(model).eval() + decoder_prefill_chunk = Qwen35EmbedsCausalLMPrefillChunkAdapter(model).eval() decoder_step = Qwen35CausalLMStepAdapter(model, pad_token_id=_resolve_model_pad_token_id(model)).eval() image_features: torch.Tensor | None = None decoder_inputs: tuple[torch.Tensor, ...] | None = None + prefill_decoder_inputs: tuple[torch.Tensor, ...] | None = None with torch.no_grad(): if "vision_encoder" in expanded_components and ("lm_encoder" in expanded_components or "decoder" in expanded_components): image_features = vision_encoder(pixel_values) @@ -4054,6 +4205,16 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values) decoder_inputs = lm_encoder(input_ids, attention_mask, position_ids, image_features) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_QWEN_PREFILL_CHUNK", "128") or "128")) + prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) + chunk_input_ids = input_ids[:, :prefill_chunk_size].contiguous() + chunk_position_ids = torch.arange( + prefill_chunk_size, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + if "decoder_prefill_chunk" in expanded_components: + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) common_graph_meta = { "weights_dir": weights_dir, @@ -4107,6 +4268,38 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, )) + if "lm_encoder_text_chunk" in expanded_components: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "position_ids"), + graph_meta={**common_graph_meta, "component": "lm_encoder_text_chunk"}, + metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, + )) + if "decoder_prefill_chunk" in expanded_components: + if prefill_decoder_inputs is None: + raise RuntimeError("Qwen3.5 decoder_prefill_chunk spec requires precomputed text chunk decoder inputs") + max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + specs.append(ComponentModuleSpec( + component="decoder_prefill_chunk", + module=decoder_prefill_chunk, + example_inputs=prefill_decoder_inputs, + input_keys=("inputs_embeds", "position_ids"), + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_prefill_chunk", + "use_internal_kv_cache": True, + "use_internal_conv_cache": True, + "use_internal_gated_deltanet_cache": True, + "max_cache_seq_len": max_cache_seq_len, + "cache_sink_size": 4, + "prefill_chunk_size": prefill_chunk_size, + }, + metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, + )) if "decoder_media_step" in expanded_components: if decoder_inputs is None: raise RuntimeError("Qwen3.5 decoder_media_step spec requires precomputed decoder inputs") @@ -4388,6 +4581,8 @@ def _family_key(model: torch.nn.Module) -> str: return "gemma" if module_name.startswith("transformers.models.qwen3_5."): return "qwen3_5" + if module_name.startswith("transformers.models.qwen3_vl."): + return "qwen3_5" if module_name.startswith("transformers.models.qwen3."): return "qwen3" if module_name.startswith("transformers.models.lfm2_vl."): @@ -4567,7 +4762,7 @@ def _build_lfm2_vl_multimodal_component_specs( dtype=attention_mask.dtype, device=attention_mask.device, ) - prefill_chunk = max(2, int(os.environ.get("CACTUS_LFM2_PREFILL_CHUNK", "64") or "64")) + prefill_chunk = max(2, int(os.environ.get("CACTUS_LFM2_PREFILL_CHUNK", "128") or "128")) prefill_chunk = min(prefill_chunk, int(input_ids.shape[1])) requested_components = tuple(components or ("vision_encoder", "lm_encoder", "text_lm_encoder", "decoder")) @@ -4588,28 +4783,28 @@ def _require(component: str) -> None: _require("decoder") _require("text_decoder") _require("lm_encoder_step") + _require("lm_encoder_text_chunk") _require("decoder_prefill_chunk") _require("decoder_step") - elif "lm_encoder" in requested_set: + if "lm_encoder" in requested_set: _require("vision_encoder") _require("lm_encoder") - elif "lm_encoder_step" in requested_set: + if "lm_encoder_step" in requested_set: _require("lm_encoder_step") - if "decoder_step" in requested_set: - _require("decoder_step") - elif "decoder_step" in requested_set: + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "decoder_step" in requested_set: _require("lm_encoder_step") _require("decoder_step") - elif "decoder_prefill_chunk" in requested_set: + if "decoder_prefill_chunk" in requested_set: + _require("lm_encoder_text_chunk") _require("decoder_prefill_chunk") - elif "text_lm_encoder" in requested_set: + if "text_lm_encoder" in requested_set: _require("text_lm_encoder") - if "text_decoder" in requested_set: - _require("text_decoder") - elif "text_decoder" in requested_set: + if "text_decoder" in requested_set: _require("text_lm_encoder") _require("text_decoder") - elif "vision_encoder" in requested_set: + if "vision_encoder" in requested_set: _require("vision_encoder") vision_encoder = Lfm2VlVisionEncoderAdapter( @@ -4621,6 +4816,7 @@ def _require(component: str) -> None: lm_encoder = Lfm2VlLMEncoderAdapter(model, input_ids=input_ids, weights_dir=weights_dir).eval() text_lm_encoder = Lfm2VlTextLMEncoderAdapter(model, weights_dir=weights_dir).eval() lm_encoder_step = Lfm2VlLMEncoderStepAdapter(model, weights_dir=weights_dir).eval() + lm_encoder_text_chunk = Lfm2VlLMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() decoder = Lfm2VlDecoderAdapter(model, weights_dir=weights_dir, last_token_only=False).eval() decoder_last_token = Lfm2VlDecoderAdapter(model, weights_dir=weights_dir, last_token_only=True).eval() @@ -4637,6 +4833,13 @@ def _require(component: str) -> None: if "text_lm_encoder" in expanded_components: text_decoder_inputs = text_lm_encoder(text_input_ids, text_attention_mask) + chunk_input_ids = input_ids[:, :prefill_chunk].contiguous() + chunk_position_ids = torch.arange( + prefill_chunk, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + common_graph_meta = { "weights_dir": weights_dir, "task": "multimodal_causal_lm_logits", @@ -4715,6 +4918,20 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, )) + if "lm_encoder_text_chunk" in expanded_components: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "attention_mask", "position_ids"), + graph_meta={ + **common_graph_meta, + "component": "lm_encoder_text_chunk", + "encoder_chunk_size": prefill_chunk, + }, + metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, + )) if "decoder_prefill_chunk" in expanded_components: if decoder_inputs is None: if image_features is None: diff --git a/python/cactus/transpile/model_profiles.py b/python/cactus/transpile/model_profiles.py index 7e84e4c69..6a07d9c7c 100644 --- a/python/cactus/transpile/model_profiles.py +++ b/python/cactus/transpile/model_profiles.py @@ -166,7 +166,7 @@ class ModelProfile: QWEN_PROFILE = ModelProfile( family="qwen", - model_types=("qwen", "qwen2", "qwen3", "qwen3_5", "qwen3.5"), + model_types=("qwen", "qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"), multimodal_context_tokens=512, input_combinations=((), ("image",)), model_id_aliases=( @@ -177,16 +177,25 @@ class ModelProfile: ("qwen3", "Qwen/Qwen3-1.7B"), ), model_id_markers=("qwen",), - family_aliases=("qwen2", "qwen3", "qwen3_5", "qwen3.5"), + family_aliases=("qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"), stop_tokens=("<|im_end|>",), avoid_native_loader=True, - cached_step_components=("decoder_step",), + cached_step_components=("decoder_media_step",), cached_step_skip_components=("decoder",), - fp16_kv_cache_components=("decoder_step",), + fp16_kv_cache_components=("decoder_prefill_chunk", "decoder_media_step", "decoder_step"), prompt_style="qwen_chat", multimodal_preprocessor="qwen3_5", default_task="multimodal_causal_lm_logits", - default_components=("vision_encoder", "lm_encoder", "decoder", "lm_encoder_step", "decoder_media_step", "decoder_step"), + default_components=( + "vision_encoder", + "lm_encoder", + "decoder", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + "lm_encoder_step", + "decoder_media_step", + "decoder_step", + ), needs_image=True, force_component_pipeline=True, ) diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py index 676c2eeb2..6061ddbc9 100644 --- a/python/tests/test_cli_transpile_defaults.py +++ b/python/tests/test_cli_transpile_defaults.py @@ -30,7 +30,7 @@ def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[s "--task", "multimodal_causal_lm_logits", "--max-new-tokens", - "32", + "512", "--component-pipeline", "on", "--prompt", @@ -198,14 +198,24 @@ def _fake_cmd_transpile(transpile_args): assert "--audio-file" in extra_args -def test_cmd_convert_infers_text_tasks_for_qwen_and_lfm(monkeypatch, tmp_path: Path) -> None: +def test_cmd_convert_infers_profile_defaults_for_qwen_and_lfm(monkeypatch, tmp_path: Path) -> None: parser = cli.create_parser() import cactus.convert.cli as cq_cli - for alias, model_type, arch in ( - ("qwen", "qwen3", "Qwen3ForCausalLM"), - ("lfm", "lfm2_vl", "Lfm2VlForConditionalGeneration"), + for alias, model_type, arch, expected_components in ( + ( + "qwen", + "qwen3", + "Qwen3ForCausalLM", + "vision_encoder,lm_encoder,decoder,lm_encoder_text_chunk,decoder_prefill_chunk,lm_encoder_step,decoder_media_step,decoder_step", + ), + ( + "lfm", + "lfm2_vl", + "Lfm2VlForConditionalGeneration", + "vision_encoder,lm_encoder,decoder", + ), ): output_dir = tmp_path / alias args = parser.parse_args(["convert", alias, str(output_dir)]) @@ -230,7 +240,10 @@ def _fake_cmd_transpile(transpile_args): assert rc == 0 extra_args = transpile_calls[0].extra_args - assert extra_args[extra_args.index("--task") + 1] == "causal_lm_logits" + assert extra_args[extra_args.index("--task") + 1] == "multimodal_causal_lm_logits" + assert extra_args[extra_args.index("--component-pipeline") + 1] == "on" + assert extra_args[extra_args.index("--components") + 1] == expected_components + assert "--image-file" in extra_args assert "--audio-file" not in extra_args From 4629dade1eb380c938ad5847abaa1a412ce2feb1 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 20 May 2026 17:52:52 -0700 Subject: [PATCH 2/8] Fix Gemma4 multimodal chunked prefill Add dynamic Gemma4 media merge for prompts with audio so runtime prefill uses the actual prompt token layout instead of a static captured lm_encoder merge plan. This keeps image-only and non-audio paths on the existing chunked path while routing Gemma4 audio/image token rows through lm_encoder_media_step before decoder_prefill_chunk. Fix Gemma4 audio conversion policy by keeping audio bias tensors in FP16, and preserve the vision/runtime fixes needed for native Gemma4 image quality. Add local integration probes and investigation docs covering Gemma image/audio/mixed, Qwen image, Parakeet file/PCM, and smoke validation. Also ignore generated local integration and investigation outputs while preserving benchmark source/results and reproducible test harnesses. Signed-off-by: Noah Cylich --- .gitignore | 8 + cactus-engine/src/complete.cpp | 28 +- cactus-engine/src/engine.h | 15 + cactus-engine/src/engine_image.cpp | 215 +++- cactus-engine/src/model.cpp | 315 ++++- cactus-graph/src/ops_math.cpp | 194 ++- cactus-graph/src/ops_tensor.cpp | 7 + local_integration_tests/CMakeLists.txt | 62 + local_integration_tests/README.md | 43 + local_integration_tests/conftest.py | 156 +++ local_integration_tests/native_runner.cpp | 1108 +++++++++++++++++ local_integration_tests/test_modalities.py | 210 ++++ .../gemma4_multimodal/BACKGROUND.md | 53 + .../gemma4_multimodal/EXPERIMENTS.md | 80 ++ .../gemma4_multimodal/FIXES.md | 52 + .../gemma4_multimodal/HYPOTHESES.md | 71 ++ .../gemma4_multimodal/PLAN.md | 29 + .../gemma4_multimodal/README.md | 23 + .../gemma4_multimodal/SUMMARY.md | 9 + .../gemma4_multimodal_v2/00_INDEX.md | 48 + .../gemma4_multimodal_v2/01_BACKGROUND.md | 48 + .../gemma4_multimodal_v2/02_QUESTIONS.md | 41 + .../gemma4_multimodal_v2/03_HYPOTHESES.md | 66 + .../gemma4_multimodal_v2/04_EXPERIMENTS.md | 356 ++++++ .../gemma4_multimodal_v2/05_DECISIONS.md | 118 ++ .../gemma4_multimodal_v2/06_FIX_LOG.md | 156 +++ .../gemma4_multimodal_v2/07_ACCEPTANCE.md | 50 + .../gemma4_multimodal_v2/08_HANDOFF.md | 44 + .../09_STATUS_FOR_NEXT_AGENT.md | 74 ++ .../10_CONTINUATION_PLAN.md | 58 + .../gemma4_multimodal_v2/scripts/run_probe.py | 785 ++++++++++++ .../convert/cactus_adapters/tensor_io.py | 3 +- .../cactus/convert/model_adapters/adapters.py | 2 +- .../cactus/convert/model_adapters/naming.py | 1 - .../cactus/convert/model_adapters/policy.py | 4 +- python/cactus/convert/tests/test_policy.py | 8 +- python/cactus/transpile/lower.py | 4 +- python/cactus/transpile/model_adapters.py | 16 +- python/cactus/transpile/runtime_compat.py | 22 +- 39 files changed, 4493 insertions(+), 89 deletions(-) create mode 100644 local_integration_tests/CMakeLists.txt create mode 100644 local_integration_tests/README.md create mode 100644 local_integration_tests/conftest.py create mode 100644 local_integration_tests/native_runner.cpp create mode 100644 local_integration_tests/test_modalities.py create mode 100644 local_investigations/gemma4_multimodal/BACKGROUND.md create mode 100644 local_investigations/gemma4_multimodal/EXPERIMENTS.md create mode 100644 local_investigations/gemma4_multimodal/FIXES.md create mode 100644 local_investigations/gemma4_multimodal/HYPOTHESES.md create mode 100644 local_investigations/gemma4_multimodal/PLAN.md create mode 100644 local_investigations/gemma4_multimodal/README.md create mode 100644 local_investigations/gemma4_multimodal/SUMMARY.md create mode 100644 local_investigations/gemma4_multimodal_v2/00_INDEX.md create mode 100644 local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md create mode 100644 local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md create mode 100644 local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md create mode 100644 local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md create mode 100644 local_investigations/gemma4_multimodal_v2/05_DECISIONS.md create mode 100644 local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md create mode 100644 local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md create mode 100644 local_investigations/gemma4_multimodal_v2/08_HANDOFF.md create mode 100644 local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md create mode 100644 local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md create mode 100644 local_investigations/gemma4_multimodal_v2/scripts/run_probe.py diff --git a/.gitignore b/.gitignore index 99271876e..7795b9ebe 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,8 @@ bindings/flutter/*.xcframework .DS_Store __pycache__ +__pycache__/ +*.pyc *.egg-info venv @@ -45,3 +47,9 @@ vision_activations.json gemma-4-eap-extras/ .githooks/ ruy + +# Local integration/investigation outputs +local_integration_tests/.tmp/ +local_integration_tests/__pycache__/ +local_investigations/**/__pycache__/ +local_investigations/**/artifacts/ diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index d355eb586..641b8c1b9 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -676,14 +676,6 @@ int cactus_complete( bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); - const bool has_gemma4_mixed_media = prompt.model_type == Config::ModelType::GEMMA4 && has_images && has_audio; - auto decode_gemma4_mixed_media = [&](const std::vector& tokens, float* out_entropy) -> uint32_t { - (void)tokens; - (void)out_entropy; - throw std::runtime_error( - "Gemma4 mixed-media native decode is not present in this build. " - "Use cactus run-transpiled for transpiled graph bundles."); - }; auto stop_token_sequences = build_stop_sequences(tokenizer, prompt.options.stop_sequences, prompt.model_type, !prompt.tools.empty()); @@ -693,7 +685,7 @@ int cactus_complete( uint32_t next_token; size_t prompt_tokens; - if ((has_gemma4_mixed_media || has_audio) && !handle->processed_tokens.empty()) { + if (has_audio && !handle->processed_tokens.empty()) { auto& cache = handle->processed_tokens; size_t common = 0; size_t limit = std::min(cache.size(), prompt.tokens.size()); @@ -709,14 +701,9 @@ int cactus_complete( } } - if (has_gemma4_mixed_media) { - prompt_tokens = prompt.tokens.size(); - next_token = decode_gemma4_mixed_media(prompt.tokens, &first_token_entropy); - } else { - auto prefill_result = do_prefill(handle, prompt, prompt.tokens); - prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); - next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); - } + auto prefill_result = do_prefill(handle, prompt, prompt.tokens); + prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); + next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); handle->processed_tokens = prompt.tokens; handle->processed_images = prompt.images; @@ -784,11 +771,10 @@ int cactus_complete( if (handle->should_stop) break; float token_entropy = 0.0f; - if (has_gemma4_mixed_media) { - next_token = decode_gemma4_mixed_media(handle->processed_tokens, &token_entropy); - } else if (has_audio) { + if (has_audio) { + uint32_t last_token = handle->processed_tokens.empty() ? next_token : handle->processed_tokens.back(); next_token = handle->model->decode_with_audio( - handle->processed_tokens, prompt.audio_features, + {last_token}, prompt.audio_features, prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, "", &token_entropy, prompt.options.min_p, prompt.options.repetition_penalty); diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 2656a0898..3b3478478 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -700,6 +700,11 @@ class Model { bool run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, const std::vector& audio_features); + bool build_lm_encoder_outputs_dynamic_gemma4( + const std::vector& tokens, + std::map>& store_bytes, + std::map& store_prec, + std::map>& store_shape); std::string bundle_dir_; std::map components_; @@ -875,6 +880,16 @@ struct Lfm2VlImagePreprocessed { Lfm2VlImagePreprocessed preprocess_lfm2_vl_image(const std::string& image_path, const Config& config); +struct Qwen3VlImagePreprocessed { + std::vector pixel_values; + size_t grid_t = 1; + size_t grid_h = 0; + size_t grid_w = 0; + size_t patch_dim = 0; +}; + +Qwen3VlImagePreprocessed preprocess_qwen3_vl_image(const std::string& image_path, const Config& config); + struct SpectrogramConfig { size_t n_fft = 400; diff --git a/cactus-engine/src/engine_image.cpp b/cactus-engine/src/engine_image.cpp index bd9ee452d..f73c2429a 100644 --- a/cactus-engine/src/engine_image.cpp +++ b/cactus-engine/src/engine_image.cpp @@ -19,6 +19,115 @@ namespace cactus { namespace engine { +namespace { + +constexpr int kPillowResizePrecisionBits = 22; + +static unsigned char pillow_clip8(int64_t value) { + int shifted = static_cast(value >> kPillowResizePrecisionBits); + if (shifted < 0) return 0; + if (shifted > 255) return 255; + return static_cast(shifted); +} + +static void precompute_pillow_bilinear_coeffs(int in_size, int out_size, + std::vector& bounds, + std::vector& coeffs, + int& ksize) { + const double scale = static_cast(in_size) / static_cast(out_size); + const double filterscale = std::max(scale, 1.0); + const double support = filterscale; + const double inv_filterscale = 1.0 / filterscale; + ksize = static_cast(std::ceil(support)) * 2 + 1; + bounds.assign(static_cast(out_size) * 2, 0); + coeffs.assign(static_cast(out_size) * ksize, 0); + + for (int out = 0; out < out_size; ++out) { + const double center = (static_cast(out) + 0.5) * scale; + int xmin = static_cast(center - support + 0.5); + if (xmin < 0) xmin = 0; + int xmax = static_cast(center + support + 0.5); + if (xmax > in_size) xmax = in_size; + xmax -= xmin; + + bounds[static_cast(out) * 2] = xmin; + bounds[static_cast(out) * 2 + 1] = xmax; + + std::vector weights(static_cast(ksize), 0.0); + double total = 0.0; + for (int i = 0; i < xmax; ++i) { + double x = (static_cast(i + xmin) - center + 0.5) * inv_filterscale; + if (x < 0.0) x = -x; + double weight = x < 1.0 ? 1.0 - x : 0.0; + weights[static_cast(i)] = weight; + total += weight; + } + for (int i = 0; i < ksize; ++i) { + double weight = total != 0.0 ? weights[static_cast(i)] / total : 0.0; + coeffs[static_cast(out) * ksize + i] = static_cast( + weight < 0.0 + ? -0.5 + weight * static_cast(1 << kPillowResizePrecisionBits) + : 0.5 + weight * static_cast(1 << kPillowResizePrecisionBits)); + } + } +} + +static std::vector resize_rgb_uint8_pillow_bilinear(const unsigned char* src, + int src_w, + int src_h, + int dst_w, + int dst_h) { + std::vector x_bounds; + std::vector y_bounds; + std::vector x_coeffs; + std::vector y_coeffs; + int x_ksize = 0; + int y_ksize = 0; + precompute_pillow_bilinear_coeffs(src_w, dst_w, x_bounds, x_coeffs, x_ksize); + precompute_pillow_bilinear_coeffs(src_h, dst_h, y_bounds, y_coeffs, y_ksize); + + const int y_first = y_bounds[0]; + const int y_last = y_bounds[static_cast(dst_h - 1) * 2] + + y_bounds[static_cast(dst_h - 1) * 2 + 1]; + const int temp_h = y_last - y_first; + std::vector temp(static_cast(dst_w) * temp_h * 3); + + for (int ty = 0; ty < temp_h; ++ty) { + const int sy = y_first + ty; + for (int dx = 0; dx < dst_w; ++dx) { + const int xmin = x_bounds[static_cast(dx) * 2]; + const int count = x_bounds[static_cast(dx) * 2 + 1]; + const int32_t* k = x_coeffs.data() + static_cast(dx) * x_ksize; + for (int c = 0; c < 3; ++c) { + int64_t sum = static_cast(1) << (kPillowResizePrecisionBits - 1); + for (int x = 0; x < count; ++x) { + sum += static_cast(src[(static_cast(sy) * src_w + xmin + x) * 3 + c]) * k[x]; + } + temp[(static_cast(ty) * dst_w + dx) * 3 + c] = pillow_clip8(sum); + } + } + } + + std::vector dst(static_cast(dst_w) * dst_h * 3); + for (int dy = 0; dy < dst_h; ++dy) { + const int ymin = y_bounds[static_cast(dy) * 2] - y_first; + const int count = y_bounds[static_cast(dy) * 2 + 1]; + const int32_t* k = y_coeffs.data() + static_cast(dy) * y_ksize; + for (int dx = 0; dx < dst_w; ++dx) { + for (int c = 0; c < 3; ++c) { + int64_t sum = static_cast(1) << (kPillowResizePrecisionBits - 1); + for (int y = 0; y < count; ++y) { + sum += static_cast(temp[(static_cast(ymin + y) * dst_w + dx) * 3 + c]) * k[y]; + } + dst[(static_cast(dy) * dst_w + dx) * 3 + c] = pillow_clip8(sum); + } + } + } + return dst; +} + +} // namespace + Siglip2Preprocessor::PreprocessedImage::~PreprocessedImage() { pixel_values.clear(); pixel_attention_mask.clear(); @@ -634,16 +743,9 @@ Gemma4ImagePreprocessed preprocess_gemma4_image(const std::string& image_path, c resized[i] = static_cast(raw[i]); } } else { - std::vector src_float(static_cast(width) * height * 3); - for (size_t i = 0; i < src_float.size(); ++i) { - src_float[i] = static_cast(raw[i]); - } - if (!stbir_resize_float_linear( - src_float.data(), width, height, 0, - resized.data(), target_w, target_h, 0, - STBIR_RGB)) { - stbi_image_free(raw); - throw std::runtime_error("Failed to resize image: " + image_path); + std::vector resized_u8 = resize_rgb_uint8_pillow_bilinear(raw, width, height, target_w, target_h); + for (size_t i = 0; i < resized.size(); ++i) { + resized[i] = static_cast(resized_u8[i]); } } stbi_image_free(raw); @@ -700,6 +802,99 @@ Gemma4ImagePreprocessed preprocess_gemma4_image(const std::string& image_path, c return result; } +Qwen3VlImagePreprocessed preprocess_qwen3_vl_image(const std::string& image_path, const Config& config) { + Qwen3VlImagePreprocessed result; + const int patch_size = static_cast(config.vision_patch_size ? config.vision_patch_size : 16); + const int temporal_patch_size = 2; + const int merge_size = 2; + const size_t image_tokens = config.image_seq_len ? static_cast(config.image_seq_len) : 64; + const size_t merged_patches = image_tokens * static_cast(merge_size * merge_size); + const int grid_side = static_cast(std::sqrt(static_cast(merged_patches))); + if (patch_size <= 0 || grid_side <= 0 || static_cast(grid_side * grid_side) != merged_patches) { + throw std::runtime_error("Qwen3-VL native image preprocessing requires a square static image grid"); + } + + int width = 0, height = 0, channels = 0; + unsigned char* raw = stbi_load(image_path.c_str(), &width, &height, &channels, 3); + if (!raw) { + throw std::runtime_error("Failed to load image: " + image_path + " - " + std::string(stbi_failure_reason())); + } + if (width <= 0 || height <= 0) { + stbi_image_free(raw); + throw std::runtime_error("Loaded image has invalid dimensions: " + image_path); + } + + const int target_h = grid_side * patch_size; + const int target_w = grid_side * patch_size; + std::vector resized(static_cast(target_w) * target_h * 3); + if (target_w == width && target_h == height) { + for (size_t i = 0; i < resized.size(); ++i) { + resized[i] = static_cast(raw[i]); + } + } else { + std::vector src_float(static_cast(width) * height * 3); + for (size_t i = 0; i < src_float.size(); ++i) { + src_float[i] = static_cast(raw[i]); + } + if (!stbir_resize_float_linear( + src_float.data(), width, height, 0, + resized.data(), target_w, target_h, 0, + STBIR_RGB)) { + stbi_image_free(raw); + throw std::runtime_error("Failed to resize image: " + image_path); + } + } + stbi_image_free(raw); + + const float rescale_factor = config.rescale_factor > 0.0f ? config.rescale_factor : (1.0f / 255.0f); + const float mean_v = config.image_mean; + const float std_v = config.image_std != 0.0f ? config.image_std : 1.0f; + for (float& v : resized) { + v = (v * rescale_factor - mean_v) / std_v; + } + + const size_t channel = 3; + const size_t grid_t = 1; + const size_t grid_h = static_cast(grid_side); + const size_t grid_w = static_cast(grid_side); + const size_t patch_dim = channel * static_cast(temporal_patch_size) * patch_size * patch_size; + result.pixel_values.assign(grid_t * grid_h * grid_w * patch_dim, 0.0f); + + for (size_t gh_major = 0; gh_major < grid_h / merge_size; ++gh_major) { + for (size_t gw_major = 0; gw_major < grid_w / merge_size; ++gw_major) { + for (size_t mh = 0; mh < static_cast(merge_size); ++mh) { + for (size_t mw = 0; mw < static_cast(merge_size); ++mw) { + const size_t patch_y = gh_major * merge_size + mh; + const size_t patch_x = gw_major * merge_size + mw; + const size_t patch_index = + (((gh_major * (grid_w / merge_size) + gw_major) * merge_size + mh) * merge_size + mw); + float* dst = result.pixel_values.data() + patch_index * patch_dim; + size_t dst_i = 0; + for (size_t c = 0; c < channel; ++c) { + for (size_t t = 0; t < static_cast(temporal_patch_size); ++t) { + (void)t; + for (int y = 0; y < patch_size; ++y) { + const size_t img_y = patch_y * patch_size + static_cast(y); + for (int x = 0; x < patch_size; ++x) { + const size_t img_x = patch_x * patch_size + static_cast(x); + const size_t src_off = (img_y * static_cast(target_w) + img_x) * channel + c; + dst[dst_i++] = resized[src_off]; + } + } + } + } + } + } + } + } + + result.grid_t = grid_t; + result.grid_h = grid_h; + result.grid_w = grid_w; + result.patch_dim = patch_dim; + return result; +} + Lfm2VlImagePreprocessed preprocess_lfm2_vl_image(const std::string& image_path, const Config& config) { Siglip2Preprocessor::Config sp_cfg; sp_cfg.patch_size = config.vision_patch_size ? static_cast(config.vision_patch_size) : 16; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 64a5854f1..5c20a02f7 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -932,6 +932,7 @@ void Model::run_audio_encoder(const std::vector& audio_features) { if (!audio_encoder_) return; std::vector candidate_input_names = {"input_features", "audio_features"}; bool wrote = false; + int feature_idx = -1; for (const auto& name : candidate_input_names) { int idx = input_index(*audio_encoder_, name); if (idx < 0) continue; @@ -956,6 +957,7 @@ void Model::run_audio_encoder(const std::vector& audio_features) { if (n_elems < buf.size()) std::memset(buf.data() + n_elems, 0, buf.size() - n_elems); } wrote = true; + feature_idx = idx; break; } if (!wrote) { @@ -965,7 +967,24 @@ void Model::run_audio_encoder(const std::vector& audio_features) { int mask_idx = input_index(*audio_encoder_, "input_features_mask"); if (mask_idx >= 0) { auto& mb = audio_encoder_->input_buffers[mask_idx]; - std::fill(mb.begin(), mb.end(), static_cast(1)); + size_t feature_node = static_cast(audio_encoder_->runtime_input_node_ids[feature_idx]); + const auto& feature_desc = audio_encoder_->graph->get_output_buffer(feature_node); + size_t expected_mels = feature_desc.shape.size() >= 3 ? feature_desc.shape[2] : config_.audio_input_feat_size; + size_t valid_frames = expected_mels > 0 ? audio_features.size() / expected_mels : 0; + size_t mask_node = static_cast(audio_encoder_->runtime_input_node_ids[mask_idx]); + const auto& mask_desc = audio_encoder_->graph->get_output_buffer(mask_node); + const size_t elem = PrecisionTraits::size_of(mask_desc.precision); + const size_t cap = elem ? mb.size() / elem : 0; + const size_t n = std::min(cap, valid_frames); + for (size_t i = 0; i < n; ++i) { + switch (mask_desc.precision) { + case Precision::FP32: reinterpret_cast(mb.data())[i] = 1.0f; break; + case Precision::FP16: reinterpret_cast<__fp16*>(mb.data())[i] = static_cast<__fp16>(1.0f); break; + case Precision::INT8: reinterpret_cast(mb.data())[i] = 1; break; + default: reinterpret_cast(mb.data())[i] = 1; break; + } + } + if (n < cap) std::memset(mb.data() + n * elem, 0, (cap - n) * elem); } audio_encoder_->graph->execute(); for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { @@ -1102,13 +1121,231 @@ void write_tokens_buffer(std::vector& buf, Precision prec, } } +void write_int_vector_buffer(std::vector& buf, Precision prec, const std::vector& values) { + const size_t elem = PrecisionTraits::size_of(prec); + const size_t cap = elem ? buf.size() / elem : 0; + const size_t n = std::min(cap, values.size()); + for (size_t i = 0; i < n; ++i) { + int64_t v = values[i]; + switch (prec) { + case Precision::FP32: reinterpret_cast(buf.data())[i] = static_cast(v); break; + case Precision::FP16: reinterpret_cast<__fp16*>(buf.data())[i] = static_cast<__fp16>(v); break; + case Precision::INT8: reinterpret_cast(buf.data())[i] = static_cast(v); break; + default: + if (elem == 8) reinterpret_cast(buf.data())[i] = v; + else if (elem == 4) reinterpret_cast(buf.data())[i] = static_cast(v); + break; + } + } + if (n < cap) { + std::memset(buf.data() + n * elem, 0, (cap - n) * elem); + } +} + +std::vector qwen3_vl_position_ids(const std::vector& tokens, + size_t capacity, + const std::vector& images, + uint32_t image_token_id) { + std::vector positions(3 * capacity, 0); + size_t token_index = 0; + size_t image_index = 0; + int64_t current_pos = 0; + while (token_index < tokens.size() && token_index < capacity) { + if (image_token_id != 0 && tokens[token_index] == image_token_id) { + if (image_index >= images.size()) { + throw std::runtime_error("Qwen3-VL prompt contains more image token groups than image inputs"); + } + const auto& image = images[image_index++]; + const size_t merge_size = 2; + const size_t grid_t = image.grid_t; + const size_t llm_grid_h = image.grid_h / merge_size; + const size_t llm_grid_w = image.grid_w / merge_size; + const size_t image_seq = grid_t * llm_grid_h * llm_grid_w; + size_t count = 0; + while (token_index + count < tokens.size() + && token_index + count < capacity + && tokens[token_index + count] == image_token_id) { + ++count; + } + if (count != image_seq) { + throw std::runtime_error("Qwen3-VL image token count does not match vision feature grid"); + } + size_t local = 0; + for (size_t t = 0; t < grid_t; ++t) { + (void)t; + for (size_t h = 0; h < llm_grid_h; ++h) { + for (size_t w = 0; w < llm_grid_w; ++w) { + size_t pos = token_index + local++; + positions[pos] = current_pos; + positions[capacity + pos] = current_pos + static_cast(h); + positions[2 * capacity + pos] = current_pos + static_cast(w); + } + } + } + current_pos += static_cast(std::max(image.grid_h, image.grid_w) / merge_size); + token_index += count; + continue; + } + + size_t text_count = 0; + while (token_index + text_count < tokens.size() + && token_index + text_count < capacity + && (image_token_id == 0 || tokens[token_index + text_count] != image_token_id)) { + size_t pos = token_index + text_count; + int64_t value = current_pos + static_cast(text_count); + positions[pos] = value; + positions[capacity + pos] = value; + positions[2 * capacity + pos] = value; + ++text_count; + } + current_pos += static_cast(text_count); + token_index += text_count; + } + return positions; +} + } // namespace +bool Model::build_lm_encoder_outputs_dynamic_gemma4( + const std::vector& tokens, + std::map>& store_bytes, + std::map& store_prec, + std::map>& store_shape) { + if (!encoder_ || !lm_encoder_media_step_ || tokens.empty()) return false; + + const uint32_t image_tok = config_.image_token_id; + const uint32_t audio_tok = config_.audio_token_id; + + auto audio_it = media_features_.find("audio_features"); + const bool have_audio_features = audio_it != media_features_.end() && !audio_it->second.empty(); + size_t audio_rows = 0; + size_t audio_row_bytes = 0; + Precision audio_prec = Precision::FP16; + if (have_audio_features) { + const auto& shape = media_feature_shapes_["audio_features"]; + audio_rows = shape.size() >= 2 ? shape[shape.size() - 2] : 0; + audio_prec = media_feature_precisions_["audio_features"]; + audio_row_bytes = audio_rows > 0 ? audio_it->second.size() / audio_rows : audio_it->second.size(); + } + + auto image_it = media_features_.find("image_features"); + const bool have_image_features = image_it != media_features_.end() && !image_it->second.empty(); + size_t image_rows = 0; + size_t image_row_bytes = 0; + Precision image_prec = Precision::FP16; + if (have_image_features) { + const auto& shape = media_feature_shapes_["image_features"]; + image_rows = shape.size() >= 2 ? shape[shape.size() - 2] : 0; + image_prec = media_feature_precisions_["image_features"]; + image_row_bytes = image_rows > 0 ? image_it->second.size() / image_rows : image_it->second.size(); + } + + struct OutputInfo { + std::string name; + int text_idx = -1; + int media_idx = -1; + size_t per_token_bytes = 0; + Precision precision = Precision::FP16; + std::vector shape_template; + }; + + std::vector outputs; + for (size_t i = 0; i < encoder_->logical_outputs.size() && i < encoder_->output_node_ids.size(); ++i) { + OutputInfo info; + info.name = encoder_->logical_outputs[i]; + info.text_idx = static_cast(i); + info.media_idx = output_index(*lm_encoder_media_step_, info.name); + if (info.media_idx < 0) { + throw std::runtime_error("lm_encoder_media_step missing output " + info.name); + } + size_t node_id = static_cast(encoder_->output_node_ids[i]); + const auto& desc = encoder_->graph->get_output_buffer(node_id); + info.per_token_bytes = desc.byte_size; + info.precision = desc.precision; + info.shape_template = desc.shape; + outputs.push_back(std::move(info)); + } + if (outputs.empty()) return false; + + const size_t token_count = tokens.size(); + for (const auto& info : outputs) { + store_bytes[info.name].assign(token_count * info.per_token_bytes, 0); + store_prec[info.name] = info.precision; + std::vector shape = info.shape_template; + if (shape.size() >= 2 && shape[shape.size() - 2] == 1) { + shape[shape.size() - 2] = token_count; + } else if (shape.size() == 1) { + shape[0] = token_count; + } + store_shape[info.name] = std::move(shape); + } + + size_t audio_idx = 0; + size_t image_idx = 0; + for (size_t pos = 0; pos < token_count; ++pos) { + const uint32_t token = tokens[pos]; + Component* component = encoder_; + const uint8_t* media_row = nullptr; + size_t media_row_bytes = 0; + Precision media_prec = Precision::FP16; + + if (audio_tok != 0 && token == audio_tok && have_audio_features) { + if (audio_idx >= audio_rows) { + throw std::runtime_error("Gemma4 prompt contains more audio tokens than audio feature rows"); + } + component = lm_encoder_media_step_; + media_row = audio_it->second.data() + audio_idx * audio_row_bytes; + media_row_bytes = audio_row_bytes; + media_prec = audio_prec; + ++audio_idx; + } else if (image_tok != 0 && token == image_tok && have_image_features) { + if (image_idx >= image_rows) { + throw std::runtime_error("Gemma4 prompt contains more image tokens than image feature rows"); + } + component = lm_encoder_media_step_; + media_row = image_it->second.data() + image_idx * image_row_bytes; + media_row_bytes = image_row_bytes; + media_prec = image_prec; + ++image_idx; + } + + if (component == lm_encoder_media_step_) { + int embeds_idx = input_index(*component, "inputs_embeds"); + if (embeds_idx < 0) { + throw std::runtime_error("lm_encoder_media_step missing inputs_embeds input"); + } + auto& buf = component->input_buffers[embeds_idx]; + size_t node_id = static_cast(component->runtime_input_node_ids[embeds_idx]); + const auto& desc = component->graph->get_output_buffer(node_id); + write_typed_buffer(buf, desc.precision, media_row, media_row_bytes, media_prec); + write_int_input(*component, "input_ids", 0); + write_int_input(*component, "position_ids", static_cast(pos)); + } else { + write_int_input(*component, "input_ids", static_cast(token)); + write_int_input(*component, "position_ids", static_cast(pos)); + } + component->graph->execute(); + + for (const auto& info : outputs) { + int out_idx = component == encoder_ ? info.text_idx : info.media_idx; + size_t node_id = static_cast(component->output_node_ids[out_idx]); + const auto& desc = component->graph->get_output_buffer(node_id); + if (desc.byte_size != info.per_token_bytes || desc.precision != info.precision) { + throw std::runtime_error("Gemma4 dynamic output shape mismatch for " + info.name); + } + const void* ptr = component->graph->get_output(node_id); + std::memcpy(store_bytes[info.name].data() + pos * info.per_token_bytes, ptr, info.per_token_bytes); + } + } + return true; +} + bool Model::run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, const std::vector& audio_features) { const bool have_images = !image_paths.empty() && vision_encoder_ != nullptr; const bool have_audio = !audio_features.empty() && audio_encoder_ != nullptr; + std::vector qwen_images; if (have_images) { for (const auto& path : image_paths) { @@ -1143,6 +1380,20 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, } if (n < cap) std::memset(pm_buf.data() + n * elem, 0, (cap - n) * elem); } + } else if (family_ == "qwen3_5" || family_ == "qwen3_vl" || config_.model_type == Config::ModelType::QWEN) { + Qwen3VlImagePreprocessed prep = preprocess_qwen3_vl_image(path, config_); + int pv_idx = input_index(*vision_encoder_, "pixel_values"); + if (pv_idx < 0) { + throw std::runtime_error("Qwen3-VL vision_encoder missing pixel_values input"); + } + auto& pv_buf = vision_encoder_->input_buffers[pv_idx]; + size_t pv_node = static_cast(vision_encoder_->runtime_input_node_ids[pv_idx]); + const auto& pv_desc = vision_encoder_->graph->get_output_buffer(pv_node); + write_typed_buffer(pv_buf, pv_desc.precision, + prep.pixel_values.data(), + prep.pixel_values.size() * sizeof(float), + Precision::FP32); + qwen_images.push_back(std::move(prep)); } else { Gemma4ImagePreprocessed prep = preprocess_gemma4_image(path, config_); int pv_idx = input_index(*vision_encoder_, "pixel_values"); @@ -1198,7 +1449,16 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, run_audio_encoder(audio_features); } - { + std::map> store_bytes; + std::map store_prec; + std::map> store_shape; + const bool needs_dynamic_walk = family_ == "gemma4" && have_audio && lm_encoder_media_step_ != nullptr; + + if (needs_dynamic_walk) { + if (!build_lm_encoder_outputs_dynamic_gemma4(tokens, store_bytes, store_prec, store_shape)) { + return false; + } + } else { int ids_idx = input_index(*lm_encoder_, "input_ids"); if (ids_idx >= 0) { auto& ids_buf = lm_encoder_->input_buffers[ids_idx]; @@ -1215,6 +1475,18 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, fill_int_buffer(mb, mdesc.precision, 1, tokens.size()); } + int pos_idx = input_index(*lm_encoder_, "position_ids"); + if (pos_idx >= 0) { + auto& pos_buf = lm_encoder_->input_buffers[pos_idx]; + size_t pos_node = static_cast(lm_encoder_->runtime_input_node_ids[pos_idx]); + const auto& pos_desc = lm_encoder_->graph->get_output_buffer(pos_node); + if (pos_desc.shape.size() >= 3 && pos_desc.shape[0] == 3 && !qwen_images.empty()) { + size_t capacity = pos_desc.shape[pos_desc.shape.size() - 1]; + auto positions = qwen3_vl_position_ids(tokens, capacity, qwen_images, config_.image_token_id); + write_int_vector_buffer(pos_buf, pos_desc.precision, positions); + } + } + for (const auto& kv : media_features_) { const std::string& name = kv.first; int idx = input_index(*lm_encoder_, name); @@ -1226,23 +1498,20 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, write_typed_buffer(dst_buf, desc.precision, kv.second.data(), kv.second.size(), src_prec); } - } - lm_encoder_->graph->execute(); + lm_encoder_->graph->execute(); - std::map> store_bytes; - std::map store_prec; - std::map> store_shape; - for (size_t i = 0; i < lm_encoder_->output_node_ids.size() - && i < lm_encoder_->logical_outputs.size(); ++i) { - const std::string& name = lm_encoder_->logical_outputs[i]; - size_t node_id = static_cast(lm_encoder_->output_node_ids[i]); - const auto& desc = lm_encoder_->graph->get_output_buffer(node_id); - void* ptr = lm_encoder_->graph->get_output(node_id); - auto& slot = store_bytes[name]; - slot.assign(desc.byte_size, 0); - std::memcpy(slot.data(), ptr, desc.byte_size); - store_prec[name] = desc.precision; - store_shape[name] = desc.shape; + for (size_t i = 0; i < lm_encoder_->output_node_ids.size() + && i < lm_encoder_->logical_outputs.size(); ++i) { + const std::string& name = lm_encoder_->logical_outputs[i]; + size_t node_id = static_cast(lm_encoder_->output_node_ids[i]); + const auto& desc = lm_encoder_->graph->get_output_buffer(node_id); + void* ptr = lm_encoder_->graph->get_output(node_id); + auto& slot = store_bytes[name]; + slot.assign(desc.byte_size, 0); + std::memcpy(slot.data(), ptr, desc.byte_size); + store_prec[name] = desc.precision; + store_shape[name] = desc.shape; + } } auto embeds_shape_it = store_shape.find("inputs_embeds"); @@ -1346,8 +1615,14 @@ void Model::prefill_with_media(const std::vector& tokens, const std::vector& audio_features, const std::string& profile_file) { if (tokens.empty()) return; - const bool have_images = !image_paths.empty() && vision_encoder_ != nullptr; - const bool have_audio = !audio_features.empty() && audio_encoder_ != nullptr; + if (!image_paths.empty() && vision_encoder_ == nullptr) { + throw std::runtime_error("Model bundle does not include a vision_encoder for image input"); + } + if (!audio_features.empty() && audio_encoder_ == nullptr) { + throw std::runtime_error("Model bundle does not include an audio_encoder for audio input"); + } + const bool have_images = !image_paths.empty(); + const bool have_audio = !audio_features.empty(); if (!have_images && !have_audio) { prefill(tokens, get_prefill_chunk_size(), profile_file); return; diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index 909328e5e..e87fc110b 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -104,6 +104,33 @@ void dispatch_binary_op_f16(OpType op, const __fp16* lhs, const __fp16* rhs, __f } } +static float apply_binary_op_f32(OpType op, float lhs, float rhs) { + switch (op) { + case OpType::ADD: + case OpType::ADD_CLIPPED: + return lhs + rhs; + case OpType::SUBTRACT: + return lhs - rhs; + case OpType::MULTIPLY: + return lhs * rhs; + case OpType::DIVIDE: + return lhs / rhs; + case OpType::NOT_EQUAL: + return lhs != rhs ? 1.0f : 0.0f; + default: + return lhs; + } +} + +void dispatch_binary_op_f32(OpType op, const float* lhs, const float* rhs, float* output, size_t count) { + CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + for (size_t i = start_idx; i < end_idx; ++i) { + output[i] = apply_binary_op_f32(op, lhs[i], rhs[i]); + } + }); +} + void dispatch_unary_op_f16(OpType op, const __fp16* input, __fp16* output, size_t count, float param) { ScalarOpType scalar_op; switch (op) { @@ -154,13 +181,42 @@ void compute_binary_op_node(GraphNode& node, const std::vector(lhs.precision)) + ")"); + if (lhs.precision != rhs.precision) { + throw std::runtime_error("Binary operations require matching precision"); + } + if (lhs.precision != Precision::FP16 && lhs.precision != Precision::FP32) { + throw std::runtime_error("Binary operations only support FP16/FP32 precision"); } if (node.params.broadcast_info.needs_broadcasting) { std::vector lhs_strides = compute_strides(lhs.shape, node.params.broadcast_info.output_shape); std::vector rhs_strides = compute_strides(rhs.shape, node.params.broadcast_info.output_shape); + const auto& output_shape = node.params.broadcast_info.output_shape; + size_t total_elements = 1; + for (size_t dim : output_shape) { + total_elements *= dim; + } + if (lhs.precision == Precision::FP32) { + const float* lhs_data = lhs.data_as(); + const float* rhs_data = rhs.data_as(); + float* out_data = node.output_buffer.data_as(); + CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + std::vector coords(output_shape.size()); + size_t tmp = start_idx; + for (int axis = static_cast(output_shape.size()) - 1; axis >= 0; --axis) { + coords[static_cast(axis)] = tmp % output_shape[static_cast(axis)]; + tmp /= output_shape[static_cast(axis)]; + } + for (size_t linear_idx = start_idx; linear_idx < end_idx; ++linear_idx) { + size_t lhs_idx = broadcast_linear_index(coords, lhs_strides); + size_t rhs_idx = broadcast_linear_index(coords, rhs_strides); + out_data[linear_idx] = apply_binary_op_f32(node.op_type, lhs_data[lhs_idx], rhs_data[rhs_idx]); + increment_broadcast_coords(coords, output_shape); + } + }); + return; + } switch (node.op_type) { case OpType::ADD: @@ -196,11 +252,6 @@ void compute_binary_op_node(GraphNode& node, const std::vector(); const __fp16* rhs_data = rhs.data_as<__fp16>(); __fp16* out_data = node.output_buffer.data_as<__fp16>(); - const auto& output_shape = node.params.broadcast_info.output_shape; - size_t total_elements = 1; - for (size_t dim : output_shape) { - total_elements *= dim; - } CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ELEMENT_WISE, [&](size_t start_idx, size_t end_idx) { std::vector coords(output_shape.size()); @@ -222,9 +273,15 @@ void compute_binary_op_node(GraphNode& node, const std::vector(), - rhs.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); + if (lhs.precision == Precision::FP16) { + dispatch_binary_op_f16(node.op_type, lhs.data_as<__fp16>(), + rhs.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + } else { + dispatch_binary_op_f32(node.op_type, lhs.data_as(), + rhs.data_as(), node.output_buffer.data_as(), + node.output_buffer.total_size); + } } } @@ -329,8 +386,8 @@ void compute_reduce_node(GraphNode& node, const std::vector(axis)); + if (input_buffer.precision == Precision::FP32) { + const float* input = input_buffer.data_as(); + float* output = node.output_buffer.data_as(); + for (size_t outer = 0; outer < dims.outer; ++outer) { + for (size_t inner = 0; inner < dims.inner; ++inner) { + float running = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + size_t flat_index = ((outer * dims.axis_size) + axis_index) * dims.inner + inner; + running += input[flat_index]; + output[flat_index] = running; + } + } + } + return; + } + const __fp16* input = input_buffer.data_as<__fp16>(); __fp16* output = node.output_buffer.data_as<__fp16>(); @@ -355,6 +428,103 @@ void compute_reduce_node(GraphNode& node, const std::vector(); + float* output = node.output_buffer.data_as(); + if (axis == -1) { + float result = 0.0f; + switch (node.op_type) { + case OpType::SUM: + case OpType::MEAN: + for (size_t i = 0; i < input_buffer.total_size; ++i) result += input[i]; + if (node.op_type == OpType::MEAN && input_buffer.total_size > 0) { + result /= static_cast(input_buffer.total_size); + } + output[0] = result; + break; + case OpType::VARIANCE: { + float mean = 0.0f; + for (size_t i = 0; i < input_buffer.total_size; ++i) mean += input[i]; + mean /= static_cast(std::max(input_buffer.total_size, 1)); + float variance = 0.0f; + for (size_t i = 0; i < input_buffer.total_size; ++i) { + float diff = input[i] - mean; + variance += diff * diff; + } + output[0] = variance / static_cast(std::max(input_buffer.total_size, 1)); + break; + } + case OpType::MIN: + result = input_buffer.total_size == 0 ? 0.0f : input[0]; + for (size_t i = 1; i < input_buffer.total_size; ++i) result = std::min(result, input[i]); + output[0] = result; + break; + case OpType::MAX: + result = input_buffer.total_size == 0 ? 0.0f : input[0]; + for (size_t i = 1; i < input_buffer.total_size; ++i) result = std::max(result, input[i]); + output[0] = result; + break; + default: + break; + } + return; + } + + auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); + CactusThreading::parallel_for(dims.outer * dims.inner, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + for (size_t out_idx = start_idx; out_idx < end_idx; ++out_idx) { + size_t outer = out_idx / dims.inner; + size_t inner = out_idx % dims.inner; + size_t base = outer * dims.axis_size * dims.inner + inner; + float result = 0.0f; + switch (node.op_type) { + case OpType::SUM: + case OpType::MEAN: + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + result += input[base + axis_index * dims.inner]; + } + if (node.op_type == OpType::MEAN && dims.axis_size > 0) { + result /= static_cast(dims.axis_size); + } + output[out_idx] = result; + break; + case OpType::VARIANCE: { + float mean = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + mean += input[base + axis_index * dims.inner]; + } + mean /= static_cast(std::max(dims.axis_size, 1)); + float variance = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + float diff = input[base + axis_index * dims.inner] - mean; + variance += diff * diff; + } + output[out_idx] = variance / static_cast(std::max(dims.axis_size, 1)); + break; + } + case OpType::MIN: + result = dims.axis_size == 0 ? 0.0f : input[base]; + for (size_t axis_index = 1; axis_index < dims.axis_size; ++axis_index) { + result = std::min(result, input[base + axis_index * dims.inner]); + } + output[out_idx] = result; + break; + case OpType::MAX: + result = dims.axis_size == 0 ? 0.0f : input[base]; + for (size_t axis_index = 1; axis_index < dims.axis_size; ++axis_index) { + result = std::max(result, input[base + axis_index * dims.inner]); + } + output[out_idx] = result; + break; + default: + break; + } + } + }); + return; + } + if (axis == -1) { switch (node.op_type) { case OpType::SUM: { diff --git a/cactus-graph/src/ops_tensor.cpp b/cactus-graph/src/ops_tensor.cpp index a0dae609f..42f967ce5 100644 --- a/cactus-graph/src/ops_tensor.cpp +++ b/cactus-graph/src/ops_tensor.cpp @@ -136,6 +136,13 @@ void compute_embedding_node(GraphNode& node, const std::vector(); + } else if (indices_buffer.precision == Precision::FP16) { + indices_float.resize(num_indices); + const __fp16* half_indices = indices_buffer.data_as<__fp16>(); + for (size_t i = 0; i < num_indices; i++) { + indices_float[i] = static_cast(half_indices[i]); + } + indices_ptr = indices_float.data(); } else { indices_float.resize(num_indices); const int8_t* int_indices = indices_buffer.data_as(); diff --git a/local_integration_tests/CMakeLists.txt b/local_integration_tests/CMakeLists.txt new file mode 100644 index 000000000..317318503 --- /dev/null +++ b/local_integration_tests/CMakeLists.txt @@ -0,0 +1,62 @@ +cmake_minimum_required(VERSION 3.16) +project(CactusLocalIntegrationTests LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +if(NOT CACTUS_REPO_ROOT) + get_filename_component(CACTUS_REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.." ABSOLUTE) +endif() + +set(CACTUS_LIB "${CACTUS_REPO_ROOT}/cactus/build/libcactus.a") +if(NOT EXISTS "${CACTUS_LIB}") + message(FATAL_ERROR "Missing ${CACTUS_LIB}; run ./cactus/build.sh first") +endif() + +add_executable(integration_runner native_runner.cpp) +target_link_libraries(integration_runner PRIVATE "${CACTUS_LIB}") +target_include_directories(integration_runner PRIVATE + "${CACTUS_REPO_ROOT}/cactus-engine" + "${CACTUS_REPO_ROOT}/cactus-engine/src" + "${CACTUS_REPO_ROOT}/cactus-graph" + "${CACTUS_REPO_ROOT}/cactus-kernels" + "${CACTUS_REPO_ROOT}/cactus-kernels/src" + "${CACTUS_REPO_ROOT}/cactus-kernels/libs" +) + +find_package(Threads REQUIRED) +target_link_libraries(integration_runner PRIVATE Threads::Threads) + +if(APPLE) + find_library(COREML_FRAMEWORK CoreML REQUIRED) + find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) + find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) + find_library(SECURITY_FRAMEWORK Security REQUIRED) + find_library(SYSTEMCONFIGURATION_FRAMEWORK SystemConfiguration REQUIRED) + find_library(CFNETWORK_FRAMEWORK CFNetwork REQUIRED) + target_link_libraries(integration_runner PRIVATE + "${ACCELERATE_FRAMEWORK}" + "${COREML_FRAMEWORK}" + "${FOUNDATION_FRAMEWORK}" + "${SECURITY_FRAMEWORK}" + "${SYSTEMCONFIGURATION_FRAMEWORK}" + "${CFNETWORK_FRAMEWORK}" + ) + set(CACTUS_CURL_ROOT "${CACTUS_REPO_ROOT}/cactus-engine/libs/curl") + set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") + if(EXISTS "${CACTUS_CURL_LIBRARY_MACOS}") + target_link_libraries(integration_runner PRIVATE "${CACTUS_CURL_LIBRARY_MACOS}") + target_include_directories(integration_runner PRIVATE "${CACTUS_CURL_ROOT}/include") + else() + find_library(CURL_LIBRARY NAMES curl) + if(CURL_LIBRARY) + target_link_libraries(integration_runner PRIVATE "${CURL_LIBRARY}") + endif() + endif() +else() + find_package(CURL) + if(CURL_FOUND) + target_link_libraries(integration_runner PRIVATE ${CURL_LIBRARIES}) + target_include_directories(integration_runner PRIVATE ${CURL_INCLUDE_DIRS}) + endif() +endif() diff --git a/local_integration_tests/README.md b/local_integration_tests/README.md new file mode 100644 index 000000000..c6442606a --- /dev/null +++ b/local_integration_tests/README.md @@ -0,0 +1,43 @@ +# Local integration tests + +These tests are intentionally local and uncommitted until the suite is ready to become part of the repository. + +## Requirements + +- Run from the repository root. +- Use existing converted model artifacts only. +- Set `CACTUS_INTEGRATION_WEIGHTS` to override model discovery. +- Default discovery checks `../cactus/weights`, then `./weights`. + +## Commands + +```bash +PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -m smoke +PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -m full +PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -k parakeet +``` + +The pytest session builds `./cactus/build.sh` once and compiles `local_integration_tests/.tmp/build/integration_runner`. + +## Coverage + +- Gemma4 E2B: text and multiturn reuse with the matrix Cactus artifact, `gemma-4-e2b-it-sharedkv`. +- Gemma4 E2B: image, audio, and image+audio combined with the latest local multimodal artifact, preferring `gemma-4-e2b-it-alt-fixed-rowtuned-mm`. +- Qwen3-VL: text and long-prompt chunked prefill smoke. +- Qwen3-VL: image with the matrix Cactus artifact, `qwen3-vl-2b-instruct-reconvert`. +- LFM2-VL: text, image, long-prompt chunked prefill smoke. +- Parakeet TDT: file and PCM transcription. +- Whisper small: full-mode file and PCM transcription when a component-backed local bundle is available. + +## Current local result + +```bash +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s +# 7 passed, 2 deselected + +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -k parakeet -q -s +# 1 passed, 8 deselected + +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m full -q -s +# 2 skipped, 7 deselected +``` diff --git a/local_integration_tests/conftest.py b/local_integration_tests/conftest.py new file mode 100644 index 000000000..adcfe06a4 --- /dev/null +++ b/local_integration_tests/conftest.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from typing import Any + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +TMP_DIR = REPO_ROOT / "local_integration_tests" / ".tmp" +ASSET_DIR = REPO_ROOT / "cactus-engine" / "tests" / "assets" + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", "smoke: fast local integration coverage") + config.addinivalue_line("markers", "full: slower coverage for every available key model") + + +def _weights_root() -> Path: + explicit = os.environ.get("CACTUS_INTEGRATION_WEIGHTS") + candidates = [explicit] if explicit else [] + candidates.extend([str(REPO_ROOT.parent / "cactus" / "weights"), str(REPO_ROOT / "weights")]) + for candidate in candidates: + if not candidate: + continue + path = Path(candidate).expanduser().resolve() + if path.exists(): + return path + return Path(candidates[0]).expanduser().resolve() + + +@pytest.fixture(scope="session") +def weights_root() -> Path: + root = _weights_root() + if not root.exists(): + pytest.fail(f"weights root does not exist: {root}") + return root + + +@pytest.fixture(scope="session") +def assets() -> dict[str, Path]: + result = { + "audio": ASSET_DIR / "test.wav", + "image_monkey": ASSET_DIR / "test_monkey.png", + "image_thing": ASSET_DIR / "test_thing.png", + } + missing = [str(path) for path in result.values() if not path.exists()] + if missing: + pytest.fail("missing integration fixtures: " + ", ".join(missing)) + return result + + +@pytest.fixture(scope="session") +def runner() -> Path: + env = os.environ.copy() + env["CACTUS_NO_CLOUD_TELE"] = "1" + subprocess.run([str(REPO_ROOT / "cactus" / "build.sh")], cwd=REPO_ROOT, env=env, check=True) + + build_dir = TMP_DIR / "build" + build_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + "cmake", + "-S", + str(REPO_ROOT / "local_integration_tests"), + "-B", + str(build_dir), + f"-DCACTUS_REPO_ROOT={REPO_ROOT}", + "-DCMAKE_BUILD_TYPE=Release", + ], + cwd=REPO_ROOT, + check=True, + ) + subprocess.run( + ["cmake", "--build", str(build_dir), "--target", "integration_runner", "-j", str(os.cpu_count() or 4)], + cwd=REPO_ROOT, + check=True, + ) + binary = build_dir / "integration_runner" + if not binary.exists(): + pytest.fail(f"integration runner was not built: {binary}") + return binary + + +def _is_transpiled_bundle(path: Path) -> bool: + return (path / "components" / "manifest.json").exists() or (path / "manifest.json").exists() + + +def model_path( + weights_root: Path, + *candidates: str, + required: bool = True, + transpiled: bool = True, + native_config: bool = False, +) -> Path | None: + for name in candidates: + path = weights_root / name + if not path.exists(): + continue + if transpiled and not _is_transpiled_bundle(path): + continue + if native_config and not (path / "config.txt").exists(): + continue + if path.exists(): + return path + if required: + suffixes = [] + if transpiled: + suffixes.append("components/manifest.json") + if native_config: + suffixes.append("config.txt") + suffix = " with " + " and ".join(suffixes) if suffixes else "" + pytest.fail("missing required model artifact" + suffix + "; tried: " + ", ".join(candidates)) + pytest.skip("optional model artifact not available; tried: " + ", ".join(candidates)) + return None + + +def run_runner(runner: Path, *args: object, timeout: int = 180) -> dict[str, Any]: + env = os.environ.copy() + env["CACTUS_NO_CLOUD_TELE"] = "1" + env.pop("CACTUS_KV_CACHE_FP16", None) + completed = subprocess.run( + [str(runner), *(str(arg) for arg in args)], + cwd=REPO_ROOT, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + if completed.returncode != 0: + pytest.fail( + f"integration runner failed with {completed.returncode}\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) + try: + return json.loads(completed.stdout) + except json.JSONDecodeError as exc: + pytest.fail(f"runner did not return JSON: {exc}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}") + + +def assert_completion(result: dict[str, Any]) -> None: + response = str(result.get("response") or result.get("text") or "").strip() + token_count = int(result.get("completion_tokens") or result.get("tokens_predicted") or result.get("decode_tokens") or 0) + assert response or token_count > 0, result + + +def assert_transcript(result: dict[str, Any], fragments: tuple[str, ...] = ("hello", "testing", "voice")) -> None: + transcript = str(result.get("response") or result.get("transcript") or "").strip().lower() + assert len(transcript) > 5, result + assert any(fragment in transcript for fragment in fragments), transcript diff --git a/local_integration_tests/native_runner.cpp b/local_integration_tests/native_runner.cpp new file mode 100644 index 000000000..fc1cd5d96 --- /dev/null +++ b/local_integration_tests/native_runner.cpp @@ -0,0 +1,1108 @@ +#include "cactus_engine.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define private public +#include "engine.h" +#undef private +#include "cactus_kernels.h" +#include "stb_image_resize2.h" +#include "utils.h" +#include "wav.h" + +namespace { + +constexpr size_t kResponseBufferSize = 1 << 20; +constexpr uint64_t kFnvOffset = 1469598103934665603ull; +constexpr uint64_t kFnvPrime = 1099511628211ull; + +struct Args { + std::string command; + std::string model; + std::string prompt; + std::string image; + std::string audio; + std::string messages_json; + int max_tokens = 32; +}; + +std::string arg_value(int& i, int argc, char** argv) { + if (i + 1 >= argc) throw std::runtime_error(std::string("missing value for ") + argv[i]); + return argv[++i]; +} + +Args parse_args(int argc, char** argv) { + if (argc < 2) throw std::runtime_error("usage: integration_runner [options]"); + Args args; + args.command = argv[1]; + for (int i = 2; i < argc; ++i) { + std::string flag = argv[i]; + if (flag == "--model") args.model = arg_value(i, argc, argv); + else if (flag == "--prompt") args.prompt = arg_value(i, argc, argv); + else if (flag == "--image") args.image = arg_value(i, argc, argv); + else if (flag == "--audio") args.audio = arg_value(i, argc, argv); + else if (flag == "--messages-json") args.messages_json = arg_value(i, argc, argv); + else if (flag == "--max-tokens") args.max_tokens = std::max(0, std::stoi(arg_value(i, argc, argv))); + else throw std::runtime_error("unknown flag: " + flag); + } + if (args.model.empty()) throw std::runtime_error("--model is required"); + return args; +} + +std::string escape_json(const std::string& value) { + std::ostringstream out; + for (char ch : value) { + switch (ch) { + case '\\': out << "\\\\"; break; + case '"': out << "\\\""; break; + case '\n': out << "\\n"; break; + case '\r': out << "\\r"; break; + case '\t': out << "\\t"; break; + default: + if (static_cast(ch) < 0x20) { + out << "\\u00"; + const char* hex = "0123456789abcdef"; + out << hex[(ch >> 4) & 0x0f] << hex[ch & 0x0f]; + } else { + out << ch; + } + } + } + return out.str(); +} + +template +std::string json_array(const std::vector& values, size_t limit = std::numeric_limits::max()) { + std::ostringstream out; + out << "["; + const size_t n = std::min(values.size(), limit); + for (size_t i = 0; i < n; ++i) { + if (i) out << ","; + out << values[i]; + } + out << "]"; + return out.str(); +} + +std::string json_float_array(const std::vector& values, size_t limit = std::numeric_limits::max()) { + std::ostringstream out; + out << "["; + const size_t n = std::min(values.size(), limit); + for (size_t i = 0; i < n; ++i) { + if (i) out << ","; + if (std::isfinite(values[i])) out << values[i]; + else out << "null"; + } + out << "]"; + return out.str(); +} + +uint64_t fnv1a_bytes(const void* data, size_t size) { + const auto* bytes = static_cast(data); + uint64_t hash = kFnvOffset; + for (size_t i = 0; i < size; ++i) { + hash ^= static_cast(bytes[i]); + hash *= kFnvPrime; + } + return hash; +} + +std::string hex64(uint64_t value) { + std::ostringstream out; + out << std::hex << std::setw(16) << std::setfill('0') << value; + return out.str(); +} + +template +std::vector first_values(const std::vector& values, size_t n) { + return std::vector(values.begin(), values.begin() + std::min(values.size(), n)); +} + +std::vector last_values(const std::vector& values, size_t n) { + if (values.size() <= n) return values; + return std::vector(values.end() - static_cast(n), values.end()); +} + +int run_e002_tokens(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for token probe: " + args.model); + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() ? "Respond briefly." : args.prompt; + message.images.push_back(args.image); + std::vector messages{message}; + + const std::string full_prompt = tokenizer->format_chat_prompt(messages, true); + const std::string no_generation_prompt = tokenizer->format_chat_prompt(messages, false); + const std::vector tokens = tokenizer->encode(full_prompt); + const std::vector prefix_tokens = tokenizer->encode(no_generation_prompt); + const uint32_t image_token_id = model->get_config().image_token_id ? model->get_config().image_token_id : tokenizer->get_image_token_id(); + + size_t image_start = tokens.size(); + size_t image_count = 0; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i] == image_token_id) { + if (image_count == 0) image_start = i; + ++image_count; + } + } + + std::ostringstream out; + out << "{" + << "\"token_count\":" << tokens.size() << "," + << "\"assistant_generation_start\":" << prefix_tokens.size() << "," + << "\"image_token_id\":" << image_token_id << "," + << "\"image_token_start\":" << (image_count ? static_cast(image_start) : -1) << "," + << "\"image_token_count\":" << image_count << "," + << "\"first_20\":" << json_array(first_values(tokens, 20)) << "," + << "\"last_20\":" << json_array(last_values(tokens, 20)) << "," + << "\"input_ids\":" << json_array(tokens) << "," + << "\"prompt_prefix\":\"" << escape_json(full_prompt.substr(0, 320)) << "\"" + << "}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_preprocess(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + cactus::engine::Config config; + if (!config.from_json(args.model + "/config.txt")) { + throw std::runtime_error("failed to read config.txt for preprocess probe: " + args.model); + } + auto prep = cactus::engine::preprocess_gemma4_image(args.image, config); + double sum = 0.0; + float min_v = prep.pixel_values.empty() ? 0.0f : prep.pixel_values[0]; + float max_v = min_v; + for (float v : prep.pixel_values) { + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + size_t valid_positions = 0; + for (size_t i = 0; i + 1 < prep.pixel_position_ids.size(); i += 2) { + if (prep.pixel_position_ids[i] >= 0 && prep.pixel_position_ids[i + 1] >= 0) ++valid_positions; + } + std::vector first_positions; + const size_t position_rows = std::min(16, prep.pixel_position_ids.size() / 2); + first_positions.reserve(position_rows * 2); + for (size_t i = 0; i < position_rows * 2; ++i) first_positions.push_back(prep.pixel_position_ids[i]); + + std::ostringstream out; + out << std::setprecision(9) + << "{" + << "\"pixel_values_shape\":[1," << prep.max_patches << "," << prep.patch_dim << "]," + << "\"pixel_position_ids_shape\":[1," << prep.max_patches << ",2]," + << "\"num_patches\":" << prep.num_patches << "," + << "\"max_patches\":" << prep.max_patches << "," + << "\"patch_dim\":" << prep.patch_dim << "," + << "\"pixel_values_min\":" << min_v << "," + << "\"pixel_values_max\":" << max_v << "," + << "\"pixel_values_mean\":" << (prep.pixel_values.empty() ? 0.0 : sum / static_cast(prep.pixel_values.size())) << "," + << "\"pixel_values_hash\":\"" << hex64(fnv1a_bytes(prep.pixel_values.data(), prep.pixel_values.size() * sizeof(float))) << "\"," + << "\"pixel_position_ids_hash\":\"" << hex64(fnv1a_bytes(prep.pixel_position_ids.data(), prep.pixel_position_ids.size() * sizeof(int64_t))) << "\"," + << "\"valid_position_count\":" << valid_positions << "," + << "\"first_pixel_values\":" << json_array(first_values(prep.pixel_values, 32)) << "," + << "\"first_position_rows_flat\":" << json_array(first_positions) << "," + << "\"pixel_values\":" << json_array(prep.pixel_values) << "," + << "\"pixel_position_ids\":" << json_array(prep.pixel_position_ids) + << "}\n"; + std::cout << out.str(); + return 0; +} + +struct E003Variant { + std::string name; + std::vector resized; + std::vector pixel_values; +}; + +void resize_float_default(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { + std::vector src_float(static_cast(width) * height * 3); + for (size_t i = 0; i < src_float.size(); ++i) src_float[i] = static_cast(raw[i]); + if (!stbir_resize_float_linear(src_float.data(), width, height, 0, resized.data(), target_w, target_h, 0, STBIR_RGB)) { + throw std::runtime_error("stbir_resize_float_linear failed"); + } +} + +void resize_uint8_default(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { + std::vector tmp(static_cast(target_w) * target_h * 3); + if (!stbir_resize_uint8_linear(raw, width, height, 0, tmp.data(), target_w, target_h, 0, STBIR_RGB)) { + throw std::runtime_error("stbir_resize_uint8_linear failed"); + } + for (size_t i = 0; i < tmp.size(); ++i) resized[i] = static_cast(tmp[i]); +} + +void resize_float_triangle(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { + std::vector src_float(static_cast(width) * height * 3); + for (size_t i = 0; i < src_float.size(); ++i) src_float[i] = static_cast(raw[i]); + STBIR_RESIZE resize; + stbir_resize_init(&resize, src_float.data(), width, height, 0, resized.data(), target_w, target_h, 0, STBIR_RGB, STBIR_TYPE_FLOAT); + stbir_set_filters(&resize, STBIR_FILTER_TRIANGLE, STBIR_FILTER_TRIANGLE); + if (!stbir_resize_extended(&resize)) throw std::runtime_error("triangle float resize failed"); +} + +void resize_uint8_triangle(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { + std::vector tmp(static_cast(target_w) * target_h * 3); + STBIR_RESIZE resize; + stbir_resize_init(&resize, raw, width, height, 0, tmp.data(), target_w, target_h, 0, STBIR_RGB, STBIR_TYPE_UINT8); + stbir_set_filters(&resize, STBIR_FILTER_TRIANGLE, STBIR_FILTER_TRIANGLE); + if (!stbir_resize_extended(&resize)) throw std::runtime_error("triangle uint8 resize failed"); + for (size_t i = 0; i < tmp.size(); ++i) resized[i] = static_cast(tmp[i]); +} + +std::vector pack_gemma4_patches(const std::vector& resized, int target_w, int target_h, int patch_size, + size_t max_patches, size_t patch_dim, float rescale_factor) { + const int patch_h = target_h / patch_size; + const int patch_w = target_w / patch_size; + std::vector pixel_values(max_patches * patch_dim, 0.0f); + for (int py = 0; py < patch_h; ++py) { + for (int px = 0; px < patch_w; ++px) { + const size_t patch_idx = static_cast(py) * patch_w + px; + float* dst = pixel_values.data() + patch_idx * patch_dim; + for (int y = 0; y < patch_size; ++y) { + const int img_y = py * patch_size + y; + for (int x = 0; x < patch_size; ++x) { + const int img_x = px * patch_size + x; + const size_t src_off = (static_cast(img_y) * target_w + img_x) * 3; + const size_t dst_off = (static_cast(y) * patch_size + x) * 3; + dst[dst_off + 0] = resized[src_off + 0] * rescale_factor; + dst[dst_off + 1] = resized[src_off + 1] * rescale_factor; + dst[dst_off + 2] = resized[src_off + 2] * rescale_factor; + } + } + } + } + return pixel_values; +} + +void write_variant_json(std::ostream& out, const E003Variant& variant, bool comma) { + double sum = 0.0; + float min_v = variant.pixel_values.empty() ? 0.0f : variant.pixel_values[0]; + float max_v = min_v; + for (float v : variant.pixel_values) { + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + float resized_min = variant.resized.empty() ? 0.0f : variant.resized[0]; + float resized_max = resized_min; + for (float v : variant.resized) { + resized_min = std::min(resized_min, v); + resized_max = std::max(resized_max, v); + } + if (comma) out << ","; + out << "\"" << variant.name << "\":{" + << "\"resized_min\":" << resized_min << "," + << "\"resized_max\":" << resized_max << "," + << "\"first_resized_values\":" << json_array(first_values(variant.resized, 32)) << "," + << "\"pixel_values_min\":" << min_v << "," + << "\"pixel_values_max\":" << max_v << "," + << "\"pixel_values_mean\":" << (variant.pixel_values.empty() ? 0.0 : sum / static_cast(variant.pixel_values.size())) << "," + << "\"first_pixel_values\":" << json_array(first_values(variant.pixel_values, 32)) << "," + << "\"pixel_values\":" << json_array(variant.pixel_values) + << "}"; +} + +std::string precision_name(Precision precision) { + switch (precision) { + case Precision::FP32: return "FP32"; + case Precision::FP16: return "FP16"; + case Precision::INT8: return "INT8"; + default: return "UNKNOWN"; + } +} + +std::string op_type_name(OpType op) { + switch (op) { + case OpType::INPUT: return "INPUT"; + case OpType::PRECISION_CAST: return "PRECISION_CAST"; + case OpType::ADD: return "ADD"; + case OpType::ADD_CLIPPED: return "ADD_CLIPPED"; + case OpType::SUBTRACT: return "SUBTRACT"; + case OpType::MULTIPLY: return "MULTIPLY"; + case OpType::DIVIDE: return "DIVIDE"; + case OpType::ABS: return "ABS"; + case OpType::POW: return "POW"; + case OpType::FLATTEN: return "FLATTEN"; + case OpType::VIEW: return "VIEW"; + case OpType::MATMUL: return "MATMUL"; + case OpType::TRANSPOSE: return "TRANSPOSE"; + case OpType::RESHAPE: return "RESHAPE"; + case OpType::SLICE: return "SLICE"; + case OpType::GATHER: return "GATHER"; + case OpType::EMBEDDING: return "EMBEDDING"; + case OpType::BILINEAR_INTERPOLATION: return "BILINEAR_INTERPOLATION"; + case OpType::SUM: return "SUM"; + case OpType::MEAN: return "MEAN"; + case OpType::VARIANCE: return "VARIANCE"; + case OpType::MIN: return "MIN"; + case OpType::MAX: return "MAX"; + case OpType::CUMSUM: return "CUMSUM"; + case OpType::RMS_NORM: return "RMS_NORM"; + case OpType::ROPE: return "ROPE"; + case OpType::ROPE_GPTJ: return "ROPE_GPTJ"; + case OpType::SOFTMAX: return "SOFTMAX"; + case OpType::ATTENTION: return "ATTENTION"; + case OpType::ATTENTION_INT8_HYBRID: return "ATTENTION_INT8_HYBRID"; + case OpType::REL_POS_BIAS: return "REL_POS_BIAS"; + case OpType::CONV1D_CAUSAL: return "CONV1D_CAUSAL"; + case OpType::CONV1D_K3: return "CONV1D_K3"; + case OpType::CONV1D_K7S3: return "CONV1D_K7S3"; + case OpType::CONV1D: return "CONV1D"; + case OpType::CONV1D_SAME_DEPTHWISE_K9: return "CONV1D_SAME_DEPTHWISE_K9"; + case OpType::CONV1D_POINTWISE: return "CONV1D_POINTWISE"; + case OpType::CONV2D_K3S2P1: return "CONV2D_K3S2P1"; + case OpType::CONV2D_DEPTHWISE_K3S2P1: return "CONV2D_DEPTHWISE_K3S2P1"; + case OpType::CONV2D_POINTWISE_1X1: return "CONV2D_POINTWISE_1X1"; + case OpType::GLU: return "GLU"; + case OpType::BATCHNORM: return "BATCHNORM"; + case OpType::SCALAR_ADD: return "SCALAR_ADD"; + case OpType::SCALAR_SUBTRACT: return "SCALAR_SUBTRACT"; + case OpType::SCALAR_MULTIPLY: return "SCALAR_MULTIPLY"; + case OpType::SCALAR_DIVIDE: return "SCALAR_DIVIDE"; + case OpType::SCALAR_EXP: return "SCALAR_EXP"; + case OpType::SCALAR_SQRT: return "SCALAR_SQRT"; + case OpType::SCALAR_COS: return "SCALAR_COS"; + case OpType::SCALAR_SIN: return "SCALAR_SIN"; + case OpType::SCALAR_LOG: return "SCALAR_LOG"; + case OpType::RELU: return "RELU"; + case OpType::SILU: return "SILU"; + case OpType::GELU: return "GELU"; + case OpType::GELU_ERF: return "GELU_ERF"; + case OpType::SIGMOID: return "SIGMOID"; + case OpType::TANH: return "TANH"; + case OpType::SAMPLE: return "SAMPLE"; + case OpType::CONCAT: return "CONCAT"; + case OpType::CAT: return "CAT"; + case OpType::SCATTER_TOPK: return "SCATTER_TOPK"; + case OpType::TOPK: return "TOPK"; + case OpType::LAYERNORM: return "LAYERNORM"; + case OpType::GROUPNORM: return "GROUPNORM"; + case OpType::MOE_LAYER: return "MOE_LAYER"; + case OpType::INDEX: return "INDEX"; + case OpType::PERSISTENT: return "PERSISTENT"; + case OpType::LSTM_CELL: return "LSTM_CELL"; + case OpType::GATED_DELTANET_DECODE: return "GATED_DELTANET_DECODE"; + case OpType::GATED_DELTANET_PREFILL: return "GATED_DELTANET_PREFILL"; + case OpType::STFT: return "STFT"; + case OpType::ALTUP_PREDICT: return "ALTUP_PREDICT"; + case OpType::ALTUP_CORRECT: return "ALTUP_CORRECT"; + case OpType::GAUSSIAN_TOPK: return "GAUSSIAN_TOPK"; + case OpType::MAXPOOL1D: return "MAXPOOL1D"; + case OpType::BILSTM_SEQUENCE: return "BILSTM_SEQUENCE"; + case OpType::LEAKY_RELU: return "LEAKY_RELU"; + case OpType::CONV2D_K3S1P1: return "CONV2D_K3S1P1"; + case OpType::STATS_POOL: return "STATS_POOL"; + case OpType::WEIGHTED_STATS_POOL: return "WEIGHTED_STATS_POOL"; + case OpType::KV_CACHE_STATE: return "KV_CACHE_STATE"; + case OpType::KV_CACHE_APPEND: return "KV_CACHE_APPEND"; + case OpType::ATTENTION_CACHED: return "ATTENTION_CACHED"; + case OpType::CONV_CACHE_STATE: return "CONV_CACHE_STATE"; + case OpType::CONV_CACHE_APPEND: return "CONV_CACHE_APPEND"; + case OpType::RFFT: return "RFFT"; + case OpType::IRFFT: return "IRFFT"; + case OpType::MEL_FILTER_BANK: return "MEL_FILTER_BANK"; + case OpType::SPECTROGRAM: return "SPECTROGRAM"; + case OpType::IMAGE_PREPROCESS: return "IMAGE_PREPROCESS"; + case OpType::CLAMP: return "CLAMP"; + case OpType::DENSE_MLP_TQ_FUSED: return "DENSE_MLP_TQ_FUSED"; + case OpType::NOT_EQUAL: return "NOT_EQUAL"; + case OpType::SCALAR_NOT_EQUAL: return "SCALAR_NOT_EQUAL"; + } + return "UNKNOWN"; +} + +std::vector bytes_to_floats(const std::vector& bytes, Precision precision) { + std::vector values; + if (precision == Precision::FP32) { + const size_t count = bytes.size() / sizeof(float); + values.resize(count); + const auto* src = reinterpret_cast(bytes.data()); + for (size_t i = 0; i < count; ++i) values[i] = src[i]; + } else if (precision == Precision::FP16) { + const size_t count = bytes.size() / sizeof(__fp16); + values.resize(count); + const auto* src = reinterpret_cast(bytes.data()); + for (size_t i = 0; i < count; ++i) values[i] = static_cast(src[i]); + } else if (precision == Precision::INT8) { + const size_t count = bytes.size(); + values.resize(count); + const auto* src = reinterpret_cast(bytes.data()); + for (size_t i = 0; i < count; ++i) values[i] = static_cast(src[i]); + } + return values; +} + +std::vector buffer_to_floats(const BufferDesc& desc) { + std::vector values; + const void* data = desc.get_data(); + if (!data) return values; + if (desc.precision == Precision::FP32) { + values.resize(desc.byte_size / sizeof(float)); + const auto* src = static_cast(data); + for (size_t i = 0; i < values.size(); ++i) values[i] = src[i]; + } else if (desc.precision == Precision::FP16) { + values.resize(desc.byte_size / sizeof(__fp16)); + const auto* src = static_cast(data); + for (size_t i = 0; i < values.size(); ++i) values[i] = static_cast(src[i]); + } else if (desc.precision == Precision::INT8) { + values.resize(desc.byte_size); + const auto* src = static_cast(data); + for (size_t i = 0; i < values.size(); ++i) values[i] = static_cast(src[i]); + } + return values; +} + +void write_float_feature_json(std::ostream& out, + const std::string& name, + const std::vector& values, + const std::vector& shape, + Precision precision, + bool comma) { + double sum = 0.0; + float min_v = values.empty() ? 0.0f : values[0]; + float max_v = min_v; + double l2 = 0.0; + size_t finite_count = 0; + size_t nonfinite_count = 0; + for (float v : values) { + if (!std::isfinite(v)) { + ++nonfinite_count; + continue; + } + if (finite_count == 0) { + min_v = v; + max_v = v; + } + ++finite_count; + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + l2 += static_cast(v) * static_cast(v); + } + if (comma) out << ","; + out << "\"" << escape_json(name) << "\":{" + << "\"shape\":" << json_array(shape) << "," + << "\"precision\":\"" << precision_name(precision) << "\"," + << "\"count\":" << values.size() << "," + << "\"finite_count\":" << finite_count << "," + << "\"nonfinite_count\":" << nonfinite_count << "," + << "\"min\":" << min_v << "," + << "\"max\":" << max_v << "," + << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," + << "\"l2\":" << std::sqrt(l2) << "," + << "\"first_values\":" << json_float_array(values, 32) << "," + << "\"values\":" << json_float_array(values) + << "}"; +} + +int run_e003_preprocess_variants(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + cactus::engine::Config config; + if (!config.from_json(args.model + "/config.txt")) { + throw std::runtime_error("failed to read config.txt for E003 probe: " + args.model); + } + + const int patch_size = static_cast(config.vision_patch_size ? config.vision_patch_size : 16); + const int pooling_k = static_cast(config.vision_pooling_kernel_size ? config.vision_pooling_kernel_size : 3); + const size_t max_patches = static_cast(config.vision_default_output_length ? config.vision_default_output_length : 280) * pooling_k * pooling_k; + const int side_multiple = pooling_k * patch_size; + const size_t patch_dim = static_cast(3) * patch_size * patch_size; + const float rescale_factor = config.rescale_factor > 0.0f ? config.rescale_factor : (1.0f / 255.0f); + + int width = 0, height = 0, channels = 0; + unsigned char* raw = cactus_image_load(args.image.c_str(), &width, &height, &channels, 3); + if (!raw) throw std::runtime_error("failed to load image for E003 probe: " + args.image); + + const double target_pixels = static_cast(max_patches) * patch_size * patch_size; + const double pixel_count = std::max(1.0, static_cast(width) * static_cast(height)); + const double factor = std::sqrt(target_pixels / pixel_count); + int target_h = static_cast(std::floor(factor * height / side_multiple)) * side_multiple; + int target_w = static_cast(std::floor(factor * width / side_multiple)) * side_multiple; + if (target_h == 0) target_h = side_multiple; + if (target_w == 0) target_w = side_multiple; + + std::vector variants; + for (const std::string& name : {"stb_default_float", "stb_default_uint8", "stb_triangle_float", "stb_triangle_uint8"}) { + E003Variant variant; + variant.name = name; + variant.resized.assign(static_cast(target_w) * target_h * 3, 0.0f); + if (name == "stb_default_float") resize_float_default(raw, width, height, variant.resized, target_w, target_h); + else if (name == "stb_default_uint8") resize_uint8_default(raw, width, height, variant.resized, target_w, target_h); + else if (name == "stb_triangle_float") resize_float_triangle(raw, width, height, variant.resized, target_w, target_h); + else resize_uint8_triangle(raw, width, height, variant.resized, target_w, target_h); + variant.pixel_values = pack_gemma4_patches(variant.resized, target_w, target_h, patch_size, max_patches, patch_dim, rescale_factor); + variants.push_back(std::move(variant)); + } + cactus_image_free(raw); + + std::ostringstream out; + out << std::setprecision(9) + << "{" + << "\"source_width\":" << width << "," + << "\"source_height\":" << height << "," + << "\"target_width\":" << target_w << "," + << "\"target_height\":" << target_h << "," + << "\"patch_size\":" << patch_size << "," + << "\"pooling_kernel_size\":" << pooling_k << "," + << "\"max_patches\":" << max_patches << "," + << "\"patch_dim\":" << patch_dim << "," + << "\"rescale_factor\":" << rescale_factor << "," + << "\"variants\":{"; + for (size_t i = 0; i < variants.size(); ++i) { + write_variant_json(out, variants[i], i > 0); + } + out << "}}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_vision(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for vision probe: " + args.model); + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() + ? "What animal is in this image? Reply in one complete short sentence." + : args.prompt; + message.images.push_back(args.image); + std::vector messages{message}; + std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); + if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { + throw std::runtime_error("chunk prefill path failed for vision probe"); + } + + std::ostringstream out; + out << std::setprecision(9) << "{\"inputs\":{"; + if (model->vision_encoder_) { + for (size_t i = 0; i < model->vision_encoder_->logical_inputs.size(); ++i) { + if (i) out << ","; + const std::string& input_name = model->vision_encoder_->logical_inputs[i]; + const size_t node_id = static_cast(model->vision_encoder_->runtime_input_node_ids[i]); + const auto& desc = model->vision_encoder_->graph->get_output_buffer(node_id); + out << "\"" << escape_json(input_name) << "\":{" + << "\"node_id\":" << node_id << "," + << "\"shape\":" << json_array(desc.shape) << "," + << "\"precision\":\"" << precision_name(desc.precision) << "\"," + << "\"byte_size\":" << desc.byte_size + << "}"; + } + } + out << "},\"features\":{"; + bool first = true; + for (const auto& kv : model->media_features_) { + const std::string& name = kv.first; + Precision precision = model->media_feature_precisions_[name]; + std::vector values = bytes_to_floats(kv.second, precision); + write_float_feature_json(out, name, values, model->media_feature_shapes_[name], precision, !first); + first = false; + } + out << "}}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_vision_node_summary(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for vision node summary: " + args.model); + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() + ? "What animal is in this image? Reply in one complete short sentence." + : args.prompt; + message.images.push_back(args.image); + std::vector messages{message}; + std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); + if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { + throw std::runtime_error("chunk prefill path failed for vision node summary"); + } + if (!model->vision_encoder_ || !model->vision_encoder_->graph) { + throw std::runtime_error("model has no vision encoder graph"); + } + + const auto& graph = *model->vision_encoder_->graph; + std::ostringstream out; + out << std::setprecision(9) << "{\"nodes\":["; + bool first = true; + for (const auto& node_ptr : graph.nodes_) { + const GraphNode& node = *node_ptr; + const auto& desc = node.output_buffer; + const bool interesting = + node.id < 25 || + (node.id >= 660 && node.id <= 710) || + desc.shape == std::vector{1, 2520} || + desc.shape == std::vector{1, 2520, 1} || + desc.shape == std::vector{1, 2520, 2} || + desc.shape == std::vector{1, 2520, 768} || + desc.shape == std::vector{1, 256, 768} || + desc.shape == std::vector{256, 768} || + desc.shape == std::vector{256, 1536}; + if (!interesting) continue; + std::vector values = buffer_to_floats(desc); + double sum = 0.0; + float min_v = values.empty() ? 0.0f : values[0]; + float max_v = min_v; + size_t finite_count = 0; + size_t nonfinite_count = 0; + for (float v : values) { + if (!std::isfinite(v)) { + ++nonfinite_count; + continue; + } + if (finite_count == 0) { + min_v = v; + max_v = v; + } + ++finite_count; + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + if (!first) out << ","; + first = false; + out << "{" + << "\"node_id\":" << node.id << "," + << "\"op\":\"" << op_type_name(node.op_type) << "\"," + << "\"inputs\":" << json_array(node.input_ids) << "," + << "\"shape\":" << json_array(desc.shape) << "," + << "\"precision\":\"" << precision_name(desc.precision) << "\"," + << "\"byte_size\":" << desc.byte_size << "," + << "\"value_count\":" << values.size() << "," + << "\"finite_count\":" << finite_count << "," + << "\"nonfinite_count\":" << nonfinite_count << "," + << "\"min\":" << min_v << "," + << "\"max\":" << max_v << "," + << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," + << "\"first_values\":" << json_float_array(values, 32) << "," + << "\"values\":" << json_float_array(values) + << "}"; + } + out << "]}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_vision_selected_nodes(const Args& args) { + if (args.image.empty()) throw std::runtime_error("--image is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for selected vision nodes: " + args.model); + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() + ? "What animal is in this image? Reply in one complete short sentence." + : args.prompt; + message.images.push_back(args.image); + std::vector messages{message}; + std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); + if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { + throw std::runtime_error("chunk prefill path failed for selected vision nodes"); + } + if (!model->vision_encoder_ || !model->vision_encoder_->graph) { + throw std::runtime_error("model has no vision encoder graph"); + } + + const std::vector selected = { + 672, 675, 699, 700, + 752, 753, 756, 757, 789, 792, 793, 825, 828, 829, 846, 847, 874, 875, + 2757, 2758, 2804, 2805, 2806, 2807, 2812, 2813, 2814, 2815, 2816, + }; + const auto& graph = *model->vision_encoder_->graph; + std::ostringstream out; + out << std::setprecision(9) << "{\"nodes\":{"; + bool first = true; + for (size_t node_id : selected) { + auto it = graph.node_index_map_.find(node_id); + if (it == graph.node_index_map_.end()) continue; + const GraphNode& node = *graph.nodes_[it->second]; + const auto& desc = node.output_buffer; + std::vector values = buffer_to_floats(desc); + if (!first) out << ","; + first = false; + out << "\"" << node_id << "\":{" + << "\"node_id\":" << node.id << "," + << "\"op\":\"" << op_type_name(node.op_type) << "\"," + << "\"inputs\":" << json_array(node.input_ids) << "," + << "\"shape\":" << json_array(desc.shape) << "," + << "\"precision\":\"" << precision_name(desc.precision) << "\"," + << "\"values\":" << json_float_array(values) + << "}"; + } + out << "}}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_audio_summary(const Args& args) { + if (args.audio.empty()) throw std::runtime_error("--audio is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for audio summary: " + args.model); + } + AudioFP32 wav = load_wav(args.audio); + std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); + auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); + model->run_audio_encoder(prep.features); + + std::ostringstream out; + out << std::setprecision(9) << "{" + << "\"num_frames\":" << prep.num_frames << "," + << "\"num_soft_tokens\":" << prep.num_soft_tokens << "," + << "\"feature_count\":" << prep.features.size() << "," + << "\"feature_first_values\":" << json_float_array(prep.features, 32) << "," + << "\"media_features\":{"; + bool first_feature = true; + for (const auto& kv : model->media_features_) { + const std::string& name = kv.first; + Precision precision = model->media_feature_precisions_[name]; + std::vector values = bytes_to_floats(kv.second, precision); + double sum = 0.0; + float min_v = values.empty() ? 0.0f : values[0]; + float max_v = min_v; + size_t finite_count = 0; + size_t nonfinite_count = 0; + for (float v : values) { + if (!std::isfinite(v)) { + ++nonfinite_count; + continue; + } + if (finite_count == 0) { + min_v = v; + max_v = v; + } + ++finite_count; + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + if (!first_feature) out << ","; + first_feature = false; + out << "\"" << name << "\":{" + << "\"shape\":" << json_array(model->media_feature_shapes_[name]) << "," + << "\"precision\":\"" << precision_name(precision) << "\"," + << "\"value_count\":" << values.size() << "," + << "\"finite_count\":" << finite_count << "," + << "\"nonfinite_count\":" << nonfinite_count << "," + << "\"min\":" << min_v << "," + << "\"max\":" << max_v << "," + << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," + << "\"first_values\":" << json_float_array(values, 32) << "," + << "\"values\":" << json_float_array(values) + << "}"; + } + out << "}}\n"; + std::cout << out.str(); + return 0; +} + +int run_e002_audio_node_summary(const Args& args) { + if (args.audio.empty()) throw std::runtime_error("--audio is required"); + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for audio node summary: " + args.model); + } + AudioFP32 wav = load_wav(args.audio); + std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); + auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); + model->run_audio_encoder(prep.features); + if (!model->audio_encoder_ || !model->audio_encoder_->graph) { + throw std::runtime_error("model has no audio encoder graph"); + } + + const auto& graph = *model->audio_encoder_->graph; + std::ostringstream out; + out << std::setprecision(9) << "{\"nodes\":["; + bool first = true; + for (size_t order = 0; order < graph.nodes_.size(); ++order) { + const GraphNode& node = *graph.nodes_[order]; + const auto& desc = node.output_buffer; + const auto& shape = desc.shape; + bool interesting = node.op_type != OpType::INPUT; + if (shape.empty()) interesting = false; + if (!interesting) continue; + + std::vector values = buffer_to_floats(desc); + double sum = 0.0; + float min_v = values.empty() ? 0.0f : values[0]; + float max_v = min_v; + size_t finite_count = 0; + size_t nonfinite_count = 0; + for (float v : values) { + if (!std::isfinite(v)) { + ++nonfinite_count; + continue; + } + if (finite_count == 0) { + min_v = v; + max_v = v; + } + ++finite_count; + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += static_cast(v); + } + if (!first) out << ","; + first = false; + out << "{" + << "\"order\":" << order << "," + << "\"node_id\":" << node.id << "," + << "\"op\":\"" << op_type_name(node.op_type) << "\"," + << "\"inputs\":" << json_array(node.input_ids) << "," + << "\"shape\":" << json_array(shape) << "," + << "\"precision\":\"" << precision_name(desc.precision) << "\"," + << "\"value_count\":" << values.size() << "," + << "\"finite_count\":" << finite_count << "," + << "\"nonfinite_count\":" << nonfinite_count << "," + << "\"min\":" << min_v << "," + << "\"max\":" << max_v << "," + << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," + << "\"first_values\":" << json_float_array(values, 16) + << "}"; + } + out << "]}\n"; + std::cout << out.str(); + return 0; +} + +int run_gemma_first_token(const Args& args) { + auto model = cactus::engine::create_model(args.model); + if (!model || !model->init(args.model, 4096, "", false)) { + throw std::runtime_error("failed to initialize model for first token probe: " + args.model); + } + if (args.command == "gemma-first-token-media-step") { + model->lm_encoder_ = nullptr; + } + auto* tokenizer = model->get_tokenizer(); + if (!tokenizer) throw std::runtime_error("model has no tokenizer"); + + cactus::engine::ChatMessage message; + message.role = "user"; + message.content = args.prompt.empty() ? "Respond briefly." : args.prompt; + if (!args.image.empty()) message.images.push_back(args.image); + + std::vector audio_features; + size_t audio_num_frames = 0; + if (!args.audio.empty()) { + message.audio.push_back(args.audio); + AudioFP32 wav = load_wav(args.audio); + std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); + auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); + audio_features = std::move(prep.features); + audio_num_frames = prep.num_frames; + message.audio_soft_token_count = prep.num_soft_tokens; + } + + std::vector messages{message}; + std::string prompt = tokenizer->format_chat_prompt(messages, true); + std::vector tokens = tokenizer->encode(prompt); + if (tokens.empty()) throw std::runtime_error("first token probe produced empty prompt"); + std::vector prefill_tokens(tokens.begin(), tokens.end() - 1); + if (!prefill_tokens.empty()) { + model->prefill_with_media(prefill_tokens, args.image.empty() ? std::vector{} : std::vector{args.image}, audio_features); + } + uint32_t first = model->decode({tokens.back()}, 0.0f, 1.0f, 1); + std::string text = tokenizer->decode({first}); + std::vector> top_logits; + if (model->decoder_ && !model->decoder_->output_node_ids.empty()) { + size_t out_node = static_cast(model->decoder_->output_node_ids[0]); + const auto& desc = model->decoder_->graph->get_output_buffer(out_node); + std::vector logits = buffer_to_floats(desc); + if (!desc.shape.empty() && !logits.empty()) { + size_t vocab = desc.shape.back(); + size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; + size_t row = seq > 0 ? seq - 1 : 0; + size_t start = row * vocab; + size_t end = std::min(start + vocab, logits.size()); + for (size_t i = start; i < end; ++i) top_logits.emplace_back(i - start, logits[i]); + size_t keep = std::min(10, top_logits.size()); + std::partial_sort( + top_logits.begin(), + top_logits.begin() + keep, + top_logits.end(), + [](const auto& a, const auto& b) { return a.second > b.second; }); + top_logits.resize(keep); + } + } + + const uint32_t image_token_id = model->get_config().image_token_id ? model->get_config().image_token_id : tokenizer->get_image_token_id(); + const uint32_t audio_token_id = model->get_config().audio_token_id; + size_t image_token_count = 0; + size_t audio_token_count = 0; + for (uint32_t token : tokens) { + if (image_token_id != 0 && token == image_token_id) ++image_token_count; + if (audio_token_id != 0 && token == audio_token_id) ++audio_token_count; + } + + std::ostringstream out; + out << "{" + << "\"prompt_token_count\":" << tokens.size() << "," + << "\"prefill_token_count\":" << prefill_tokens.size() << "," + << "\"last_prompt_token\":" << tokens.back() << "," + << "\"first_token\":" << first << "," + << "\"first_text\":\"" << escape_json(text) << "\"," + << "\"eos_token\":" << tokenizer->get_eos_token() << "," + << "\"is_eos\":" << (first == tokenizer->get_eos_token() ? "true" : "false") << "," + << "\"image_token_count\":" << image_token_count << "," + << "\"audio_token_id\":" << audio_token_id << "," + << "\"audio_token_count\":" << audio_token_count << "," + << "\"audio_num_frames\":" << audio_num_frames << "," + << "\"top_logits\":["; + for (size_t i = 0; i < top_logits.size(); ++i) { + if (i) out << ","; + out << "{\"token\":" << top_logits[i].first + << ",\"score\":" << top_logits[i].second + << ",\"text\":\"" << escape_json(tokenizer->decode({static_cast(top_logits[i].first)})) << "\"}"; + } + out << "]," + << "\"prompt_prefix\":\"" << escape_json(prompt.substr(0, 320)) << "\"" + << "}\n"; + std::cout << out.str(); + return 0; +} + +std::string build_messages(const Args& args) { + if (!args.messages_json.empty()) return args.messages_json; + std::ostringstream out; + out << "[{\"role\":\"user\",\"content\":\"" << escape_json(args.prompt.empty() ? "Respond briefly." : args.prompt) << "\""; + if (!args.image.empty()) out << ",\"images\":[\"" << escape_json(args.image) << "\"]"; + if (!args.audio.empty()) out << ",\"audio\":[\"" << escape_json(args.audio) << "\"]"; + out << "}]"; + return out.str(); +} + +std::string complete_options(int max_tokens) { + std::ostringstream out; + out << "{" + << "\"max_tokens\":" << max_tokens << "," + << "\"temperature\":0.0," + << "\"top_p\":1.0," + << "\"top_k\":1," + << "\"telemetry_enabled\":false," + << "\"auto_handoff\":false," + << "\"confidence_threshold\":0.0," + << "\"stop_sequences\":[\"<|im_end|>\",\"\"]" + << "}"; + return out.str(); +} + +std::vector read_wav_pcm16(const std::string& path) { + std::ifstream in(path, std::ios::binary); + if (!in) throw std::runtime_error("failed to open wav: " + path); + in.seekg(0, std::ios::end); + std::streamoff size = in.tellg(); + if (size <= 44) throw std::runtime_error("wav file too small: " + path); + in.seekg(44, std::ios::beg); + std::vector pcm(static_cast(size - 44)); + in.read(reinterpret_cast(pcm.data()), static_cast(pcm.size())); + if (!in) throw std::runtime_error("failed to read wav pcm: " + path); + return pcm; +} + +int run_complete(const Args& args) { + cactus_model_t model = cactus_init(args.model.c_str(), nullptr, false); + if (!model) throw std::runtime_error("cactus_init failed for " + args.model); + std::vector response(kResponseBufferSize, 0); + std::string messages = build_messages(args); + std::string options = complete_options(args.max_tokens); + int rc = cactus_complete( + model, + messages.c_str(), + response.data(), + response.size(), + options.c_str(), + nullptr, + nullptr, + nullptr, + nullptr, + 0); + cactus_destroy(model); + std::cout << response.data() << "\n"; + return rc > 0 ? 0 : 2; +} + +int run_transcribe(const Args& args, bool pcm) { + if (args.audio.empty()) throw std::runtime_error("--audio is required"); + cactus_model_t model = cactus_init(args.model.c_str(), nullptr, false); + if (!model) throw std::runtime_error("cactus_init failed for " + args.model); + std::vector response(kResponseBufferSize, 0); + std::vector pcm_data; + const char* audio_path = args.audio.c_str(); + const uint8_t* pcm_buffer = nullptr; + size_t pcm_size = 0; + if (pcm) { + pcm_data = read_wav_pcm16(args.audio); + audio_path = nullptr; + pcm_buffer = pcm_data.data(); + pcm_size = pcm_data.size(); + } + int rc = cactus_transcribe( + model, + audio_path, + nullptr, + response.data(), + response.size(), + "{\"max_tokens\":200,\"telemetry_enabled\":false,\"auto_handoff\":false}", + nullptr, + nullptr, + pcm_buffer, + pcm_size); + cactus_destroy(model); + std::cout << response.data() << "\n"; + return rc > 0 ? 0 : 2; +} + +} // namespace + +int main(int argc, char** argv) { + try { + Args args = parse_args(argc, argv); + if (args.command == "e002-tokens") return run_e002_tokens(args); + if (args.command == "e002-preprocess") return run_e002_preprocess(args); + if (args.command == "e002-vision") return run_e002_vision(args); + if (args.command == "e002-vision-node-summary") return run_e002_vision_node_summary(args); + if (args.command == "e002-vision-selected-nodes") return run_e002_vision_selected_nodes(args); + if (args.command == "e002-audio-summary") return run_e002_audio_summary(args); + if (args.command == "e002-audio-node-summary") return run_e002_audio_node_summary(args); + if (args.command == "gemma-first-token") return run_gemma_first_token(args); + if (args.command == "gemma-first-token-media-step") return run_gemma_first_token(args); + if (args.command == "e003-preprocess-variants") return run_e003_preprocess_variants(args); + if (args.command == "complete") return run_complete(args); + if (args.command == "transcribe") return run_transcribe(args, false); + if (args.command == "transcribe-pcm") return run_transcribe(args, true); + throw std::runtime_error("unknown command: " + args.command); + } catch (const std::exception& exc) { + std::cerr << exc.what() << "\n"; + return 1; + } +} diff --git a/local_integration_tests/test_modalities.py b/local_integration_tests/test_modalities.py new file mode 100644 index 000000000..62251f3e2 --- /dev/null +++ b/local_integration_tests/test_modalities.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from conftest import assert_completion, assert_transcript, model_path, run_runner + + +def long_prompt() -> str: + sentence = "Summarize this integration test context in one concise sentence. " + return sentence * 40 + + +@pytest.mark.smoke +def test_gemma4_e2b_text_image_and_audio(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + text_model = model_path( + weights_root, + "gemma-4-e2b-it-sharedkv", + "gemma-4-e2b-it-sharedkv-c128", + "gemma-4-e2b-it", + native_config=True, + ) + media_model = model_path( + weights_root, + "gemma-4-e2b-it-alt-fixed-rowtuned-mm", + "gemma-4-e2b-it", + native_config=True, + ) + cases = [ + [text_model, "--prompt", "Reply with one short sentence about local inference."], + [media_model, "--prompt", "Describe this image in one short sentence.", "--image", assets["image_monkey"]], + [media_model, "--prompt", "Transcribe or summarize this audio in one short sentence.", "--audio", assets["audio"]], + ] + for case in cases: + model, *args = case + result = run_runner(runner, "complete", "--model", model, "--max-tokens", 24, *args, timeout=240) + assert_completion(result) + + +@pytest.mark.smoke +def test_gemma4_e2b_combined_image_audio( + runner: Path, + weights_root: Path, + assets: dict[str, Path], +) -> None: + model = model_path( + weights_root, + "gemma-4-e2b-it-alt-fixed-rowtuned-mm", + "gemma-4-e2b-it", + native_config=True, + ) + result = run_runner( + runner, + "complete", + "--model", + model, + "--prompt", + "Describe the image and summarize the audio in one short sentence.", + "--image", + assets["image_thing"], + "--audio", + assets["audio"], + "--max-tokens", + 24, + timeout=240, + ) + assert_completion(result) + + +@pytest.mark.smoke +def test_gemma4_e2b_multiturn_text_reuse(runner: Path, weights_root: Path) -> None: + model = model_path( + weights_root, + "gemma-4-e2b-it-sharedkv", + "gemma-4-e2b-it-sharedkv-c128", + "gemma-4-e2b-it", + native_config=True, + ) + messages = [ + {"role": "system", "content": "Be brief."}, + {"role": "user", "content": "Remember the name Henry."}, + {"role": "assistant", "content": "I will remember the name Henry."}, + {"role": "user", "content": "What name should you remember?"}, + ] + result = run_runner( + runner, + "complete", + "--model", + model, + "--messages-json", + json.dumps(messages), + "--max-tokens", + 24, + timeout=240, + ) + assert_completion(result) + + +@pytest.mark.full +def test_gemma4_e4b_all_modalities_when_available(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + model = model_path(weights_root, "gemma-4-e4b-it", required=False, native_config=True) + result = run_runner( + runner, + "complete", + "--model", + model, + "--prompt", + "Describe the image and audio briefly.", + "--image", + assets["image_monkey"], + "--audio", + assets["audio"], + "--max-tokens", + 24, + timeout=360, + ) + assert_completion(result) + + +@pytest.mark.smoke +def test_qwen3_vl_text_and_chunked_prefill(runner: Path, weights_root: Path) -> None: + model = model_path( + weights_root, + "qwen3-vl-2b-instruct-reconvert", + "qwen3-vl-2b-instruct-reconvert-c256", + "qwen3-vl-2b-instruct-reconvert-chunks", + "qwen3-vl-2b-instruct", + "qwen3-vl-2b", + native_config=True, + ) + for prompt in ("Reply with one short sentence about cactus.", long_prompt()): + result = run_runner(runner, "complete", "--model", model, "--prompt", prompt, "--max-tokens", 16, timeout=300) + assert_completion(result) + + +@pytest.mark.smoke +def test_qwen3_vl_image(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + model = model_path( + weights_root, + "qwen3-vl-2b-instruct-reconvert", + "qwen3-vl-2b-instruct-reconvert-c256", + "qwen3-vl-2b-instruct-reconvert-c512", + native_config=True, + ) + result = run_runner( + runner, + "complete", + "--model", + model, + "--prompt", + "Describe this image in one short sentence.", + "--image", + assets["image_monkey"], + "--max-tokens", + 16, + timeout=300, + ) + assert_completion(result) + + +@pytest.mark.smoke +def test_lfm2_vl_text_image_and_chunked_prefill(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + model = model_path( + weights_root, + "lfm2.5-vl-1.6b-retranspile-4096", + "lfm2.5-vl-1.6b-rechunk-c256", + "lfm2.5-vl-1.6b", + "lfm2.5-vl-1.6b-rechunk-c128", + "lfm2-vl-450m", + native_config=True, + ) + for prompt, image in ( + ("Reply with one short sentence about local model testing.", None), + ("Describe this image in one short sentence.", assets["image_thing"]), + (long_prompt(), None), + ): + args: list[object] = ["complete", "--model", model, "--prompt", prompt, "--max-tokens", 16] + if image is not None: + args.extend(["--image", image]) + result = run_runner(runner, *args, timeout=300) + assert_completion(result) + + +@pytest.mark.smoke +def test_parakeet_tdt_file_and_pcm_transcription(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: + model = model_path( + weights_root, + "parakeet-tdt-0.6b-v3-transpiled", + "parakeet-tdt-0.6b-v3", + native_config=True, + ) + file_result = run_runner(runner, "transcribe", "--model", model, "--audio", assets["audio"], timeout=240) + pcm_result = run_runner(runner, "transcribe-pcm", "--model", model, "--audio", assets["audio"], timeout=240) + assert_transcript(file_result) + assert_transcript(pcm_result) + + +@pytest.mark.full +def test_whisper_small_file_and_pcm_transcription_when_available( + runner: Path, + weights_root: Path, + assets: dict[str, Path], +) -> None: + model = model_path(weights_root, "whisper-small", required=False, native_config=True) + file_result = run_runner(runner, "transcribe", "--model", model, "--audio", assets["audio"], timeout=300) + pcm_result = run_runner(runner, "transcribe-pcm", "--model", model, "--audio", assets["audio"], timeout=300) + assert_transcript(file_result) + assert_transcript(pcm_result) diff --git a/local_investigations/gemma4_multimodal/BACKGROUND.md b/local_investigations/gemma4_multimodal/BACKGROUND.md new file mode 100644 index 000000000..8a7cf2031 --- /dev/null +++ b/local_investigations/gemma4_multimodal/BACKGROUND.md @@ -0,0 +1,53 @@ +# Background + +## Local artifact inventory + +Multimodal Gemma candidates: +- `gemma-4-e2b-it` +- `gemma-4-e2b-it-alt-fixed-rowtuned-mm` +- `gemma-4-e2b-it-fresh-mm` + +Text-only/shared-KV Gemma candidates: +- `gemma-4-e2b-it-sharedkv` +- `gemma-4-e2b-it-sharedkv-c128` +- `gemma-4-e2b-it-sharedkv-c256` +- `gemma-4-e2b-it-sharedkv-c512` + +The shared-KV bundles do not include `vision_encoder` or `audio_encoder`, so they are not valid vision/audio quality candidates. + +## Multimodal component shape facts + +Both local multimodal Gemma E2B bundles point at `google/gemma-4-E2B-it` and use the same local HF snapshot. + +The two older multimodal bundles point at missing HF snapshot `b324173c7d5721c2baba7f3b17b3b9b3d34ab1e9`. + +The fresh bundle points at current cached HF snapshot `905e84b50c4d2a365ebde34e685027578e6728db`. + +Component order: +- `audio_encoder` +- `vision_encoder` +- `lm_encoder` +- `decoder_prefill_chunk` +- `lm_encoder_step` +- `lm_encoder_media_step` +- `decoder_step` + +Important logical inputs: +- `vision_encoder`: `pixel_values`, `pixel_position_ids` +- `audio_encoder`: `input_features`, `input_features_mask` +- `lm_encoder`: `input_ids`, `image_features`, `audio_features` +- `lm_encoder_media_step`: `inputs_embeds`, `input_ids`, `position_ids` + +## Controls + +The paired audio created from `../cactus/what_is_in_this_image.mp3` was converted to RIFF/WAV because `cactus_complete` currently rejects MP3 with `Not RIFF`. + +Parakeet transcribes the converted WAV as: + +```text +What is in this image? +``` + +Qwen3-VL identifies `test_monkey.png` as a proboscis monkey, so the asset itself is a good vision control. + +HF reference on the current cached snapshot identifies the same image with the same general prompt as an orangutan. HF also works when using the Cactus-native expanded image prompt/tensors after renaming `pixel_position_ids` to HF's `image_position_ids`. diff --git a/local_investigations/gemma4_multimodal/EXPERIMENTS.md b/local_investigations/gemma4_multimodal/EXPERIMENTS.md new file mode 100644 index 000000000..0e5f36f6d --- /dev/null +++ b/local_investigations/gemma4_multimodal/EXPERIMENTS.md @@ -0,0 +1,80 @@ +# Experiments + +## E1: Paired audio validation + +Purpose: +- Verify that the new audio fixture says what the user intended before using it for Gemma mixed testing. + +Inputs: +- `../cactus/what_is_in_this_image.mp3` +- Converted WAV: `local_integration_tests/.tmp/what_is_in_this_image.wav` +- Parakeet model: `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` + +Observed: + +```text +What is in this image? +``` + +Interpretation: +- The audio fixture is good. +- The native completion path currently requires WAV input, not MP3. + +## E2: Gemma and Qwen image controls + +Purpose: +- Confirm whether the monkey image is a valid vision test and whether the Gemma failure is model/path-specific. + +Observed: +- Qwen3-VL identifies the image as a proboscis monkey. +- Gemma multimodal artifacts fail to identify the monkey in image-only prompts. + +Interpretation: +- The asset is a valid vision control. +- The failure is specific to Gemma artifact/code path behavior, not the image itself. + +## E2b: HF reference with Cactus-native prompt style + +Purpose: +- Check whether Cactus-native expanded media prompts and native image tensors are themselves valid. + +Observed: +- HF current snapshot with normal processor chat template answered: `The animal in this image is an orangutan.` +- HF current snapshot with Cactus-native expanded prompt/tensors also answered: `The animal in this image is an orangutan.` + +Interpretation: +- Prompt style and native image tensor preparation are not the high-level cause. +- The remaining fault is in the transpiled/C++ component execution path or artifact lowering/runtime behavior. + +## E3: Force Gemma media through media-step path + +Purpose: +- Test whether Gemma image-only quality fails because arbitrary prompts are incorrectly routed through the static `lm_encoder` chunk path. + +Change tested: +- Temporarily disabled chunk media prefill for Gemma so `prefill_with_media` used the `lm_encoder_media_step` loop. + +Observed: +- Gemma image-only monkey prompt produced malformed/gibberish output instead of identifying the image. +- Repeating the same temporary routing change with `gemma-4-e2b-it-fresh-mm` also produced malformed/gibberish output. + +Interpretation: +- The simple routing fallback is not valid. +- The investigation should compare actual component contracts and traced prompt assumptions before changing routing further. + +## E4: Fresh reconvert artifact + +Purpose: +- Determine whether stale artifacts caused the Gemma vision failure. + +Artifact: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +Observed: +- Fresh artifact points at current cached snapshot `905e84b50c4d2a365ebde34e685027578e6728db`. +- Image-only monkey prompt produced: `The image shows a close-up of a human nose.` +- Similar prompt produced: `The image shows a close-up of a fawn's nose.` + +Interpretation: +- Reconvert was necessary to remove stale-artifact uncertainty. +- Reconvert alone does not solve Gemma native vision quality. diff --git a/local_investigations/gemma4_multimodal/FIXES.md b/local_investigations/gemma4_multimodal/FIXES.md new file mode 100644 index 000000000..854813740 --- /dev/null +++ b/local_investigations/gemma4_multimodal/FIXES.md @@ -0,0 +1,52 @@ +# Fixes + +## F1: Audio incremental decode prompt replay + +Status: applied before this investigation workspace was created + +Rationale: +- `cactus_complete` passed the entire processed prompt into `decode_with_audio` on each generated token. +- `decode_with_audio` currently delegates to incremental `decode`, so this replayed the prompt into KV cache repeatedly. + +Observed effect: +- Mixed-media generation speed improved. +- Gemma audio semantics remained poor, so this was not the full root cause. + +## F2: Audio encoder mask valid-frame handling + +Status: applied before this investigation workspace was created + +Rationale: +- The audio frontend pads feature buffers to graph capacity. +- The mask should mark only valid frames, not all padded frames. + +Observed effect: +- Build and smoke tests still pass. +- Gemma audio semantics remained poor, so this was not the full root cause. + +## F3: Force Gemma media-step fallback + +Status: reverted + +Rationale: +- Tested whether the static chunk `lm_encoder` path was the immediate cause of Gemma image-only failure. + +Observed effect: +- Output became gibberish. +- The change was reverted. +- The same experiment was repeated with the fresh artifact and was still gibberish. + +## F4: Fresh Gemma4 multimodal reconvert + +Status: created for investigation + +Artifact: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +Rationale: +- Existing multimodal manifests referenced a missing older HF snapshot. +- The current cached HF snapshot produces a valid HF reference answer. + +Observed effect: +- Fresh artifact no longer depends on the missing snapshot. +- Native Cactus vision quality is still wrong, so further runtime/component investigation is required. diff --git a/local_investigations/gemma4_multimodal/HYPOTHESES.md b/local_investigations/gemma4_multimodal/HYPOTHESES.md new file mode 100644 index 000000000..c90541186 --- /dev/null +++ b/local_investigations/gemma4_multimodal/HYPOTHESES.md @@ -0,0 +1,71 @@ +# Hypotheses + +## H1: Current multimodal artifact is stale or malformed + +Status: supported but not sufficient + +Evidence so far: +- Both local multimodal Gemma E2B bundles have full multimodal components. +- Both reference the same HF snapshot. +- Shared-KV artifacts are text-only and should not be used for vision/audio quality. +- The older multimodal bundles reference a missing local HF snapshot. +- A fresh bundle was reconverted from the current cached HF snapshot. +- The fresh bundle improves the failure mode but still does not match HF vision quality. + +Next evidence needed: +- Keep using `gemma-4-e2b-it-fresh-mm` as the best Cactus artifact for further debugging. +- Do not assume reconversion alone fixes the native runtime path. + +## H2: Gemma native image preprocessing diverges from Python/HF preprocessing + +Status: open + +Evidence so far: +- C++ and Python both appear to use Gemma4-style resizing, patching, and position IDs. +- No tensor-level diff has been run yet. + +Next evidence needed: +- Compare `pixel_values` and `pixel_position_ids` between C++ and Python for the same image. + +## H3: Gemma multimodal chunk prefill uses a static traced merge plan that is invalid for arbitrary prompts + +Status: partially supported, simple fallback refuted + +Evidence so far: +- The `lm_encoder` component was traced with a fixed processor prompt embedded in the graph metadata. +- The Python adapter builds a native merge plan during transpile from the example `input_ids`. +- C++ currently sends arbitrary prompt tokens through this `lm_encoder` for media prefill. +- Gemma image-only fails even though Qwen identifies the same image. + +Experiment: +- Route Gemma media prefill through `lm_encoder_media_step` instead of the chunk `lm_encoder` path, then re-run image-only quality probes. + +Result: +- The simple dynamic fallback produced obvious gibberish for image-only monkey prompts. +- The same result reproduced with the fresh artifact. +- This suggests `lm_encoder_media_step` is not a drop-in prefill replacement as currently wired, or it needs additional exact contract handling. +- The static-merge concern remains plausible, but the fix is not simply disabling chunk prefill for all Gemma media. + +## H5: HF/native prompt and preprocessing are sound; the divergence is in transpiled/C++ execution + +Status: supported + +Evidence so far: +- HF with the normal processor chat template identifies the monkey image as an orangutan. +- HF with Cactus-native expanded prompt/tensors also identifies the image as an orangutan when `pixel_position_ids` are passed as HF `image_position_ids`. +- Fresh Cactus native with the same image says the image is a human/fawn nose. + +Next evidence needed: +- Compare component-level outputs or implement/test the intended dynamic Gemma media chunk path using `lm_encoder_text_chunk` and `lm_encoder_media_chunk`. + +## H4: Audio frontend or audio mask is wrong + +Status: open + +Evidence so far: +- The completion decode loop previously replayed the whole prompt for audio generation; this was fixed. +- The audio encoder mask was changed to mark only valid frames. +- Audio-only still does not show evidence of understanding the spoken content. + +Next evidence needed: +- Work on audio only after vision-only is explained or fixed. diff --git a/local_investigations/gemma4_multimodal/PLAN.md b/local_investigations/gemma4_multimodal/PLAN.md new file mode 100644 index 000000000..5fc3bdad7 --- /dev/null +++ b/local_investigations/gemma4_multimodal/PLAN.md @@ -0,0 +1,29 @@ +# Plan + +## Current phase + +Isolate Gemma vision-only correctness before treating mixed image+audio as a product behavior. + +## Method + +Use a hypothesis-driven loop: +1. Record a hypothesis in `HYPOTHESES.md`. +2. Run the smallest experiment that can support or refute it. +3. Record exact output and interpretation in `EXPERIMENTS.md`. +4. If supported, make the smallest fix and document it in `FIXES.md`. +5. Re-run the focused quality probe and the nearest regression suite. + +## Layer order + +1. Artifact validity and freshness. +2. Prompt/token construction. +3. Image preprocessing. +4. Vision encoder output. +5. LM media merge behavior. +6. Decoder/cache behavior. +7. Audio preprocessing and encoder behavior. +8. Mixed image+audio behavior. + +## Current decision + +Investigate whether Gemma multimodal prefill is incorrectly using the static chunk `lm_encoder` path for arbitrary prompts instead of the dynamic `lm_encoder_media_step` path. diff --git a/local_investigations/gemma4_multimodal/README.md b/local_investigations/gemma4_multimodal/README.md new file mode 100644 index 000000000..0cde142cb --- /dev/null +++ b/local_investigations/gemma4_multimodal/README.md @@ -0,0 +1,23 @@ +# Gemma4 multimodal investigation + +Local, uncommitted workspace for investigating Gemma4 native vision/audio quality. + +Current status: +- Vision-only Gemma native does not reliably identify `cactus-engine/tests/assets/test_monkey.png`. +- Audio-only Gemma native does not yet show evidence that it is using the audio content. +- Mixed image+audio with `what_is_in_this_image.wav` currently fails because the underlying unimodal paths are not known-good. +- Qwen3-VL and Parakeet are useful controls: Qwen identifies the monkey, and Parakeet transcribes the paired audio. + +Primary artifacts: +- `../cactus/weights/gemma-4-e2b-it` +- `../cactus/weights/gemma-4-e2b-it-alt-fixed-rowtuned-mm` +- `../cactus/weights/qwen3-vl-2b-instruct-reconvert` +- `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` + +Read in order: +1. `BACKGROUND.md` +2. `PLAN.md` +3. `HYPOTHESES.md` +4. `EXPERIMENTS.md` +5. `FIXES.md` +6. `SUMMARY.md` diff --git a/local_investigations/gemma4_multimodal/SUMMARY.md b/local_investigations/gemma4_multimodal/SUMMARY.md new file mode 100644 index 000000000..e39aa24e0 --- /dev/null +++ b/local_investigations/gemma4_multimodal/SUMMARY.md @@ -0,0 +1,9 @@ +# Summary + +Investigation is in progress. + +Current leading hypothesis: +- Gemma media prefill has a component-contract mismatch or lowering/runtime mismatch after HF-native prompt/preprocessing, likely around transpiled media encoder/LM merge execution. + +Current next step: +- Compare component-level outputs or implement/test the intended dynamic Gemma media chunk path using `lm_encoder_text_chunk` and `lm_encoder_media_chunk`. diff --git a/local_investigations/gemma4_multimodal_v2/00_INDEX.md b/local_investigations/gemma4_multimodal_v2/00_INDEX.md new file mode 100644 index 000000000..71ed21eb7 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/00_INDEX.md @@ -0,0 +1,48 @@ +# Gemma4 Multimodal V2 Index + +Status: Gemma4 image-only, audio-only, and mixed image+audio now pass local end-to-end probes after the dynamic audio merge fix; Qwen, Parakeet, policy, and smoke controls pass + +Active phase: post-fix review / audit. Do not commit or push until the user reviews. + +Default artifact under investigation: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +Key controls: +- HF reference snapshot: `905e84b50c4d2a365ebde34e685027578e6728db` +- Qwen image control: `../cactus/weights/qwen3-vl-2b-instruct-reconvert` +- Parakeet audio control: `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` + +Read order: +1. `09_STATUS_FOR_NEXT_AGENT.md` +2. `10_CONTINUATION_PLAN.md` +3. `01_BACKGROUND.md` +4. `02_QUESTIONS.md` +5. `03_HYPOTHESES.md` +6. `04_EXPERIMENTS.md` +7. `05_DECISIONS.md` +8. `06_FIX_LOG.md` +9. `07_ACCEPTANCE.md` +10. `08_HANDOFF.md` + +Latest proven artifacts: + +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json`: image preprocessing parity recorded as `status: pass`, max abs diff `4.987010955859184e-07`. +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json`: latest recorded vision encoder output is FP16-close to the Python/HF reference for `pooler_output`, cosine `0.9999955754245847`, mean abs diff `0.0015996834738693023`. +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json`: image-only Cactus Gemma answered `The animal in the image is a golden monkey.` +- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json`: after the Gemma4 audio bias FP16 fix, first 64 audio rows are FP16-close to Python, cosine `0.9978210926`, mean abs diff `0.00645508`. +- `artifacts/E029_first_token_after_dynamic_merge/`: dynamic merge first-token probes now produce normal text starts, audio `Please`, mixed `This`. +- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json`: audio-only Cactus Gemma responds coherently: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json`: mixed Cactus Gemma answers: `This image is of a golden monkey.` +- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json`: image-only still answers: `The animal in the image is a golden monkey.` +- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json`: Qwen image control identifies a proboscis monkey. +- `artifacts/E034_parakeet_regression_after_dynamic_merge/`: Parakeet file and PCM controls both transcribe `What is in this image?` +- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json`: full emitted audio tensor matches the E023 post-bias tensor exactly, max diff `0`. + +Latest test commands: +- `source ../cactus/venv/bin/activate && ./cactus/build.sh` +- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q` -> `16 passed` +- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s` -> `7 passed, 2 deselected` + +Continuation contract: +- Follow `10_CONTINUATION_PLAN.md`. +- The acceptance probes now pass locally. Next work is review, cleanup, and any additional audit the user requests. diff --git a/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md b/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md new file mode 100644 index 000000000..a4336c5f5 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md @@ -0,0 +1,48 @@ +# Background + +## Stable facts copied from V1 + +The older local Gemma multimodal bundles are: +- `../cactus/weights/gemma-4-e2b-it` +- `../cactus/weights/gemma-4-e2b-it-alt-fixed-rowtuned-mm` + +Both older bundles reference missing HF snapshot: +- `b324173c7d5721c2baba7f3b17b3b9b3d34ab1e9` + +The fresh investigation bundle is: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +The fresh bundle references current cached HF snapshot: +- `905e84b50c4d2a365ebde34e685027578e6728db` + +The shared-KV Gemma artifacts are text-only for this investigation because they do not include `vision_encoder` or `audio_encoder`. + +## Confirmed prior observations + +- HF reference identifies `test_monkey.png` as an orangutan/monkey-class animal. +- HF still works when using Cactus-native expanded image prompt/tensors with `pixel_position_ids` passed as HF `image_position_ids`. +- Fresh Cactus native Gemma originally failed image quality in E001, describing the monkey image as a human/fawn nose. +- Later current artifacts record image preprocessing parity and image-only generation success: + - `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` + - `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` +- Later artifacts record audio-only and mixed image+audio passing after the audio bias FP16 fix and dynamic Gemma4 audio merge: + - `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` + - `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` +- Qwen3-VL identifies the same image as a proboscis monkey. +- Parakeet transcribes the paired audio as: `What is in this image?` +- Direct MP3 input to native completion fails with `Not RIFF`; the paired audio must be converted to WAV. + +## Local paths + +Image: +- `cactus-engine/tests/assets/test_monkey.png` + +Audio: +- Source MP3: `../cactus/what_is_in_this_image.mp3` +- Converted WAV when needed: `local_integration_tests/.tmp/what_is_in_this_image.wav` + +Harness: +- `local_investigations/gemma4_multimodal_v2/scripts/run_probe.py` + +Artifacts: +- `local_investigations/gemma4_multimodal_v2/artifacts/` diff --git a/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md b/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md new file mode 100644 index 000000000..46965b759 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md @@ -0,0 +1,41 @@ +# Questions + +## Q1: Where is the first bad boundary? + +Status: answered for vision and audio in the current artifact set + +Candidate boundaries: +- Prompt/token IDs and media spans. +- Image preprocessing tensors. +- Vision encoder output. +- LM media merge output. +- Decoder first-token logits. +- Autoregressive decode loop. + +Resolution: +- For vision, E002/E003/E014/E015 have current artifacts and image-only is recorded as passing. +- For audio, E022/E023 localized and fixed the audio encoder bias issue, and E027/E029 localized and fixed the static media merge issue. + +## Q2: Is the fresh Gemma artifact good enough to be the default investigation artifact? + +Status: yes for current investigation + +Known evidence: +- It points at the current cached HF snapshot. +- E001 confirmed HF on the same snapshot worked while Cactus image-only originally did not. +- Later artifacts record Cactus image-only success on the fresh artifact. +- E030/E031 record audio-only and mixed image+audio passing on the fresh artifact. + +Next falsifier: +- If a fresh rerun contradicts E030/E031, inspect the changed artifact/runtime state before changing prompts. + +## Q3: Is the previous media-step fallback worth pursuing? + +Status: no as a wholesale fallback; dynamic media-step use inside chunked prefill is accepted. + +Known evidence: +- Forcing all Gemma media through the old step fallback produced gibberish on mixed prompts. +- The accepted fix uses `lm_encoder_media_step` only to assemble real prompt-position outputs before chunked decoder prefill. + +Next falsifier: +- Only revisit after proving the correct `lm_encoder_media_chunk`/step contract and how cache copies should work. diff --git a/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md b/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md new file mode 100644 index 000000000..46eacf99b --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md @@ -0,0 +1,66 @@ +# Hypotheses + +## H1: Artifact staleness caused the original failure + +Status: supported but not sufficient; not the active boundary + +Evidence: +- Older multimodal artifacts reference a missing HF snapshot. +- Fresh artifact was reconverted from current cached snapshot. +- Fresh artifact originally failed native Gemma vision quality in E001. +- Later artifacts show image-only now passes after targeted fixes. + +Decision: +- Use the fresh artifact for V2 unless a stronger artifact is produced for a specific experiment. + +## H2: Prompt style or native image preprocessing is invalid + +Status: refuted for current image-only path + +Evidence: +- HF works with the normal processor prompt. +- HF also works with Cactus-native expanded prompt/tensors. +- E001 stores both records under `artifacts/E001/`. +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records image preprocessing parity as `status: pass`. +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records image-only output `The animal in the image is a golden monkey.` + +Resolution: +- Audio and mixed image+audio now pass in E030/E031; do not reopen image preprocessing without contradictory fresh evidence. + +## H3: The transpiled/C++ component path diverges after preprocessing + +Status: historically true for vision and audio; currently fixed in local artifacts + +Evidence: +- Vision had later graph/runtime/conversion fixes recorded in `06_FIX_LOG.md`. +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` records FP16-close vision output agreement. +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` records a substantial emitted `audio_features` mismatch against the Python native-like reference. +- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` records audio features as FP16-close after the audio bias fix. + +Next test: +- Rerun E030/E031 if the artifact or runtime changes. + +## H4: The C++ media merge/cache path is using the wrong component contract + +Status: supported and fixed for Gemma4 prompts containing audio + +Evidence: +- Simple `lm_encoder_media_step` fallback produced gibberish. +- Fresh artifact includes `lm_encoder_text_chunk`, `lm_encoder_media_step`, and `lm_encoder_media_chunk`, but C++ currently uses the monolithic `lm_encoder` chunk path for media. +- E025/E026 previously returned empty responses after the audio encoder fix, which made media merge/cache the next boundary. +- After the audio encoder fix, E027 showed the static `lm_encoder` chunk path selected `` for audio/mixed first tokens. +- E029/E030/E031 show dynamic prompt-position media merge fixes first-token and generation behavior. + +Next test: +- Keep dynamic merge active for Gemma4 prompts with audio and rerun regressions after runtime changes. + +## H5: Audio should remain blocked until vision is explained + +Status: retired + +Evidence: +- Vision-only has a recorded passing artifact in `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json`. +- Audio-only and mixed image+audio pass in E030/E031. + +Next test: +- None unless a fresh rerun contradicts the passing artifacts. diff --git a/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md b/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md new file mode 100644 index 000000000..37ef0129d --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md @@ -0,0 +1,356 @@ +# Experiments + +## Experiment format + +Each experiment must include: +- ID +- Question +- Setup +- Command +- Raw result path +- Interpretation +- Next decision + +## E001: Baseline ledger + +Question: +- Can the harness reproduce the known HF/Cactus/control outputs and store them in a comparable format? + +Setup: +- Fresh Gemma artifact: `../cactus/weights/gemma-4-e2b-it-fresh-mm` +- Image: `cactus-engine/tests/assets/test_monkey.png` +- Audio: converted `what_is_in_this_image.wav` + +Command: + +```bash +python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case all-baseline --experiment E001 +``` + +Raw result path: +- `artifacts/E001/` + +Interpretation: +- Completed. +- HF Gemma normal prompt: `The animal in this image is an orangutan.` +- HF Gemma Cactus-native expanded prompt/tensors: `The animal in this image is an orangutan.` +- Cactus fresh Gemma native: `The image shows a close-up of a human nose.` +- Cactus Qwen control identifies the proboscis monkey. +- Parakeet file and PCM controls both transcribe: `What is in this image?` + +Next decision: +- Move to E002 component-boundary comparison. +- The next probe must localize the first bad boundary after HF-native prompt/preprocessing, not attempt another runtime fix. + +## E002: Component-boundary comparison + +Question: +- At which component boundary does fresh Cactus native first diverge from the HF/Python reference? + +Setup: +- Same artifact, prompt, and image as E001. +- Compare in this order: input IDs/media spans, image tensors, vision encoder output, LM merge output, decoder first-token logits. + +Command: +```bash +source ../cactus/venv/bin/activate +./cactus/build.sh +python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case e002-boundary --experiment E002 +``` + +Raw result path: +- `artifacts/E002/` +- `artifacts/E002/token_spans.json` +- `artifacts/E002/image_preprocess_compare.json` +- `artifacts/E002/hf_native_inputs.json` +- `artifacts/E002/cactus_token_probe.json` +- `artifacts/E002/cactus_preprocess.json` + +Interpretation: +- Completed 2026-05-20. +- Gate A passed: HF/Python native and C++ Cactus native `input_ids` match exactly. +- Gate A image span also matches exactly: token count `282`, assistant generation start `279`, image token id `258880`, image start `5`, image token count `256`. +- Gate B failed at `pixel_values`. +- `pixel_values` shape matches: `[1, 2520, 768]`. +- `pixel_position_ids` shape and values match, with valid position count `2304`. +- `pixel_values` differ immediately at element `0`; max absolute difference is `0.1298471341896057`, mean absolute difference is `0.002023158910580417`. +- HF/Python pixel range is `[0.0, 1.0]`; C++ pixel range is `[-0.0213023126, 1.00900519]`. +- The first divergent boundary is Gemma4 native image preprocessing, before the vision encoder. + +Next decision: +- Stop E002 at Gate B for now, then continue with E003. Do not run vision encoder, LM merge, or decoder comparison until preprocessing parity is fixed or a stricter preprocessing reference proves the current C++ tensor is acceptable. +- The next fix should be scoped to `preprocess_gemma4_image` parity with `_prepare_gemma4_native_image_tensors`, including resize/interpolation and numeric range behavior. +- This is not final acceptance; the mission continues until Gemma image-only, audio-only, and mixed image+audio pass end to end. + +Superseded by later artifacts below: +- E003 / `E002_after_E003_pillow_resize` records preprocessing parity as passing. +- E015 records image-only generation as passing. +- Audio-only and mixed image+audio are now the active failures. + +## Current status correction: later artifacts after E002 + +This section records the newer artifacts currently present in the worktree. It does not erase E001/E002 history; it updates the active boundary for future agents. + +## E003 / E002_after_E003_pillow_resize: Image preprocessing parity + +Question: +- Does the C++ Gemma4 image preprocessing output match the Python/HF native reference after the Pillow-compatible resize path? + +Raw result paths: +- `artifacts/E003/preprocess_operation_compare.json` +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` + +Interpretation: +- Completed in current artifacts. +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. +- `pixel_values` max abs diff is `4.987010955859184e-07`. +- `pixel_values` mean abs diff is `2.2151478535820782e-07`. +- HF/Python range is `[0.0, 1.0]`; C++ range is `[0, 1]`. + +Next decision: +- Treat image preprocessing as fixed unless a fresh rerun contradicts this artifact. +- Continue to the vision encoder boundary. + +## E014: Latest recorded vision encoder comparison + +Question: +- Does the Cactus `vision_encoder` output match the Python/HF reference closely enough after the recorded graph/runtime/conversion fixes? + +Raw result path: +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` + +Interpretation: +- The latest recorded `pooler_output` comparison has matching shape `[256, 1536]`. +- Cosine similarity is `0.9999955754245847`. +- Max abs diff is `0.03515437499999985`. +- Mean abs diff is `0.0015996834738693023`. +- This is recorded as FP16-close. The artifact does not by itself prove every later LM merge/logit boundary, so do not overstate it as full multimodal acceptance. + +Next decision: +- Use image-only generation quality as the next acceptance check for the vision path. + +## E015: Gemma image-only quality check after vision fixes + +Question: +- Does fresh Cactus Gemma image-only generation identify `test_monkey.png` as a monkey-class animal? + +Raw result path: +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` + +Interpretation: +- Cactus returned success. +- Response: `The animal in the image is a golden monkey.` +- `prefill_tokens: 282`. +- `decode_tokens: 10`. +- This satisfies the image-only semantic acceptance class recorded in `07_ACCEPTANCE.md`. + +Next decision: +- Treat Gemma image-only as passing in the current artifact set. +- Continue to Gemma audio-only and mixed image+audio. + +## E016 / E020: Gemma audio-only quality checks + +Question: +- Does fresh Cactus Gemma audio-only generation produce coherent text using the paired audio prompt? + +Raw result paths: +- `artifacts/E016_gemma_audio_after_vision_gate/cactus_gemma_audio.json` +- `artifacts/E020_gemma_audio_after_audio_retranspile/cactus_gemma_audio.json` + +Interpretation: +- Both recorded Cactus runs returned success but no generated text. +- E020 response is empty: `response: ""`. +- E020 has `prefill_tokens: 83` and `decode_tokens: 0`. +- Retranspiling the audio encoder in the recorded E019/E020 sequence did not resolve audio-only generation. + +Next decision: +- Audio-only remains an active failure. +- Continue with audio boundary comparison before changing prompts or decoder settings. + +## E017: Gemma mixed image+audio quality check + +Question: +- Does fresh Cactus Gemma mixed image+audio generation answer the spoken question about `test_monkey.png`? + +Raw result path: +- `artifacts/E017_gemma_mixed_after_vision_gate/cactus_gemma_mixed.json` + +Interpretation: +- Cactus returned success but no generated text. +- Response is empty: `response: ""`. +- `prefill_tokens: 349`. +- `decode_tokens: 0`. + +Next decision: +- Mixed image+audio remains an active failure. +- Do not treat image-only success as full acceptance. + +## E018 / E019: Audio summary before and after audio retranspile + +Question: +- Do the recorded Cactus audio preprocessing and emitted audio feature summaries change after retranspiling the audio encoder? + +Raw result paths: +- `artifacts/E018_audio_summary/audio_summary.json` +- `artifacts/E019_audio_summary_after_retranspile/audio_summary.json` + +Interpretation: +- Both artifacts record `num_frames: 255`, `num_soft_tokens: 64`, and `feature_count: 32640`. +- Both artifacts record emitted `audio_features` shape `[1, 753, 1536]` with precision `FP16`. +- The first emitted values are unchanged across E018 and E019. +- These summaries are not enough to prove audio preprocessing parity because they only include prefixes and aggregate statistics. + +Next decision: +- Use full-tensor comparison before selecting an audio fix. + +## E021: Full Cactus-vs-Python audio feature comparison + +Question: +- Do the Cactus emitted `audio_features` match the Python native-like audio feature reference? + +Raw result paths: +- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` + +Interpretation: +- `cactus_audio_summary.json` now includes full `audio_features.values`. +- Cactus and Python output shapes match: `[1, 753, 1536]`. +- Python prepared feature shape is `[1, 3012, 128]`; Python input feature mask sum is `255`. +- For the first 64 rows, cosine similarity is `0.36957991123199463`, max abs diff is `2.796870708465576`, and mean abs diff is `0.16708998382091522`. +- For all 753 rows, cosine similarity is `0.7203282713890076`, max abs diff is `2.796870708465576`, and mean abs diff is `0.10658295452594757`. +- The `python_audio_token_count` field in this artifact is not used for conclusions because the comparison script counted a token id that is not validated here. + +Next decision: +- The current best proven active boundary is the audio encoder output / emitted audio feature boundary. +- Next work should localize the first divergent internal audio encoder operation or weight. Do not infer a specific internal cause from E021 alone. + +## E022 / E023 / E024: Audio encoder localization and FP16 bias fix + +Question: +- What caused the emitted Gemma4 `audio_features` mismatch from E021? + +Raw result paths: +- `artifacts/E022_audio_node_localization/` +- `artifacts/E023_audio_after_fp16_bias/` +- `artifacts/E024_retranspile_audio_fp16_bias/` + +Interpretation: +- E022 localized the first important divergence to the audio output projection bias handling. +- The active artifact had stored `audio_output_proj.bias` as an INT8 fallback, which made Cactus add raw values like `[1, -2, 0, 70]` instead of HF FP16-like bias values. +- After changing Gemma4 audio bias policy to FP16 and updating the active artifact, E023 records first 64 audio rows as FP16-close to Python: cosine `0.9978210926`, mean abs diff `0.00645508`. + +Next decision: +- Keep Gemma4 audio bias tensors as FP16. +- Continue to generation and first-token probes because audio-only and mixed still returned empty strings after E023. + +## E025 / E026 / E027: Post-audio-encoder generation and first-token failure + +Question: +- After fixing the audio encoder boundary, why do audio-only and mixed generation still fail? + +Raw result paths: +- `artifacts/E025_gemma_audio_after_fp16_bias/cactus_gemma_audio.json` +- `artifacts/E026_gemma_mixed_after_fp16_bias/cactus_gemma_mixed.json` +- `artifacts/E027_first_token_after_audio_encoder_fix/` + +Interpretation: +- Audio-only and mixed still returned empty responses after the audio encoder fix. +- First-token probes showed the static chunk path selected `` for both audio-only and mixed prompts, unlike HF references (`The` / `This` starts). +- Forcing the old media-step path improved audio-only first token to `Please`, but mixed produced whitespace, so wholesale fallback was not acceptable. +- The static `lm_encoder` graph was captured with stale media span assumptions and could insert audio at the wrong positions/counts for the current prompt. + +Next decision: +- Use a Gemma4-audio-only dynamic media merge inside the chunked prefill path instead of relying on stale static `lm_encoder` merge captures. + +## E029: First-token probes after dynamic merge + +Question: +- Does the dynamic Gemma4 audio merge fix first-token behavior for audio-only and mixed prompts? + +Raw result path: +- `artifacts/E029_first_token_after_dynamic_merge/` + +Interpretation: +- Audio-only first token is `Please`, with normal text alternatives in top logits. +- Mixed image+audio first token is `This`, matching the HF-style first-token direction. +- Neither probe selects ``, ``, or whitespace-only as the top token. + +Next decision: +- Continue to full generation quality checks. + +## E030: Gemma audio-only after dynamic merge + +Question: +- Does audio-only Gemma use the paired spoken prompt coherently? + +Raw result path: +- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` + +Interpretation: +- Cactus generated: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- This is coherent with the audio prompt saying `what is in this image` and no image being supplied. +- `decode_tokens: 21`. + +Next decision: +- Audio-only quality gate passes. Continue to mixed image+audio. + +## E031: Gemma mixed image+audio after dynamic merge + +Question: +- Does mixed Gemma answer the spoken question about `test_monkey.png`? + +Raw result path: +- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` + +Interpretation: +- Cactus generated: `This image is of a golden monkey.` +- This satisfies the mixed acceptance class. +- `decode_tokens: 8`. + +Next decision: +- Mixed gate passes. Run regressions. + +## E032 / E033 / E034 / E035: Regression checks after dynamic merge + +Question: +- Did the dynamic merge fix regress image-only Gemma, Qwen, Parakeet, or the audio encoder boundary? + +Raw result paths: +- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` +- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` +- `artifacts/E034_parakeet_regression_after_dynamic_merge/` +- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` + +Interpretation: +- Gemma image-only still generated: `The animal in the image is a golden monkey.` +- Qwen image control generated a proboscis-monkey answer. +- Parakeet file and PCM controls both transcribed: `What is in this image?` +- E035 shows the emitted Cactus audio tensor is unchanged from E023, with full-tensor max diff `0`. +- Policy tests passed: `16 passed`. +- Smoke tests passed: `7 passed, 2 deselected`. + +Next decision: +- The planned local acceptance gates pass. Keep the working tree uncommitted for user audit. + +## E036 / E037 / E038 / E039 / E040: Cleanup verification probes + +Question: +- After ignoring/removing generated local outputs, do the required quality probes still regenerate needed files and pass? + +Raw result paths: +- `artifacts/E036_cleanup_gemma_image/cactus_gemma_image.json` +- `artifacts/E037_cleanup_gemma_audio/cactus_gemma_audio.json` +- `artifacts/E038_cleanup_gemma_mixed/cactus_gemma_mixed.json` +- `artifacts/E039_cleanup_qwen_image/cactus_qwen_image.json` +- `artifacts/E040_cleanup_parakeet/` + +Interpretation: +- Gemma image-only still generated: `The animal in the image is a golden monkey.` +- Gemma audio-only still generated: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- Gemma mixed still generated: `This image is of a golden monkey.` +- Qwen image still generated a proboscis-monkey answer. +- Parakeet file and PCM still transcribed: `What is in this image?` +- The ignored `.tmp` directory was recreated successfully by the harness. + +Next decision: +- Cleanup did not regress the acceptance probes or remove benchmark/test harness source. diff --git a/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md b/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md new file mode 100644 index 000000000..922eef32b --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md @@ -0,0 +1,118 @@ +# Decisions + +## D1: Do not use older multimodal Gemma artifacts as the default + +Decision: +- Use `gemma-4-e2b-it-fresh-mm` for V2 baseline work. + +Reason: +- Older artifacts reference a missing HF snapshot. + +## D2: Do not continue the simple media-step fallback + +Decision: +- Do not disable Gemma chunk prefill wholesale to force `lm_encoder_media_step`. + +Reason: +- It produced gibberish on old and fresh artifacts. + +## D3: Treat quality probes as acceptance, not root-cause tools + +Decision: +- Use quality probes to confirm behavior, but debug via component boundary comparison. + +Reason: +- Human text output identified the issue but did not localize the first bad boundary. + +## D4: First bad boundary is Gemma4 image preprocessing + +Decision: +- Treat E002 Gate B as the first proven divergence. +- Do not debug vision encoder, LM merge, or decoder logits until Gemma4 native preprocessing matches the HF/Python reference or a stronger reference refutes the HF/Python preprocessing tensor as the correct target. + +Reason: +- E002 Gate A proves prompt `input_ids` and image token spans match exactly between HF/Python native and C++ Cactus native. +- E002 Gate B proves `pixel_position_ids` match, but `pixel_values` differ immediately and outside tiny tolerance. +- The mismatch exists before any graph runtime, media merge, cache, or decoder behavior is involved. + +## D5: Localization is not acceptance + +Decision: +- Boundary localization is now treated only as a checkpoint. +- The investigation remains active until Gemma4 native works end to end across image-only, audio-only, and mixed image+audio, with controls and smoke still passing. + +Reason: +- The user explicitly raised the acceptance bar from identifying the cause to making the model fully work end to end. + +## D6: Image preprocessing is no longer the active boundary + +Decision: +- Treat Gemma4 image preprocessing parity as fixed in the current artifact set. + +Reason: +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. +- The recorded `pixel_values` max abs diff is `4.987010955859184e-07`, and mean abs diff is `2.2151478535820782e-07`. + +## D7: Image-only generation is currently accepted, but full multimodal is not + +Decision: +- Treat Gemma image-only as passing in the current artifact set. +- Do not treat the investigation as complete. + +Reason: +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records the response `The animal in the image is a golden monkey.` +- `07_ACCEPTANCE.md` allows monkey/orangutan/proboscis monkey class answers for image-only. +- Audio-only and mixed image+audio still return empty responses in the latest recorded artifacts. + +## D8: Superseded historical audio / mixed failure + +Decision: +- This was the correct decision at E021, but it is superseded by D10 and D11 after E023/E029/E030/E031. +- Do not return to prompt tweaks or decoder parameters unless a fresh rerun contradicts the passing artifacts. + +Reason: +- `artifacts/E020_gemma_audio_after_audio_retranspile/cactus_gemma_audio.json` records `response: ""` and `decode_tokens: 0`. +- `artifacts/E017_gemma_mixed_after_vision_gate/cactus_gemma_mixed.json` records `response: ""` and `decode_tokens: 0`. +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` records matching Cactus/Python audio feature output shapes but poor first-64-row agreement: cosine `0.36957991123199463`, mean abs diff `0.16708998382091522`. + +## D9: E021 localizes a boundary, not an internal cause + +Decision: +- Use E021 only to justify investigating inside the audio encoder path. +- Do not claim a specific audio op, weight, runtime kernel, or conversion policy is wrong until a smaller comparison identifies it. + +Reason: +- E021 compares full emitted audio features against a Python native-like reference and proves a mismatch at that boundary. +- It does not identify the first divergent node inside the audio encoder. + +## D10: Gemma4 audio bias tensors must stay FP16 + +Decision: +- Keep Gemma4 audio component bias tensors as FP16 fallback tensors. + +Reason: +- E022 localized the audio encoder divergence to the output projection bias handling. +- E023 shows the FP16 bias policy brings the emitted audio features close to the Python reference. +- Storing these biases as INT8 caused Cactus to add raw integer-like values instead of the expected FP16 bias values. + +## D11: Use dynamic Gemma4 merge for prompts with audio + +Decision: +- For Gemma4 prompts with audio, do not use the static full `lm_encoder` merge graph. +- Dynamically walk the actual prompt tokens, route text through `lm_encoder_step`, route image/audio token rows through `lm_encoder_media_step`, then feed the assembled tensors to `decoder_prefill_chunk`. + +Reason: +- E027 showed the static chunk path selected `` for audio-only and mixed prompts after the audio encoder boundary was fixed. +- The static `lm_encoder` graph can encode stale media span assumptions from capture time. +- E029/E030/E031 show the dynamic merge produces normal first tokens and passing end-to-end audio/mixed outputs. + +## D12: Do not carry vision changes from `justin/v2-cpp-fix` + +Decision: +- Treat `origin/justin/v2-cpp-fix` as already absorbed baseline infrastructure. +- Carry over only the later dynamic Gemma4 audio merge idea from `77f808ed`, adapted to the current branch structure. + +Reason: +- The latest `justin/v2-cpp-fix` commit is already in this branch line. +- Vision already has passing local artifacts. +- The relevant audio fix pattern is the dynamic media walk, not a vision-path change. diff --git a/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md b/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md new file mode 100644 index 000000000..5f2ec53d7 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md @@ -0,0 +1,156 @@ +# Fix Log + +Current status: V2 has recorded fixes in the worktree. Do not use the old `No V2 fixes yet` state. + +Rules: +- A code change must cite a supported hypothesis and experiment ID. +- A failed fix must be reverted or explicitly documented as intentionally retained. +- Each fix record must include before/after quality and nearest regression result. + +Historical fixes from before V2: +- Audio incremental decode changed to feed only the last token. +- Audio encoder mask changed to mark valid frames only. +- Fresh Gemma artifact reconverted as `gemma-4-e2b-it-fresh-mm`. +- Simple media-step fallback was tested and reverted. + +## Local investigation tooling + +2026-05-20: +- Added local-only E002 probe commands to `local_integration_tests/native_runner.cpp`: + - `e002-tokens` + - `e002-preprocess` +- Updated `local_integration_tests/CMakeLists.txt` so the local runner can include internal engine headers. +- Extended `scripts/run_probe.py` with `--case e002-boundary`. +- These are investigation artifacts only, not runtime fixes. + +## F1: Gemma4 image preprocessing parity + +Supported by: +- `artifacts/E003/preprocess_operation_compare.json` +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` + +Change: +- `cactus-engine/src/engine_image.cpp` now has a Pillow-compatible uint8 bilinear resize path used by `preprocess_gemma4_image`. + +Before: +- `artifacts/E002/image_preprocess_compare.json` recorded `pixel_values` max abs diff `0.1298471341896057` and mean abs diff `0.002023158910580417`. +- C++ pixel range extended outside `[0, 1]`. + +After: +- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. +- `pixel_values` max abs diff is `4.987010955859184e-07`. +- `pixel_values` mean abs diff is `2.2151478535820782e-07`. + +## F2: Vision graph/runtime/transpile fixes recorded by later artifacts + +Supported by: +- `artifacts/E006_gateC_after_fp16_embedding_indices/vision_encoder_compare.json` +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` + +Changes visible in the worktree: +- `cactus-graph/src/ops_tensor.cpp` handles FP16 embedding indices in `compute_embedding_node`. +- `cactus-graph/src/ops_math.cpp` adds FP32 binary and reduction support used by scalar/math lowering paths. +- `python/cactus/transpile/lower.py` preserves tensor dtype in `_ensure_scalar_math_tensor`. +- `python/cactus/transpile/runtime_compat.py` preserves tensor dtype for scalar ops, `abs`, and `pow`. +- `python/cactus/transpile/model_adapters.py` updates Gemma4 native-like vision pooling/projection behavior. + +Evidence: +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` records FP16-close `pooler_output` agreement: cosine `0.9999955754245847`, mean abs diff `0.0015996834738693023`. +- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records image-only output: `The animal in the image is a golden monkey.` + +Limit: +- This does not prove audio-only or mixed image+audio acceptance. + +## F3: Gemma4 vision projection conversion policy and active artifact adjustment + +Supported by: +- `artifacts/E013_gateC_after_fp16_projection_graph/vision_encoder_compare.json` +- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` + +Changes visible in the worktree: +- `python/cactus/convert/model_adapters/policy.py` makes `model.embed_vision.embedding_projection.weight` an FP16 fallback with reason `vision embedding projection scale-sensitive`. +- `python/cactus/convert/model_adapters/adapters.py` no longer overrides that policy back to CQ. +- `python/cactus/convert/model_adapters/naming.py` and `python/cactus/convert/cactus_adapters/tensor_io.py` remove `embed_vision_proj` from Gemma4 divide-scale handling. +- `python/cactus/convert/tests/test_policy.py` now expects FP16 for the Gemma4 vision projection policy. + +Evidence: +- The latest recorded vision comparison improves to FP16-close agreement in `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json`. + +## F4: Audio full-tensor diagnostic instrumentation + +Supported by: +- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` + +Change: +- `local_integration_tests/native_runner.cpp` now emits full `audio_features.values` for `e002-audio-summary`. + +Evidence: +- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` contains full emitted `audio_features.values`. +- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` compares those values against Python native-like audio features. + +Limit: +- This is diagnostic instrumentation only. +- It proves the emitted audio feature boundary mismatches, but not which internal audio encoder operation or weight is the first cause. + +## F5: Gemma4 audio bias FP16 policy and artifact adjustment + +Supported by: +- `artifacts/E022_audio_node_localization/` +- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` +- `artifacts/E024_retranspile_audio_fp16_bias/` + +Changes: +- `python/cactus/convert/model_adapters/policy.py` keeps Gemma4 audio component bias tensors in FP16. +- `python/cactus/convert/tests/test_policy.py` expects FP16 for Gemma4 audio bias policy. +- The active fresh Gemma artifact has an FP16 `audio_output_proj.bias` and regenerated `audio_encoder` graph artifacts. + +Before: +- E021 first 64 emitted audio rows had cosine `0.3695799112` and mean abs diff `0.16708998` versus Python. + +After: +- E023 first 64 emitted audio rows have cosine `0.9978210926` and mean abs diff `0.00645508` versus Python. + +## F6: Dynamic Gemma4 audio/media merge for chunked prefill + +Supported by: +- `artifacts/E027_first_token_after_audio_encoder_fix/` +- `artifacts/E029_first_token_after_dynamic_merge/` +- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` +- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` + +Changes: +- `cactus-engine/src/model.cpp` now builds Gemma4 LM encoder outputs dynamically for prompts containing audio. +- The dynamic path walks actual prompt tokens, uses `lm_encoder_step` for text, uses `lm_encoder_media_step` for image/audio token rows, and feeds the assembled outputs into `decoder_prefill_chunk`. +- `cactus-engine/src/engine.h` declares the dynamic helper. +- The existing static `lm_encoder` merge path remains for non-audio cases. + +Before: +- E025/E026 still returned empty audio-only and mixed responses after the audio encoder fix. +- E027 first-token probes selected `` for both audio-only and mixed prompts on the static chunk path. + +After: +- E029 first-token probes produce audio `Please` and mixed `This`. +- E030 audio-only produces coherent text responding to the spoken image question. +- E031 mixed image+audio produces `This image is of a golden monkey.` +- E032/E033/E034/E035 and smoke/policy tests record no local regression. + +## F7: Cleanup of generated local outputs + +Supported by: +- `artifacts/E036_cleanup_gemma_image/cactus_gemma_image.json` +- `artifacts/E037_cleanup_gemma_audio/cactus_gemma_audio.json` +- `artifacts/E038_cleanup_gemma_mixed/cactus_gemma_mixed.json` +- `artifacts/E039_cleanup_qwen_image/cactus_qwen_image.json` +- `artifacts/E040_cleanup_parakeet/` + +Changes: +- `.gitignore` now ignores local integration `.tmp` output, local Python bytecode, and local investigation artifact directories. +- Removed rebuildable local `.tmp` and `__pycache__` directories. +- Benchmark source/results under `benchmarks/` were not changed. + +Evidence: +- Policy tests passed after cleanup: `16 passed`. +- Smoke tests passed after cleanup: `7 passed, 2 deselected`. +- E036-E040 quality probes passed after cleanup. diff --git a/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md b/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md new file mode 100644 index 000000000..3e6887129 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md @@ -0,0 +1,50 @@ +# Acceptance + +## End-to-end gate + +Gemma4 native multimodal work is accepted only when the full user path works: +- Image-only Gemma correctly identifies `test_monkey.png` as a monkey/orangutan/proboscis monkey class animal. +- Audio-only Gemma shows it uses the paired spoken prompt coherently. +- Mixed image+audio Gemma answers the spoken question about `test_monkey.png` correctly. +- Any component boundary divergence found during debugging has a before/after artifact showing the fix. +- Qwen image control still works. +- Parakeet file and PCM controls still transcribe coherently. +- Smoke tests pass. + +Localization is not acceptance. It is a checkpoint that determines the next fix. + +## Vision gate + +Gemma native vision is accepted only when: +- HF and Cactus use the same image and semantically equivalent prompt. +- The first bad component boundary has been identified and fixed or ruled out. +- Cactus output identifies the monkey image as a monkey/orangutan/proboscis monkey class animal. +- Qwen image control still works. + +## Audio gate + +Gemma audio work starts after vision has a known-good path. A documented vision boundary failure is not enough to accept the mission. + +Audio is accepted only when: +- The paired WAV is verified by Parakeet. +- Gemma audio preprocessing and audio encoder behavior are compared against the Python/HF reference. +- Cactus output shows evidence of using the spoken content. + +## Mixed gate + +Mixed image+audio is accepted only after vision and audio have each passed their gates and the combined path works. + +Mixed image+audio must use the actual paired test assets: +- Image: `cactus-engine/tests/assets/test_monkey.png` +- Audio prompt: `what is in this image` +- Expected answer: a short coherent answer identifying the monkey/orangutan/proboscis monkey class. + +## Regression gate + +After any code fix: + +```bash +source ../cactus/venv/bin/activate +./cactus/build.sh +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s +``` diff --git a/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md b/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md new file mode 100644 index 000000000..9a27aa908 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md @@ -0,0 +1,44 @@ +# Handoff + +Status: local acceptance probes pass; leave uncommitted for user audit. + +Current proven state: +- Gemma4 image preprocessing parity is fixed. +- Gemma4 vision/image-only generation passes. +- Gemma4 audio encoder output is FP16-close to Python after the audio bias FP16 fix. +- Gemma4 audio-only generation is coherent. +- Gemma4 mixed image+audio answers the spoken question about `test_monkey.png` correctly. +- Qwen image, Parakeet file/PCM, policy tests, and smoke tests pass. + +## Key artifacts + +- Image preprocessing: `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` +- Vision encoder: `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` +- Audio encoder after bias fix: `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` +- First-token dynamic merge: `artifacts/E029_first_token_after_dynamic_merge/` +- Gemma audio-only: `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` +- Gemma mixed: `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` +- Gemma image regression: `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` +- Qwen regression: `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` +- Parakeet regression: `artifacts/E034_parakeet_regression_after_dynamic_merge/` +- Audio encoder unchanged after dynamic merge: `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` + +## Latest outputs + +- Gemma image-only: `The animal in the image is a golden monkey.` +- Gemma audio-only: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- Gemma mixed: `This image is of a golden monkey.` +- Qwen image: identifies a proboscis monkey. +- Parakeet file and PCM: `What is in this image?` + +## Tests + +- Build completed with `source ../cactus/venv/bin/activate && ./cactus/build.sh`. +- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q` -> `16 passed`. +- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s` -> `7 passed, 2 deselected`. + +## Review focus + +- Runtime: dynamic Gemma4 audio merge in `cactus-engine/src/model.cpp`. +- Policy: Gemma4 audio bias FP16 handling in `python/cactus/convert/model_adapters/policy.py`. +- Artifact changes: active fresh Gemma audio encoder graph/bias files were adjusted during the investigation. diff --git a/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md b/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md new file mode 100644 index 000000000..da4b6605a --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md @@ -0,0 +1,74 @@ +# Status For Next Agent + +## One-line status + +Gemma4 image-only, audio-only, and mixed image+audio now pass local end-to-end probes after the audio bias FP16 fix and dynamic Gemma4 audio merge. Qwen, Parakeet, policy, and smoke controls also pass. + +## Current working state + +Repo: +- `/Users/noahcylich/Documents/Desert/cactus-v2-chunked-prefill` + +Branch: +- `v2-chunked-prefill` + +Do not commit or push unless the user explicitly asks. The worktree contains unrelated and related tracked changes plus local investigation artifacts. + +Default Gemma artifact: +- `../cactus/weights/gemma-4-e2b-it-fresh-mm` + +HF reference snapshot: +- `/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db` + +Assets: +- Image: `cactus-engine/tests/assets/test_monkey.png` +- Audio WAV: `local_integration_tests/.tmp/what_is_in_this_image.wav` +- Source MP3: `../cactus/what_is_in_this_image.mp3` + +## Fixes applied + +- Gemma4 audio component bias tensors are kept FP16. E023 shows the audio encoder boundary became FP16-close to Python after this. +- Gemma4 prompts containing audio now bypass the stale static full `lm_encoder` merge graph. The runtime dynamically walks the actual token sequence, routes text through `lm_encoder_step`, routes image/audio token rows through `lm_encoder_media_step`, assembles the LM encoder outputs, and feeds `decoder_prefill_chunk`. +- This keeps the existing static path for image-only and non-audio cases. + +## Latest proof artifacts + +- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json`: first 64 audio rows cosine `0.9978210926`, mean abs diff `0.00645508` versus Python. +- `artifacts/E029_first_token_after_dynamic_merge/`: audio first token `Please`, mixed first token `This`; no `` or immediate stop. +- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json`: audio-only response is coherent with the spoken prompt: `Please provide the image you are referring to. I need an image to tell you what is in it.` +- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json`: mixed response: `This image is of a golden monkey.` +- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json`: image-only response: `The animal in the image is a golden monkey.` +- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json`: Qwen identifies a proboscis monkey. +- `artifacts/E034_parakeet_regression_after_dynamic_merge/`: Parakeet file and PCM both transcribe `What is in this image?` +- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json`: full emitted audio tensor matches E023 exactly, max diff `0`. + +## Latest commands run + +Required build: + +```bash +source ../cactus/venv/bin/activate +./cactus/build.sh +``` + +Policy: + +```bash +PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q +``` + +Result: +- `16 passed` + +Smoke: + +```bash +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s +``` + +Result: +- `7 passed, 2 deselected` + +## Remaining work + +No known local acceptance failure remains from the planned test matrix. Next step is user audit/review of the uncommitted changes. diff --git a/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md b/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md new file mode 100644 index 000000000..643082b7b --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md @@ -0,0 +1,58 @@ +# Continuation Plan + +## Goal + +Keep fresh Gemma4 Cactus native working end to end for image-only, audio-only, and mixed image+audio, while preserving Qwen, Parakeet, and smoke controls. + +The local acceptance plan now passes. This document is for review and any follow-up hardening, not for continuing a known active failure. + +## Current local result + +- Gemma image-only: passes, identifies `test_monkey.png` as a golden monkey. +- Gemma audio-only: passes, responds coherently to the spoken `what is in this image` prompt when no image is supplied. +- Gemma mixed image+audio: passes, answers `This image is of a golden monkey.` +- Qwen image control: passes, identifies a proboscis monkey. +- Parakeet file and PCM controls: pass, both transcribe `What is in this image?` +- Policy tests: pass, `16 passed`. +- Smoke tests: pass, `7 passed, 2 deselected`. + +## Important implementation decisions + +- Keep Gemma4 audio bias tensors FP16. +- For Gemma4 prompts with audio, use the dynamic media merge path rather than the static full `lm_encoder` merge graph. +- Do not carry additional vision behavior from `justin/v2-cpp-fix`; that branch is already in the ancestry and vision already passes. +- Keep non-audio/image-only behavior on the existing static chunk path. + +## Verification artifacts + +- Audio encoder boundary: `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` +- First-token dynamic merge: `artifacts/E029_first_token_after_dynamic_merge/` +- Gemma audio-only: `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` +- Gemma mixed: `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` +- Gemma image regression: `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` +- Qwen regression: `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` +- Parakeet regression: `artifacts/E034_parakeet_regression_after_dynamic_merge/` +- Audio encoder unchanged after dynamic merge: `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` + +## If further testing is requested + +Run the same sequence under the required build rule: + +```bash +source ../cactus/venv/bin/activate +./cactus/build.sh +``` + +Then rerun: + +```bash +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-audio --experiment +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-mixed --experiment +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-image --experiment +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-qwen-image --experiment +PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case parakeet-audio --experiment +PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q +PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s +``` + +Do not commit or push until the user audits the changes. diff --git a/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py b/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py new file mode 100644 index 000000000..cb12337b1 --- /dev/null +++ b/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py @@ -0,0 +1,785 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[3] +INVESTIGATION_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_WEIGHTS_ROOT = REPO_ROOT.parent / "cactus" / "weights" +ASSET_DIR = REPO_ROOT / "cactus-engine" / "tests" / "assets" +TMP_DIR = REPO_ROOT / "local_integration_tests" / ".tmp" + + +def run(cmd: list[str], *, timeout: int = 300) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["CACTUS_NO_CLOUD_TELE"] = "1" + return subprocess.run( + cmd, + cwd=REPO_ROOT, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def command_record(case: str, command: list[str], completed: subprocess.CompletedProcess[str]) -> dict[str, Any]: + parsed: Any = None + if completed.stdout.strip().startswith("{"): + try: + parsed = json.loads(completed.stdout) + except json.JSONDecodeError: + parsed = None + return { + "case": case, + "command": command, + "returncode": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + "parsed_stdout": parsed, + } + + +def ensure_runner() -> Path: + build_dir = TMP_DIR / "build" + runner = build_dir / "integration_runner" + sources = [ + REPO_ROOT / "local_integration_tests" / "native_runner.cpp", + REPO_ROOT / "local_integration_tests" / "CMakeLists.txt", + REPO_ROOT / "cactus" / "build" / "libcactus.a", + ] + if runner.exists() and all(runner.stat().st_mtime >= path.stat().st_mtime for path in sources if path.exists()): + return runner + cmake = run( + [ + "cmake", + "-S", + str(REPO_ROOT / "local_integration_tests"), + "-B", + str(build_dir), + f"-DCACTUS_REPO_ROOT={REPO_ROOT}", + "-DCMAKE_BUILD_TYPE=Release", + ], + timeout=120, + ) + if cmake.returncode != 0: + raise RuntimeError(cmake.stderr or cmake.stdout) + build = run(["cmake", "--build", str(build_dir), "--target", "integration_runner", "-j", str(os.cpu_count() or 4)], timeout=120) + if build.returncode != 0: + raise RuntimeError(build.stderr or build.stdout) + if not runner.exists(): + raise RuntimeError(f"integration runner was not built: {runner}") + return runner + + +def ensure_question_wav() -> Path: + wav = TMP_DIR / "what_is_in_this_image.wav" + if wav.exists(): + return wav + mp3 = REPO_ROOT.parent / "cactus" / "what_is_in_this_image.mp3" + if not mp3.exists(): + raise RuntimeError(f"missing paired mp3: {mp3}") + wav.parent.mkdir(parents=True, exist_ok=True) + completed = run( + [ + "ffmpeg", + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(mp3), + "-ac", + "1", + "-ar", + "16000", + "-sample_fmt", + "s16", + str(wav), + ], + timeout=120, + ) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + return wav + + +def cactus_complete(model: Path, prompt: str, image: Path | None = None, audio: Path | None = None, max_tokens: int = 60) -> dict[str, Any]: + runner = ensure_runner() + command: list[str] = [ + str(runner), + "complete", + "--model", + str(model), + "--prompt", + prompt, + "--max-tokens", + str(max_tokens), + ] + if image is not None: + command.extend(["--image", str(image)]) + if audio is not None: + command.extend(["--audio", str(audio)]) + return command_record("cactus_complete", command, run(command, timeout=360)) + + +def native_json(command: list[str], *, timeout: int = 360) -> dict[str, Any]: + completed = run(command, timeout=timeout) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + try: + return json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError(f"native command did not return JSON: {command}\n{completed.stdout[:1000]}") from exc + + +def parakeet_transcribe(model: Path, audio: Path, pcm: bool = False) -> dict[str, Any]: + runner = ensure_runner() + command = [str(runner), "transcribe-pcm" if pcm else "transcribe", "--model", str(model), "--audio", str(audio)] + return command_record("parakeet_transcribe_pcm" if pcm else "parakeet_transcribe", command, run(command, timeout=240)) + + +def hf_gemma_image(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: + script = f""" +import json +import torch +from pathlib import Path +from transformers import AutoProcessor, AutoModelForImageTextToText + +model_dir = Path({str(model_dir)!r}) +image_path = Path({str(image)!r}) +prompt = {prompt!r} +processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) +model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='auto') +messages = [{{'role': 'user', 'content': [{{'type': 'image', 'image': str(image_path)}}, {{'type': 'text', 'text': prompt}}]}}] +inputs = processor.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors='pt') +inputs = {{k: v.to(model.device) if hasattr(v, 'to') else v for k, v in inputs.items()}} +with torch.no_grad(): + out = model.generate(**inputs, max_new_tokens=40, do_sample=False) +text = processor.decode(out[0][inputs['input_ids'].shape[-1]:], skip_special_tokens=True) +print(json.dumps({{'response': text, 'input_tokens': int(inputs['input_ids'].shape[-1])}})) +""" + command = [sys.executable, "-c", script] + return command_record("hf_gemma_image", command, run(command, timeout=360)) + + +def hf_gemma_native_image(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: + script = f""" +import json +import torch +from pathlib import Path +from transformers import AutoProcessor, AutoModelForImageTextToText +from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs + +model_dir = Path({str(model_dir)!r}) +image_path = Path({str(image)!r}) +prompt = {prompt!r} +processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) +prepared = prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=(str(image_path),), + audio_file=None, + torch_dtype=torch.float16, + use_gemma4_chat_template=True, +) +model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='auto') +inputs = {{}} +summaries = {{}} +for name, tensor in zip(prepared.names, prepared.tensors): + key = 'image_position_ids' if name == 'pixel_position_ids' else name + inputs[key] = tensor.to(model.device) + summaries[name] = {{ + 'shape': list(tensor.shape), + 'dtype': str(tensor.dtype), + 'min': float(tensor.float().min().item()) if tensor.numel() else 0.0, + 'max': float(tensor.float().max().item()) if tensor.numel() else 0.0, + 'mean': float(tensor.float().mean().item()) if tensor.numel() else 0.0, + }} +with torch.no_grad(): + out = model.generate(**inputs, max_new_tokens=40, do_sample=False) +text = processor.decode(out[0][inputs['input_ids'].shape[-1]:], skip_special_tokens=True) +input_ids = prepared.tensors[prepared.names.index('input_ids')] +print(json.dumps({{ + 'response': text, + 'input_tokens': int(input_ids.shape[-1]), + 'input_shapes': prepared.metadata.get('input_shapes'), + 'processor_prompt_prefix': str(prepared.metadata.get('processor_prompt'))[:240], + 'tensor_summaries': summaries, +}})) +""" + command = [sys.executable, "-c", script] + return command_record("hf_gemma_native_image", command, run(command, timeout=360)) + + +def hf_gemma_e002_inputs(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: + script = f""" +import hashlib +import json +import torch +from pathlib import Path +from transformers import AutoProcessor +from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs + +model_dir = Path({str(model_dir)!r}) +image_path = Path({str(image)!r}) +prompt = {prompt!r} +processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) +prepared = prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=(str(image_path),), + audio_file=None, + torch_dtype=torch.float32, + use_gemma4_chat_template=True, +) +by_name = {{name: tensor.detach().cpu() for name, tensor in zip(prepared.names, prepared.tensors)}} +input_ids = by_name['input_ids'].reshape(-1).to(torch.int64) +pixel_values = by_name['pixel_values'].to(torch.float32).contiguous() +pixel_position_ids = by_name['pixel_position_ids'].reshape(-1).to(torch.int64).contiguous() +tokenizer = getattr(processor, 'tokenizer', processor) +image_token = getattr(processor, 'image_token', None) +image_token_id = tokenizer.convert_tokens_to_ids(image_token) if image_token is not None and hasattr(tokenizer, 'convert_tokens_to_ids') else -1 +ids = input_ids.tolist() +image_positions = [i for i, tok in enumerate(ids) if tok == image_token_id] +processor_prompt = str(prepared.metadata.get('processor_prompt')) +assistant_generation_start = len(ids) +if processor_prompt.endswith('<|turn>model\\n'): + no_generation_prompt = processor_prompt[:-len('<|turn>model\\n')] + no_gen = tokenizer(no_generation_prompt, return_tensors='pt', add_special_tokens=False)['input_ids'].reshape(-1) + assistant_generation_start = int(no_gen.numel()) +valid_positions = int(((by_name['pixel_position_ids'] != -1).any(dim=-1)).sum().item()) +pixel_np = pixel_values.numpy() +pos_np = by_name['pixel_position_ids'].to(torch.int64).contiguous().numpy() +payload = {{ + 'token_count': len(ids), + 'assistant_generation_start': assistant_generation_start, + 'image_token_id': int(image_token_id), + 'image_token_start': int(image_positions[0]) if image_positions else -1, + 'image_token_count': len(image_positions), + 'first_20': ids[:20], + 'last_20': ids[-20:], + 'input_ids': ids, + 'prompt_prefix': processor_prompt[:320], + 'pixel_values_shape': list(pixel_values.shape), + 'pixel_position_ids_shape': list(by_name['pixel_position_ids'].shape), + 'pixel_values_min': float(pixel_values.min().item()), + 'pixel_values_max': float(pixel_values.max().item()), + 'pixel_values_mean': float(pixel_values.mean().item()), + 'pixel_values_sha256': hashlib.sha256(pixel_np.tobytes()).hexdigest(), + 'pixel_position_ids_sha256': hashlib.sha256(pos_np.tobytes()).hexdigest(), + 'valid_position_count': valid_positions, + 'first_pixel_values': pixel_values.reshape(-1)[:32].tolist(), + 'first_position_rows_flat': pixel_position_ids[:32].tolist(), + 'pixel_values': pixel_values.reshape(-1).tolist(), + 'pixel_position_ids': pixel_position_ids.tolist(), +}} +print(json.dumps(payload)) +""" + command = [sys.executable, "-c", script] + completed = run(command, timeout=360) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + return json.loads(completed.stdout) + + +def e002_cactus_tokens(model: Path, image: Path, prompt: str) -> dict[str, Any]: + runner = ensure_runner() + command = [ + str(runner), + "e002-tokens", + "--model", + str(model), + "--prompt", + prompt, + "--image", + str(image), + ] + return native_json(command, timeout=360) + + +def e002_cactus_preprocess(model: Path, image: Path) -> dict[str, Any]: + runner = ensure_runner() + command = [ + str(runner), + "e002-preprocess", + "--model", + str(model), + "--image", + str(image), + ] + return native_json(command, timeout=360) + + +def e003_cactus_preprocess_variants(model: Path, image: Path) -> dict[str, Any]: + runner = ensure_runner() + command = [ + str(runner), + "e003-preprocess-variants", + "--model", + str(model), + "--image", + str(image), + ] + return native_json(command, timeout=360) + + +def e002_cactus_vision(model: Path, image: Path) -> dict[str, Any]: + runner = ensure_runner() + command = [ + str(runner), + "e002-vision", + "--model", + str(model), + "--image", + str(image), + "--prompt", + "What animal is in this image? Reply in one complete short sentence.", + ] + return native_json(command, timeout=360) + + +def hf_gemma_vision_features(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: + script = f""" +import json +import torch +from pathlib import Path +from transformers import AutoProcessor, AutoModelForImageTextToText +from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs + +model_dir = Path({str(model_dir)!r}) +image_path = Path({str(image)!r}) +prompt = {prompt!r} +processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) +prepared = prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=(str(image_path),), + audio_file=None, + torch_dtype=torch.float16, + use_gemma4_chat_template=True, +) +by_name = {{name: tensor for name, tensor in zip(prepared.names, prepared.tensors)}} +model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='cpu') +pixel_values = by_name['pixel_values'].to(dtype=torch.float16) +position_ids = by_name['pixel_position_ids'].to(dtype=torch.long) +with torch.no_grad(): + outputs = model.get_image_features(pixel_values=pixel_values, image_position_ids=position_ids) +candidates = {{}} +for name, tensor in ( + ('last_hidden_state', getattr(outputs, 'last_hidden_state', None)), + ('pooler_output', getattr(outputs, 'pooler_output', None)), +): + if tensor is None: + continue + flat = tensor.detach().float().cpu().reshape(-1) + candidates[name] = {{ + 'shape': list(tensor.shape), + 'dtype': str(tensor.dtype), + 'count': int(flat.numel()), + 'min': float(flat.min().item()) if flat.numel() else 0.0, + 'max': float(flat.max().item()) if flat.numel() else 0.0, + 'mean': float(flat.mean().item()) if flat.numel() else 0.0, + 'l2': float(torch.linalg.vector_norm(flat).item()) if flat.numel() else 0.0, + 'first_values': flat[:32].tolist(), + 'values': flat.tolist(), + }} +print(json.dumps({{'features': candidates}})) +""" + completed = run([sys.executable, "-c", script], timeout=420) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + return json.loads(completed.stdout) + + +def first_mismatch(left: list[Any], right: list[Any]) -> int | None: + for idx, (lval, rval) in enumerate(zip(left, right)): + if lval != rval: + return idx + if len(left) != len(right): + return min(len(left), len(right)) + return None + + +def sha256_float_list(values: list[float]) -> str: + import array + + arr = array.array("f", values) + return hashlib.sha256(arr.tobytes()).hexdigest() + + +def run_e002_boundary(out_dir: Path, weights_root: Path) -> None: + image = ASSET_DIR / "test_monkey.png" + prompt = "What animal is in this image? Reply in one complete short sentence." + fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" + hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") + + hf = hf_gemma_e002_inputs(hf_snapshot, image, prompt) + cactus_tokens = e002_cactus_tokens(fresh_gemma, image, prompt) + write_json(out_dir / "hf_native_inputs.json", hf) + write_json(out_dir / "cactus_token_probe.json", cactus_tokens) + + token_mismatch = first_mismatch(hf["input_ids"], cactus_tokens["input_ids"]) + token_compare = { + "gate": "prompt_and_token_spans", + "status": "pass" if token_mismatch is None else "fail", + "input_ids_match": token_mismatch is None, + "first_mismatch_index": token_mismatch, + "hf": {key: hf[key] for key in ("token_count", "assistant_generation_start", "image_token_id", "image_token_start", "image_token_count", "first_20", "last_20", "prompt_prefix")}, + "cactus": {key: cactus_tokens[key] for key in ("token_count", "assistant_generation_start", "image_token_id", "image_token_start", "image_token_count", "first_20", "last_20", "prompt_prefix")}, + } + if token_mismatch is not None: + lo = max(0, token_mismatch - 8) + hi = token_mismatch + 9 + token_compare["mismatch_window"] = { + "hf": hf["input_ids"][lo:hi], + "cactus": cactus_tokens["input_ids"][lo:hi], + "start": lo, + } + write_json(out_dir / "token_spans.json", token_compare) + if token_mismatch is not None: + return + + cactus_preprocess = e002_cactus_preprocess(fresh_gemma, image) + write_json(out_dir / "cactus_preprocess.json", cactus_preprocess) + hf_values = hf["pixel_values"] + cactus_values = cactus_preprocess["pixel_values"] + hf_positions = hf["pixel_position_ids"] + cactus_positions = cactus_preprocess["pixel_position_ids"] + if len(hf_values) != len(cactus_values): + max_abs_diff = None + mean_abs_diff = None + first_pixel_mismatch = min(len(hf_values), len(cactus_values)) + else: + diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, cactus_values)] + max_abs_diff = max(diffs) if diffs else 0.0 + mean_abs_diff = sum(diffs) / len(diffs) if diffs else 0.0 + first_pixel_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-5), None) + position_mismatch = first_mismatch(hf_positions, cactus_positions) + preprocess_pass = ( + hf["pixel_values_shape"] == cactus_preprocess["pixel_values_shape"] + and hf["pixel_position_ids_shape"] == cactus_preprocess["pixel_position_ids_shape"] + and position_mismatch is None + and max_abs_diff is not None + and max_abs_diff <= 1e-5 + ) + compare = { + "gate": "image_preprocessing_tensors", + "status": "pass" if preprocess_pass else "fail", + "pixel_values_shape_match": hf["pixel_values_shape"] == cactus_preprocess["pixel_values_shape"], + "pixel_position_ids_shape_match": hf["pixel_position_ids_shape"] == cactus_preprocess["pixel_position_ids_shape"], + "pixel_position_ids_match": position_mismatch is None, + "first_position_mismatch_index": position_mismatch, + "pixel_values_max_abs_diff": max_abs_diff, + "pixel_values_mean_abs_diff": mean_abs_diff, + "first_pixel_value_mismatch_index_at_1e-5": first_pixel_mismatch, + "hf_summary": {key: hf[key] for key in ("pixel_values_shape", "pixel_position_ids_shape", "pixel_values_min", "pixel_values_max", "pixel_values_mean", "pixel_values_sha256", "pixel_position_ids_sha256", "valid_position_count", "first_pixel_values", "first_position_rows_flat")}, + "cactus_summary": {key: cactus_preprocess[key] for key in ("pixel_values_shape", "pixel_position_ids_shape", "pixel_values_min", "pixel_values_max", "pixel_values_mean", "pixel_values_hash", "pixel_position_ids_hash", "valid_position_count", "first_pixel_values", "first_position_rows_flat")}, + } + if first_pixel_mismatch is not None: + lo = max(0, first_pixel_mismatch - 8) + hi = first_pixel_mismatch + 9 + compare["pixel_mismatch_window"] = { + "start": lo, + "hf": hf_values[lo:hi], + "cactus": cactus_values[lo:hi], + } + if position_mismatch is not None: + lo = max(0, position_mismatch - 8) + hi = position_mismatch + 9 + compare["position_mismatch_window"] = { + "start": lo, + "hf": hf_positions[lo:hi], + "cactus": cactus_positions[lo:hi], + } + write_json(out_dir / "image_preprocess_compare.json", compare) + + +def run_e003_preprocess_isolation(out_dir: Path, weights_root: Path) -> None: + image = ASSET_DIR / "test_monkey.png" + prompt = "What animal is in this image? Reply in one complete short sentence." + fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" + hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") + + hf = hf_gemma_e002_inputs(hf_snapshot, image, prompt) + variants = e003_cactus_preprocess_variants(fresh_gemma, image) + write_json(out_dir / "hf_native_inputs.json", hf) + write_json(out_dir / "cactus_resize_variants.json", variants) + + hf_values = hf["pixel_values"] + comparisons: dict[str, Any] = {} + best_name: str | None = None + best_max: float | None = None + for name, variant in variants["variants"].items(): + values = variant["pixel_values"] + if len(values) != len(hf_values): + comparison = { + "status": "fail", + "length_match": False, + "hf_length": len(hf_values), + "variant_length": len(values), + } + else: + diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, values)] + max_abs = max(diffs) if diffs else 0.0 + mean_abs = sum(diffs) / len(diffs) if diffs else 0.0 + first_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-5), None) + comparison = { + "status": "pass" if max_abs <= 1e-5 else "fail", + "length_match": True, + "max_abs_diff": max_abs, + "mean_abs_diff": mean_abs, + "first_mismatch_index_at_1e-5": first_mismatch, + "variant_summary": { + key: variant[key] + for key in ( + "resized_min", + "resized_max", + "first_resized_values", + "pixel_values_min", + "pixel_values_max", + "pixel_values_mean", + "first_pixel_values", + ) + }, + } + if first_mismatch is not None: + lo = max(0, first_mismatch - 8) + hi = first_mismatch + 9 + comparison["mismatch_window"] = { + "start": lo, + "hf": hf_values[lo:hi], + "variant": values[lo:hi], + } + if best_max is None or max_abs < best_max: + best_name = name + best_max = max_abs + comparisons[name] = comparison + + write_json( + out_dir / "preprocess_operation_compare.json", + { + "gate": "e003_preprocess_operation_isolation", + "source_width": variants["source_width"], + "source_height": variants["source_height"], + "target_width": variants["target_width"], + "target_height": variants["target_height"], + "patch_size": variants["patch_size"], + "pooling_kernel_size": variants["pooling_kernel_size"], + "max_patches": variants["max_patches"], + "patch_dim": variants["patch_dim"], + "rescale_factor": variants["rescale_factor"], + "hf_summary": { + key: hf[key] + for key in ( + "pixel_values_shape", + "pixel_values_min", + "pixel_values_max", + "pixel_values_mean", + "first_pixel_values", + ) + }, + "best_variant": best_name, + "best_variant_max_abs_diff": best_max, + "comparisons": comparisons, + }, + ) + + +def compare_float_vectors(hf_values: list[float], cactus_values: list[float]) -> dict[str, Any]: + if len(hf_values) != len(cactus_values): + return { + "status": "fail", + "length_match": False, + "hf_length": len(hf_values), + "cactus_length": len(cactus_values), + } + cactus_nonfinite = sum(1 for value in cactus_values if value is None) + hf_nonfinite = sum(1 for value in hf_values if value is None) + if cactus_nonfinite or hf_nonfinite: + return { + "status": "fail", + "length_match": True, + "hf_nonfinite_count": hf_nonfinite, + "cactus_nonfinite_count": cactus_nonfinite, + "first_nonfinite_index": next( + (i for i, (a, b) in enumerate(zip(hf_values, cactus_values)) if a is None or b is None), + None, + ), + } + diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, cactus_values)] + dot = sum(float(a) * float(b) for a, b in zip(hf_values, cactus_values)) + hf_norm = sum(float(a) * float(a) for a in hf_values) ** 0.5 + cactus_norm = sum(float(b) * float(b) for b in cactus_values) ** 0.5 + max_abs = max(diffs) if diffs else 0.0 + mean_abs = sum(diffs) / len(diffs) if diffs else 0.0 + first_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-3), None) + result: dict[str, Any] = { + "status": "pass" if max_abs <= 1e-2 and mean_abs <= 1e-3 else "fail", + "length_match": True, + "max_abs_diff": max_abs, + "mean_abs_diff": mean_abs, + "cosine_similarity": dot / (hf_norm * cactus_norm) if hf_norm and cactus_norm else 0.0, + "first_mismatch_index_at_1e-3": first_mismatch, + } + if first_mismatch is not None: + lo = max(0, first_mismatch - 8) + hi = first_mismatch + 9 + result["mismatch_window"] = { + "start": lo, + "hf": hf_values[lo:hi], + "cactus": cactus_values[lo:hi], + } + return result + + +def run_e002_vision_boundary(out_dir: Path, weights_root: Path) -> None: + image = ASSET_DIR / "test_monkey.png" + prompt = "What animal is in this image? Reply in one complete short sentence." + fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" + hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") + + hf = hf_gemma_vision_features(hf_snapshot, image, prompt) + cactus = e002_cactus_vision(fresh_gemma, image) + write_json(out_dir / "hf_vision_features.json", hf) + write_json(out_dir / "cactus_vision_features.json", cactus) + + comparisons: dict[str, Any] = {} + for cactus_name, cactus_feature in cactus["features"].items(): + comparisons[cactus_name] = {} + for hf_name, hf_feature in hf["features"].items(): + comparison = compare_float_vectors(hf_feature["values"], cactus_feature["values"]) + comparison["hf_shape"] = hf_feature["shape"] + comparison["cactus_shape"] = cactus_feature["shape"] + comparison["hf_summary"] = {key: hf_feature[key] for key in ("dtype", "count", "min", "max", "mean", "l2", "first_values")} + comparison["cactus_summary"] = {key: cactus_feature[key] for key in ("precision", "count", "finite_count", "nonfinite_count", "min", "max", "mean", "l2", "first_values")} + comparisons[cactus_name][hf_name] = comparison + + write_json( + out_dir / "vision_encoder_compare.json", + { + "gate": "vision_encoder_output", + "comparisons": comparisons, + }, + ) + + +def manifest_summary(path: Path) -> dict[str, Any]: + manifest = path / "components" / "manifest.json" + config = path / "config.txt" + result: dict[str, Any] = {"path": str(path), "manifest_exists": manifest.exists(), "config_exists": config.exists()} + if manifest.exists(): + data = json.loads(manifest.read_text()) + result.update( + { + "family": data.get("family"), + "task": data.get("task"), + "model_source": data.get("model_source"), + "component_order": data.get("component_order"), + "components": [ + { + "component": component.get("component"), + "logical_inputs": component.get("logical_inputs"), + "logical_outputs": component.get("logical_outputs"), + } + for component in data.get("components", []) + ], + } + ) + return result + + +def run_case(case: str, out_dir: Path, weights_root: Path) -> None: + image = ASSET_DIR / "test_monkey.png" + prompt = "What animal is in this image? Reply in one complete short sentence." + fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" + qwen = weights_root / "qwen3-vl-2b-instruct-reconvert" + parakeet = weights_root / "parakeet-tdt-0.6b-v3-transpiled" + hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") + + if case == "inventory": + write_json(out_dir / "inventory.json", {"gemma_fresh": manifest_summary(fresh_gemma)}) + return + if case == "hf-gemma-image": + write_json(out_dir / "hf_gemma_image.json", hf_gemma_image(hf_snapshot, image, prompt)) + return + if case == "hf-gemma-native-image": + write_json(out_dir / "hf_gemma_native_image.json", hf_gemma_native_image(hf_snapshot, image, prompt)) + return + if case == "cactus-gemma-image": + write_json(out_dir / "cactus_gemma_image.json", cactus_complete(fresh_gemma, prompt, image=image)) + return + if case == "cactus-gemma-audio": + wav = ensure_question_wav() + audio_prompt = "Listen to the audio and answer briefly." + write_json(out_dir / "cactus_gemma_audio.json", cactus_complete(fresh_gemma, audio_prompt, audio=wav)) + return + if case == "cactus-gemma-mixed": + wav = ensure_question_wav() + mixed_prompt = "Answer the spoken question using the provided image. Reply in one short sentence." + write_json(out_dir / "cactus_gemma_mixed.json", cactus_complete(fresh_gemma, mixed_prompt, image=image, audio=wav)) + return + if case == "cactus-qwen-image": + write_json(out_dir / "cactus_qwen_image.json", cactus_complete(qwen, prompt, image=image, max_tokens=40)) + return + if case == "parakeet-audio": + wav = ensure_question_wav() + write_json(out_dir / "parakeet_audio.json", parakeet_transcribe(parakeet, wav)) + write_json(out_dir / "parakeet_audio_pcm.json", parakeet_transcribe(parakeet, wav, pcm=True)) + return + if case == "all-baseline": + for subcase in ("inventory", "hf-gemma-image", "hf-gemma-native-image", "cactus-gemma-image", "cactus-qwen-image", "parakeet-audio"): + run_case(subcase, out_dir, weights_root) + return + if case == "e002-boundary": + run_e002_boundary(out_dir, weights_root) + return + if case == "e003-preprocess-isolation": + run_e003_preprocess_isolation(out_dir, weights_root) + return + if case == "e002-vision-boundary": + run_e002_vision_boundary(out_dir, weights_root) + return + raise ValueError(f"unknown case: {case}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--case", required=True) + parser.add_argument("--experiment", required=True) + parser.add_argument("--weights-root", default=str(DEFAULT_WEIGHTS_ROOT)) + args = parser.parse_args() + + out_dir = INVESTIGATION_ROOT / "artifacts" / args.experiment + out_dir.mkdir(parents=True, exist_ok=True) + write_json( + out_dir / "run_metadata.json", + { + "case": args.case, + "experiment": args.experiment, + "repo_root": str(REPO_ROOT), + "weights_root": str(Path(args.weights_root).resolve()), + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + }, + ) + run_case(args.case, out_dir, Path(args.weights_root).resolve()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/cactus/convert/cactus_adapters/tensor_io.py b/python/cactus/convert/cactus_adapters/tensor_io.py index ad36c8c38..3ce614cc7 100644 --- a/python/cactus/convert/cactus_adapters/tensor_io.py +++ b/python/cactus/convert/cactus_adapters/tensor_io.py @@ -281,8 +281,7 @@ def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=Fal data = data * GEMMA4_WEIGHT_SCALE elif 'router_scale' in filename: data = data / GEMMA4_WEIGHT_SCALE - elif filename in ('token_embeddings.weights', 'output_weight.weights', - 'embed_vision_proj.weights', 'embed_vision_embedding.weights'): + elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights'): data = data / GEMMA4_WEIGHT_SCALE elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE diff --git a/python/cactus/convert/model_adapters/adapters.py b/python/cactus/convert/model_adapters/adapters.py index bf51ec941..dbb10eb45 100644 --- a/python/cactus/convert/model_adapters/adapters.py +++ b/python/cactus/convert/model_adapters/adapters.py @@ -162,7 +162,7 @@ def policy(self, match: NameMatch, shape: tuple[int, ...], requested_bits: int) policy = super().policy(match, shape, requested_bits) name = match.source_name if name == "model.embed_vision.embedding_projection.weight": - return replace(policy, use_gptq=False, fallback_reason=policy.fallback_reason or "vision embedding projection has no stable Hessian target") + return policy if ".self_attn." in name and (name.endswith(".k_proj.weight") or name.endswith(".v_proj.weight")): parts = name.split(".") try: diff --git a/python/cactus/convert/model_adapters/naming.py b/python/cactus/convert/model_adapters/naming.py index e74c26236..eaeaa0f44 100644 --- a/python/cactus/convert/model_adapters/naming.py +++ b/python/cactus/convert/model_adapters/naming.py @@ -23,7 +23,6 @@ class NameMatch: GEMMA4_DIV_BASENAMES = { "token_embeddings", "output_weight", - "embed_vision_proj", "embed_vision_embedding", } diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index 6c0aebd71..f8e64fb05 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -25,7 +25,7 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, if "norm" in out.lower(): return TensorPolicy("fallback", "FP16", None, component, False, "none", "norm tensor") if family == "gemma4" and name == "model.embed_vision.embedding_projection.weight": - return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, False, "hadamard", "vision embedding projection has no stable Hessian target") + return TensorPolicy("fallback", "FP16", None, component, False, "none", "vision embedding projection scale-sensitive") if family in {"parakeet", "parakeet_tdt"} and out.endswith(".bias") and ( "conv_" in out or out.startswith("subsampling_") or out.startswith("ctc_head_") ): @@ -45,7 +45,7 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, if family == "whisper" and out.startswith("encoder.layer_"): return TensorPolicy("fallback", "FP16", None, component, False, "none", "whisper encoder tensor") if family == "gemma4" and component == "audio" and (name.endswith(".bias") or out.endswith(".bias") or ".bias." in out): - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "bias tensor") + return TensorPolicy("fallback", "FP16", None, component, False, "none", "audio bias tensor") if name.endswith(".bias") or out.endswith(".bias") or ".bias." in out: return TensorPolicy("fallback", "FP16", None, component, False, "none", "bias tensor") if len(shape) != 2: diff --git a/python/cactus/convert/tests/test_policy.py b/python/cactus/convert/tests/test_policy.py index 441b94d2c..3a39f9fe8 100644 --- a/python/cactus/convert/tests/test_policy.py +++ b/python/cactus/convert/tests/test_policy.py @@ -38,7 +38,7 @@ def test_gemma4_adapter_disables_gptq_for_unhookable_tensors(): vision_proj = cactus_name_for_tensor("model.embed_vision.embedding_projection.weight", "gemma4", 35) vision_policy = adapter.policy(vision_proj, (1536, 1152), 4) - assert vision_policy.precision == "CQ4" + assert vision_policy.precision == "FP16" assert not vision_policy.use_gptq vision_tower = cactus_name_for_tensor("model.vision_tower.encoder.layers.0.mlp.down_proj.linear.weight", "gemma4", 35) @@ -56,11 +56,11 @@ def test_policy_audio_no_gptq(): assert not p.use_gptq -def test_policy_bias_int8(): +def test_policy_gemma4_audio_bias_fp16(): match = cactus_name_for_tensor("model.audio_tower.output_proj.bias", "gemma4", 1) p = policy_for_tensor(match, (1536,), 4, "gemma4") - assert p.precision == "INT8" - assert p.bits == 8 + assert p.precision == "FP16" + assert p.bits is None def test_policy_position_embedding_fp16(): diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index c6639295a..5ec2e48c0 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -2761,9 +2761,7 @@ def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: def _ensure_scalar_math_tensor(g: Graph, tensor: Tensor) -> Tensor: - if tensor.dtype == Graph.FP16: - return tensor - return g.precision_cast(tensor, Graph.FP16) + return tensor def _ensure_tensor_dtype(g: Graph, tensor: Tensor, dtype: int) -> Tensor: diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 914f693f5..a31973b0e 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from dataclasses import dataclass import inspect +import math import struct import os from pathlib import Path @@ -732,7 +733,15 @@ def _gemma4_pool_vision_hidden_native_like( else: valid_bins = torch.logical_not((weights == 0).all(dim=0)) pooled_batches.append(pooled_full[valid_bins]) - return torch.cat(pooled_batches, dim=0) + pooled_hidden = pooled_batches[0] if len(pooled_batches) == 1 else torch.cat(pooled_batches, dim=0) + hidden_size = int(_module_or_config_attr(vision_tower, "hidden_size", pooled_hidden.shape[-1]) or pooled_hidden.shape[-1]) + pooled_hidden = pooled_hidden * math.sqrt(float(hidden_size)) + if bool(_module_or_config_attr(vision_tower, "standardize", False)): + std_bias = getattr(vision_tower, "std_bias", None) + std_scale = getattr(vision_tower, "std_scale", None) + if std_bias is not None and std_scale is not None: + pooled_hidden = (pooled_hidden - std_bias) * std_scale + return pooled_hidden def _gemma4_vision_encoder_hidden_states( @@ -857,6 +866,9 @@ def _gemma4_compute_native_like_image_features( projection = getattr(embed_vision, "embedding_projection", None) if not isinstance(projection, torch.nn.Linear): raise TypeError("Gemma4 vision embedder is missing embedding_projection") + pre_projection_norm = getattr(embed_vision, "embedding_pre_projection_norm", None) + if isinstance(pre_projection_norm, torch.nn.Module): + pooled_hidden = pre_projection_norm(pooled_hidden) pooled_hidden = pooled_hidden.to(dtype=projection.weight.dtype) projected = F.linear( pooled_hidden, @@ -869,7 +881,7 @@ def _gemma4_compute_native_like_image_features( post_proj_norm_weight, eps=_gemma4_layer_norm_eps(multimodal_backbone), ) - return _gemma4_rms_norm_no_scale(projected, eps=float(getattr(embed_vision, "eps", 1e-6))) + return projected def _gemma4_compute_native_like_audio_features( diff --git a/python/cactus/transpile/runtime_compat.py b/python/cactus/transpile/runtime_compat.py index 677b15cb3..9c17c722e 100644 --- a/python/cactus/transpile/runtime_compat.py +++ b/python/cactus/transpile/runtime_compat.py @@ -233,31 +233,31 @@ def permute(self, x, permutation, backend=Graph.CPU): return orig_permute(self, _ensure_scalar_tensor(self, x), permutation, backend=backend) def scalar_add(self, x, value): - return orig_scalar_add(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_add(self, self._ensure_tensor(x), value) def scalar_subtract(self, x, value): - return orig_scalar_subtract(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_subtract(self, self._ensure_tensor(x), value) def scalar_multiply(self, x, value): - return orig_scalar_multiply(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_multiply(self, self._ensure_tensor(x), value) def scalar_divide(self, x, value): - return orig_scalar_divide(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_divide(self, self._ensure_tensor(x), value) def scalar_exp(self, x): - return orig_scalar_exp(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_exp(self, self._ensure_tensor(x)) def scalar_sqrt(self, x): - return orig_scalar_sqrt(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_sqrt(self, self._ensure_tensor(x)) def scalar_cos(self, x): - return orig_scalar_cos(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_cos(self, self._ensure_tensor(x)) def scalar_sin(self, x): - return orig_scalar_sin(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_sin(self, self._ensure_tensor(x)) def scalar_log(self, x): - return orig_scalar_log(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_log(self, self._ensure_tensor(x)) def clamp(self, x, lo, hi): if orig_clamp is None: @@ -284,10 +284,10 @@ def cat(self, tensors, axis=0): return orig_cat(self, legalized, axis=axis) def abs(self, x): - return orig_abs(self, _ensure_scalar_tensor(self, x)) + return orig_abs(self, self._ensure_tensor(x)) def pow(self, x, exponent): - return orig_pow(self, _ensure_scalar_tensor(self, x), exponent) + return orig_pow(self, self._ensure_tensor(x), exponent) def expand(self, x, shape): x = self._ensure_tensor(x) From 68a422e6a6b2bd1386aa4378d1cb23ed53a540c8 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 20 May 2026 18:42:11 -0700 Subject: [PATCH 3/8] Reduce multimodal graph memory pressure Release graph temporaries after their final use while retaining declared component outputs and cache state buffers. Unload transient media, encoder, and chunk-prefill graphs after copying their outputs so Gemma4 image/audio requests do not keep large intermediate buffers resident. Validation: focused Gemma4, Qwen3-VL, and LFM2-VL integration checks pass locally. Signed-off-by: Noah Cylich --- cactus-engine/src/engine.h | 2 + cactus-engine/src/model.cpp | 92 +++++++++++++++++++++++++++--------- cactus-graph/cactus_graph.h | 5 ++ cactus-graph/src/core.cpp | 32 +++++++++++++ cactus-graph/src/execute.cpp | 71 +++++++++++++++++++++++----- 5 files changed, 169 insertions(+), 33 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 3b3478478..6da7ce58d 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -672,6 +672,8 @@ class Model { bool load_manifest(); bool setup_tokenizer(); bool load_components(const std::unordered_set& required_components); + bool load_component_graph(Component& comp); + void unload_component_graph(Component& comp); bool bind_runtime_buffers(Component& comp); void run_step(uint32_t token_id, size_t position, bool read_logits); void run_encoder_step(uint32_t token_id, size_t position); diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 5c20a02f7..2812a488d 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -389,32 +389,48 @@ bool Model::setup_tokenizer() { bool Model::load_components(const std::unordered_set& required_components) { for (auto& [name, comp] : components_) { if (!required_components.empty() && !required_components.count(name)) continue; - if (comp.graph_path.empty()) continue; - fs::path full = fs::path(bundle_dir_) / comp.graph_path; + if (!load_component_graph(comp)) return false; + } + return true; +} + +bool Model::load_component_graph(Component& comp) { + if (comp.graph) return true; + if (comp.graph_path.empty()) return true; + fs::path full = fs::path(bundle_dir_) / comp.graph_path; + try { + comp.graph = std::make_unique(CactusGraph::load(full.string())); + comp.graph->retain_outputs(comp.output_node_ids); + } catch (const std::exception& e) { + CACTUS_LOG_ERROR("model", "load " << comp.graph_path << ": " << e.what()); + return false; + } + for (const auto& b : comp.bindings) { + if (b.node_id < 0) continue; try { - comp.graph = std::make_unique(CactusGraph::load(full.string())); + fs::path weight_path(b.path); + if (weight_path.is_absolute()) { + fs::path local = fs::path(bundle_dir_) / weight_path.filename(); + if (fs::exists(local)) weight_path = local; + } else { + weight_path = fs::path(bundle_dir_) / weight_path; + } + comp.graph->bind_mmap_weights(static_cast(b.node_id), weight_path.string()); } catch (const std::exception& e) { - CACTUS_LOG_ERROR("model", "load " << comp.graph_path << ": " << e.what()); + CACTUS_LOG_ERROR("model", "bind " << b.path << ": " << e.what()); return false; } - for (const auto& b : comp.bindings) { - if (b.node_id < 0) continue; - try { - fs::path weight_path(b.path); - if (weight_path.is_absolute()) { - fs::path local = fs::path(bundle_dir_) / weight_path.filename(); - if (fs::exists(local)) weight_path = local; - } else { - weight_path = fs::path(bundle_dir_) / weight_path; - } - comp.graph->bind_mmap_weights(static_cast(b.node_id), weight_path.string()); - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("model", "bind " << b.path << ": " << e.what()); - return false; - } - } } - return true; + return bind_runtime_buffers(comp); +} + +void Model::unload_component_graph(Component& comp) { + if (comp.graph) { + comp.graph->release_runtime_buffers(); + comp.graph->release_all_weight_pages(); + } + comp.input_buffers.clear(); + comp.graph.reset(); } bool Model::bind_runtime_buffers(Component& comp) { @@ -909,6 +925,9 @@ void Model::run_media_step(size_t position, const uint8_t* feature_row, size_t f void Model::run_vision_encoder(const std::string& image_path) { if (!vision_encoder_) return; + if (!load_component_graph(*vision_encoder_)) { + throw std::runtime_error("failed to load vision_encoder"); + } Gemma4ImagePreprocessed prep = preprocess_gemma4_image(image_path, config_); write_bytes_input(*vision_encoder_, "pixel_values", prep.pixel_values.data(), prep.pixel_values.size() * sizeof(float)); @@ -926,10 +945,16 @@ void Model::run_vision_encoder(const std::string& image_path) { media_feature_shapes_[name] = desc.shape; media_feature_precisions_[name] = desc.precision; } + vision_encoder_->graph->release_runtime_buffers(); + vision_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*vision_encoder_); } void Model::run_audio_encoder(const std::vector& audio_features) { if (!audio_encoder_) return; + if (!load_component_graph(*audio_encoder_)) { + throw std::runtime_error("failed to load audio_encoder"); + } std::vector candidate_input_names = {"input_features", "audio_features"}; bool wrote = false; int feature_idx = -1; @@ -998,6 +1023,9 @@ void Model::run_audio_encoder(const std::vector& audio_features) { media_feature_shapes_[name] = desc.shape; media_feature_precisions_[name] = desc.precision; } + audio_encoder_->graph->release_runtime_buffers(); + audio_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*audio_encoder_); } uint32_t Model::argmax_last_logits() { @@ -1336,7 +1364,10 @@ bool Model::build_lm_encoder_outputs_dynamic_gemma4( const void* ptr = component->graph->get_output(node_id); std::memcpy(store_bytes[info.name].data() + pos * info.per_token_bytes, ptr, info.per_token_bytes); } + component->graph->release_runtime_buffers(); } + encoder_->graph->release_all_weight_pages(); + lm_encoder_media_step_->graph->release_all_weight_pages(); return true; } @@ -1348,6 +1379,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, std::vector qwen_images; if (have_images) { + if (!load_component_graph(*vision_encoder_)) { + throw std::runtime_error("failed to load vision_encoder"); + } for (const auto& path : image_paths) { if (family_ == "lfm2_vl") { Lfm2VlImagePreprocessed prep = preprocess_lfm2_vl_image(path, config_); @@ -1442,7 +1476,10 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, media_feature_shapes_[name] = desc.shape; media_feature_precisions_[name] = desc.precision; } + vision_encoder_->graph->release_runtime_buffers(); + vision_encoder_->graph->release_all_weight_pages(); } + unload_component_graph(*vision_encoder_); } if (have_audio) { @@ -1452,13 +1489,16 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, std::map> store_bytes; std::map store_prec; std::map> store_shape; - const bool needs_dynamic_walk = family_ == "gemma4" && have_audio && lm_encoder_media_step_ != nullptr; + const bool needs_dynamic_walk = family_ == "gemma4" && (have_images || have_audio) && lm_encoder_media_step_ != nullptr; if (needs_dynamic_walk) { if (!build_lm_encoder_outputs_dynamic_gemma4(tokens, store_bytes, store_prec, store_shape)) { return false; } } else { + if (!load_component_graph(*lm_encoder_)) { + throw std::runtime_error("failed to load lm_encoder"); + } int ids_idx = input_index(*lm_encoder_, "input_ids"); if (ids_idx >= 0) { auto& ids_buf = lm_encoder_->input_buffers[ids_idx]; @@ -1512,6 +1552,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, store_prec[name] = desc.precision; store_shape[name] = desc.shape; } + lm_encoder_->graph->release_runtime_buffers(); + lm_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*lm_encoder_); } auto embeds_shape_it = store_shape.find("inputs_embeds"); @@ -1528,6 +1571,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, size_t chunk_seq = 0; { + if (!load_component_graph(*decoder_prefill_chunk_)) { + throw std::runtime_error("failed to load decoder_prefill_chunk"); + } int idx = input_index(*decoder_prefill_chunk_, "inputs_embeds"); if (idx < 0) return false; size_t node_id = static_cast(decoder_prefill_chunk_->runtime_input_node_ids[idx]); @@ -1590,6 +1636,8 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, } if (whole_chunks_end > 0 && decoder_ != nullptr) { copy_cache_states(*decoder_prefill_chunk_, *decoder_); + decoder_prefill_chunk_->graph->release_runtime_buffers(); + unload_component_graph(*decoder_prefill_chunk_); } for (size_t pos = whole_chunks_end; pos < valid_seq; ++pos) { for (const auto& kv : store_bytes) { diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index c5373bcbe..02a6eadda 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -299,6 +299,7 @@ struct BufferDesc { void allocate(); void allocate_from_pool(BufferPool& pool); void release_to_pool(BufferPool& pool); + void release_memory(BufferPool& pool); void set_external(void* ptr); }; @@ -676,6 +677,9 @@ class CactusGraph { void release_weight_pages(size_t node_id); void prefetch_weight_pages(size_t node_id); void release_all_weight_pages(); + void release_runtime_buffers(); + void clear_buffer_pool(); + void retain_outputs(const std::vector& node_ids); size_t persistent(size_t source_node); bool is_populated(size_t persistent_node_id) const; @@ -715,6 +719,7 @@ class CactusGraph { std::unordered_set persistent_node_ids_; std::unordered_set populated_node_ids_; std::unordered_set embedded_input_node_ids_; + std::unordered_set retained_output_node_ids_; }; namespace GraphFile { diff --git a/cactus-graph/src/core.cpp b/cactus-graph/src/core.cpp index 691f57b41..78a0a6523 100644 --- a/cactus-graph/src/core.cpp +++ b/cactus-graph/src/core.cpp @@ -160,9 +160,18 @@ void BufferDesc::release_to_pool(BufferPool& pool) { } } +void BufferDesc::release_memory(BufferPool& pool) { + release_to_pool(pool); + data.reset(); + external_data = nullptr; +} + void BufferDesc::set_external(void* ptr) { external_data = ptr; data.reset(); + if (pooled_data) { + delete[] pooled_data; + } pooled_data = nullptr; } @@ -249,3 +258,26 @@ void CactusGraph::allocate_buffers() { } } } + +void CactusGraph::release_runtime_buffers() { + for (auto& node : nodes_) { + if (node->op_type == OpType::INPUT) continue; + if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) continue; + if (persistent_node_ids_.count(node->id)) continue; + node->output_buffer.release_memory(buffer_pool_); + } + buffer_pool_.clear(); + shrink_thread_local_buffers(); +} + +void CactusGraph::clear_buffer_pool() { + buffer_pool_.clear(); +} + +void CactusGraph::retain_outputs(const std::vector& node_ids) { + for (int node_id : node_ids) { + if (node_id >= 0) { + retained_output_node_ids_.insert(static_cast(node_id)); + } + } +} diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 71fdb3449..a7dc86e1f 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -359,6 +359,57 @@ void CactusGraph::execute(const std::string& profile_file) { } }; + auto can_release_node = [&](size_t node_idx) { + const auto& node = nodes_[node_idx]; + if (node->op_type == OpType::INPUT) return false; + if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) return false; + if (persistent_node_ids_.count(node->id)) return false; + if (retained_output_node_ids_.count(node->id)) return false; + return true; + }; + + auto may_alias_input = [](const GraphNode& node) { + return (node.op_type == OpType::SLICE && node.params.axis == 0) || + (node.op_type == OpType::INDEX && node.params.axis == 0); + }; + + auto preallocates_output = [](const GraphNode& node) { + return !((node.op_type == OpType::SLICE && node.params.axis == 0) || + (node.op_type == OpType::INDEX && node.params.axis == 0)); + }; + + std::vector last_use(n, 0); + std::vector use_count(n, 0); + for (size_t i = 0; i < n; ++i) { + for (size_t input_id : nodes_[i]->input_ids) { + auto it = node_index_map_.find(input_id); + if (it != node_index_map_.end()) { + last_use[it->second] = std::max(last_use[it->second], i); + ++use_count[it->second]; + } + } + } + + std::vector keep_until_graph_cleanup(n, false); + for (size_t i = 0; i < n; ++i) { + const auto& node = *nodes_[i]; + if (!may_alias_input(node) || node.input_ids.empty()) continue; + auto it = node_index_map_.find(node.input_ids[0]); + if (it == node_index_map_.end()) continue; + size_t base_idx = it->second; + if (use_count[i] == 0) { + keep_until_graph_cleanup[base_idx] = true; + } else { + last_use[base_idx] = std::max(last_use[base_idx], last_use[i]); + } + } + + std::vector> release_after(n); + for (size_t i = 0; i < n; ++i) { + if (!can_release_node(i) || use_count[i] == 0 || keep_until_graph_cleanup[i]) continue; + release_after[last_use[i]].push_back(i); + } + if (!need_debug) { for (size_t i = 0; i < n; ++i) { auto& node = nodes_[i]; @@ -366,26 +417,24 @@ void CactusGraph::execute(const std::string& profile_file) { if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) { dispatch_node(*node, nodes_, node_index_map_); populated_node_ids_.insert(node->id); + for (size_t release_idx : release_after[i]) { + nodes_[release_idx]->output_buffer.release_memory(pool); + } continue; } - node->output_buffer.allocate_from_pool(pool); + if (preallocates_output(*node)) { + node->output_buffer.allocate_from_pool(pool); + } dispatch_node(*node, nodes_, node_index_map_); trace_nonfinite(i, *node); if (node->op_type == OpType::PERSISTENT) { populated_node_ids_.insert(node->id); } - } - return; - } - - std::vector last_use(n, 0); - for (size_t i = 0; i < n; ++i) { - for (size_t input_id : nodes_[i]->input_ids) { - auto it = node_index_map_.find(input_id); - if (it != node_index_map_.end()) { - last_use[it->second] = std::max(last_use[it->second], i); + for (size_t release_idx : release_after[i]) { + nodes_[release_idx]->output_buffer.release_memory(pool); } } + return; } auto get_env_str = [](const char* name) -> std::string { From 408b59741c595d56fe1404da3e4e81645f62203f Mon Sep 17 00:00:00 2001 From: justinl66 Date: Wed, 20 May 2026 21:29:09 -0700 Subject: [PATCH 4/8] Fix multimodal multiturn Signed-off-by: justinl66 --- cactus-engine/src/complete.cpp | 85 +++++---- cactus-engine/src/engine.h | 11 +- cactus-engine/src/model.cpp | 205 ++++++++++++++-------- cactus-engine/src/transcribe.cpp | 2 +- cactus-engine/src/utils.h | 21 +-- local_integration_tests/native_runner.cpp | 4 +- 6 files changed, 200 insertions(+), 128 deletions(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 641b8c1b9..3021facb4 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -325,7 +325,7 @@ struct PreparedPrompt { size_t context_token_count = 0; std::vector> images; - std::vector audio_features; + std::vector> audio_features; size_t audio_num_frames = 0; bool has_images() const { @@ -334,7 +334,8 @@ struct PreparedPrompt { } bool has_audio() const { - return !audio_features.empty(); + return std::any_of(audio_features.begin(), audio_features.end(), + [](const auto& mel) { return !mel.empty(); }); } }; @@ -457,38 +458,36 @@ PreparedPrompt prepare_prompt( } if (prompt.model_type == Config::ModelType::GEMMA4) { - std::vector audio_samples; - if (pcm_buffer != nullptr && pcm_buffer_size > 1) { - auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); - audio_samples = resample_to_16k_fp32(waveform_fp32, 16000); - } else if (!prompt.audio_paths.empty()) { - for (auto it = prompt.messages.rbegin(); it != prompt.messages.rend(); ++it) { - if (!it->audio.empty()) { - const std::string& audio_path = it->audio.back(); - AudioFP32 wav = load_wav(audio_path); - audio_samples = resample_to_16k_fp32(wav.samples, wav.sample_rate); - break; - } - } - } std::vector user_indices; for (size_t i = 0; i < prompt.messages.size(); i++) { if (prompt.messages[i].role == "user") user_indices.push_back(i); } auto& counts = handle->user_audio_counts; - for (size_t u = 0; u + 1 < user_indices.size(); u++) { - if (u < counts.size() && counts[u] > 0) { - prompt.messages[user_indices[u]].audio_soft_token_count = counts[u]; + if (counts.size() < user_indices.size()) counts.resize(user_indices.size(), 0); + + if (pcm_buffer != nullptr && pcm_buffer_size > 1 && !user_indices.empty()) { + auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); + auto samples_16k = resample_to_16k_fp32(waveform_fp32, 16000); + if (!samples_16k.empty()) { + auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(samples_16k, handle->model->get_config()); + prompt.audio_features.push_back(std::move(audio_prep.features)); + size_t u = user_indices.size() - 1; + prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; + counts[u] = audio_prep.num_soft_tokens; + } + } else if (!prompt.audio_paths.empty()) { + for (size_t u = 0; u < user_indices.size(); u++) { + const auto& msg = prompt.messages[user_indices[u]]; + if (msg.audio.empty()) continue; + const std::string& audio_path = msg.audio.back(); + AudioFP32 wav = load_wav(audio_path); + auto samples_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); + if (samples_16k.empty()) continue; + auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(samples_16k, handle->model->get_config()); + prompt.audio_features.push_back(std::move(audio_prep.features)); + prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; + counts[u] = audio_prep.num_soft_tokens; } - } - if (!audio_samples.empty() && !user_indices.empty()) { - auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(audio_samples, handle->model->get_config()); - prompt.audio_features = std::move(audio_prep.features); - prompt.audio_num_frames = audio_prep.num_frames; - size_t u = user_indices.size() - 1; - prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; - if (counts.size() <= u) counts.resize(u + 1, 0); - counts[u] = audio_prep.num_soft_tokens; } } @@ -542,6 +541,24 @@ PrefillResult do_prefill( if (tokens_to_process.size() > 1) { std::vector prefill_tokens(tokens_to_process.begin(), tokens_to_process.end() - 1); result.prefilled_count = prefill_tokens.size(); + + auto slice_delta_audio = [&]() -> std::vector> { + if (!result.was_prefix) return prompt.audio_features; + const size_t cached_user_messages = handle->processed_images.size(); + size_t cached_audio_count = 0; + size_t seen_user = 0; + for (const auto& msg : prompt.messages) { + if (msg.role != "user") continue; + if (seen_user >= cached_user_messages) break; + if (!msg.audio.empty()) cached_audio_count++; + seen_user++; + } + cached_audio_count = std::min(cached_audio_count, prompt.audio_features.size()); + return std::vector>( + prompt.audio_features.begin() + cached_audio_count, + prompt.audio_features.end()); + }; + if (has_images && has_audio) { std::vector delta_image_paths; if (result.was_prefix) { @@ -556,7 +573,7 @@ PrefillResult do_prefill( } else { delta_image_paths = prompt.image_paths; } - handle->model->prefill_with_media(prefill_tokens, delta_image_paths, prompt.audio_features); + handle->model->prefill_with_media(prefill_tokens, delta_image_paths, slice_delta_audio()); } else if (has_images) { std::vector delta_image_paths; if (result.was_prefix) { @@ -573,7 +590,7 @@ PrefillResult do_prefill( } handle->model->prefill_with_images(prefill_tokens, delta_image_paths); } else if (has_audio) { - handle->model->prefill_with_audio(prefill_tokens, prompt.audio_features); + handle->model->prefill_with_audio(prefill_tokens, slice_delta_audio()); } else { handle->model->prefill(prefill_tokens, handle->model->get_prefill_chunk_size()); } @@ -692,12 +709,8 @@ int cactus_complete( while (common < limit && cache[common] == prompt.tokens[common]) common++; if (common < cache.size()) { CACTUS_LOG_WARN("complete", "KV cache diverges from new prompt at position " << common - << "/" << cache.size() << "; trimming and re-prefilling the divergent suffix"); - size_t kv_len = handle->model->get_cache_size(); - if (kv_len > common) { - handle->model->remove_thinking_tokens({{common, kv_len - common}}); - } - cache.resize(common); + << "/" << cache.size() << "; resetting cache for clean audio re-prefill"); + reset_cache(handle); } } diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 6da7ce58d..adff012c8 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -586,12 +586,13 @@ class Model { void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, const std::string& profile_file = ""); - void prefill_with_audio(const std::vector& tokens, const std::vector& audio_features, + void prefill_with_audio(const std::vector& tokens, + const std::vector>& audio_features_per_message, const std::string& profile_file = ""); void prefill_with_media(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features, + const std::vector>& audio_features_per_message, const std::string& profile_file = ""); uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, @@ -599,7 +600,8 @@ class Model { size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f); - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, + uint32_t decode_with_audio(const std::vector& tokens, + const std::vector>& audio_features_per_message, float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f, @@ -699,9 +701,10 @@ class Model { uint32_t argmax_last_logits(); void run_vision_encoder(const std::string& image_path); void run_audio_encoder(const std::vector& audio_features); + void run_audio_encoder_messages(const std::vector>& audio_features_per_message); bool run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features); + const std::vector>& audio_features_per_message); bool build_lm_encoder_outputs_dynamic_gemma4( const std::vector& tokens, std::map>& store_bytes, diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 2812a488d..cf6a318e3 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -793,6 +793,8 @@ size_t Model::component_output_tokens(const Component& comp, const std::string& size_t Model::run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, bool prepare_decode) { if (decode_route_ != DecodeRoute::CACHED_STEP || !encoder_ || !decoder_ || !decoder_prefill_) return 0; if (start_position != 0) return 0; + if (!load_component_graph(*decoder_prefill_)) return 0; + if (prefill_encoder_ && !load_component_graph(*prefill_encoder_)) return 0; if (!cache_states_compatible(*decoder_prefill_, *decoder_)) return 0; size_t component_tokens = component_chunk_tokens(*decoder_prefill_, "inputs_embeds"); if (component_tokens <= 1) return 0; @@ -950,82 +952,121 @@ void Model::run_vision_encoder(const std::string& image_path) { unload_component_graph(*vision_encoder_); } -void Model::run_audio_encoder(const std::vector& audio_features) { +void Model::run_audio_encoder_messages(const std::vector>& audio_features_per_message) { if (!audio_encoder_) return; + if (audio_features_per_message.empty()) return; if (!load_component_graph(*audio_encoder_)) { throw std::runtime_error("failed to load audio_encoder"); } - std::vector candidate_input_names = {"input_features", "audio_features"}; - bool wrote = false; + for (const std::string& logical : audio_encoder_->logical_outputs) { + media_features_.erase(logical); + media_feature_shapes_.erase(logical); + media_feature_precisions_.erase(logical); + } + for (const auto& mel : audio_features_per_message) { + if (mel.empty()) continue; + run_audio_encoder(mel); + } + audio_encoder_->graph->release_runtime_buffers(); + audio_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*audio_encoder_); +} + +void Model::run_audio_encoder(const std::vector& audio_features) { + if (!audio_encoder_) return; + const std::vector candidate_input_names = {"input_features", "audio_features"}; int feature_idx = -1; for (const auto& name : candidate_input_names) { int idx = input_index(*audio_encoder_, name); - if (idx < 0) continue; - size_t node_id = static_cast(audio_encoder_->runtime_input_node_ids[idx]); - const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); - auto& buf = audio_encoder_->input_buffers[idx]; - if (desc.precision == Precision::FP32) { - size_t n = std::min(audio_features.size() * sizeof(float), buf.size()); - std::memcpy(buf.data(), audio_features.data(), n); - if (n < buf.size()) std::memset(buf.data() + n, 0, buf.size() - n); - } else if (desc.precision == Precision::FP16) { - size_t n_elems = std::min(audio_features.size(), buf.size() / sizeof(__fp16)); + if (idx >= 0) { feature_idx = idx; break; } + } + if (feature_idx < 0) { + CACTUS_LOG_WARN("model", "audio_encoder has no input named input_features/audio_features; skipping"); + return; + } + const size_t feature_node = static_cast(audio_encoder_->runtime_input_node_ids[feature_idx]); + const auto& feature_desc = audio_encoder_->graph->get_output_buffer(feature_node); + const size_t mel_bins = feature_desc.shape.size() >= 3 + ? feature_desc.shape[2] + : static_cast(config_.audio_input_feat_size); + const size_t max_frames_per_chunk = feature_desc.shape.size() >= 2 ? feature_desc.shape[1] : 0; + if (max_frames_per_chunk == 0 || mel_bins == 0) { + CACTUS_LOG_WARN("model", "audio_encoder feature input has unexpected shape; skipping"); + return; + } + const size_t total_frames = audio_features.size() / mel_bins; + const size_t num_chunks = total_frames == 0 + ? 1 + : (total_frames + max_frames_per_chunk - 1) / max_frames_per_chunk; + + const int mask_idx = input_index(*audio_encoder_, "input_features_mask"); + + for (size_t chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) { + const size_t frame_start = chunk_idx * max_frames_per_chunk; + const size_t frames_in_chunk = frame_start >= total_frames + ? 0 + : std::min(max_frames_per_chunk, total_frames - frame_start); + const size_t chunk_feature_elems = frames_in_chunk * mel_bins; + const float* chunk_src = audio_features.data() + frame_start * mel_bins; + + auto& buf = audio_encoder_->input_buffers[feature_idx]; + if (feature_desc.precision == Precision::FP32) { + const size_t n_bytes = std::min(chunk_feature_elems * sizeof(float), buf.size()); + std::memcpy(buf.data(), chunk_src, n_bytes); + if (n_bytes < buf.size()) std::memset(buf.data() + n_bytes, 0, buf.size() - n_bytes); + } else if (feature_desc.precision == Precision::FP16) { + const size_t cap_elems = buf.size() / sizeof(__fp16); + const size_t n_elems = std::min(chunk_feature_elems, cap_elems); __fp16* dst = reinterpret_cast<__fp16*>(buf.data()); - for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast<__fp16>(audio_features[i]); - if (n_elems * sizeof(__fp16) < buf.size()) { - std::memset(buf.data() + n_elems * sizeof(__fp16), 0, buf.size() - n_elems * sizeof(__fp16)); + for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast<__fp16>(chunk_src[i]); + if (n_elems < cap_elems) { + std::memset(buf.data() + n_elems * sizeof(__fp16), 0, (cap_elems - n_elems) * sizeof(__fp16)); } } else { - size_t n_elems = std::min(audio_features.size(), buf.size()); + const size_t n_elems = std::min(chunk_feature_elems, buf.size()); int8_t* dst = reinterpret_cast(buf.data()); - for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast(audio_features[i]); + for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast(chunk_src[i]); if (n_elems < buf.size()) std::memset(buf.data() + n_elems, 0, buf.size() - n_elems); } - wrote = true; - feature_idx = idx; - break; - } - if (!wrote) { - CACTUS_LOG_WARN("model", "audio_encoder has no input named input_features/audio_features; skipping"); - return; - } - int mask_idx = input_index(*audio_encoder_, "input_features_mask"); - if (mask_idx >= 0) { - auto& mb = audio_encoder_->input_buffers[mask_idx]; - size_t feature_node = static_cast(audio_encoder_->runtime_input_node_ids[feature_idx]); - const auto& feature_desc = audio_encoder_->graph->get_output_buffer(feature_node); - size_t expected_mels = feature_desc.shape.size() >= 3 ? feature_desc.shape[2] : config_.audio_input_feat_size; - size_t valid_frames = expected_mels > 0 ? audio_features.size() / expected_mels : 0; - size_t mask_node = static_cast(audio_encoder_->runtime_input_node_ids[mask_idx]); - const auto& mask_desc = audio_encoder_->graph->get_output_buffer(mask_node); - const size_t elem = PrecisionTraits::size_of(mask_desc.precision); - const size_t cap = elem ? mb.size() / elem : 0; - const size_t n = std::min(cap, valid_frames); - for (size_t i = 0; i < n; ++i) { - switch (mask_desc.precision) { - case Precision::FP32: reinterpret_cast(mb.data())[i] = 1.0f; break; - case Precision::FP16: reinterpret_cast<__fp16*>(mb.data())[i] = static_cast<__fp16>(1.0f); break; - case Precision::INT8: reinterpret_cast(mb.data())[i] = 1; break; - default: reinterpret_cast(mb.data())[i] = 1; break; + + if (mask_idx >= 0) { + auto& mb = audio_encoder_->input_buffers[mask_idx]; + const size_t mask_node = static_cast(audio_encoder_->runtime_input_node_ids[mask_idx]); + const auto& mask_desc = audio_encoder_->graph->get_output_buffer(mask_node); + const size_t elem = PrecisionTraits::size_of(mask_desc.precision); + const size_t cap = elem ? mb.size() / elem : 0; + const size_t n = std::min(cap, frames_in_chunk); + for (size_t i = 0; i < n; ++i) { + switch (mask_desc.precision) { + case Precision::FP32: reinterpret_cast(mb.data())[i] = 1.0f; break; + case Precision::FP16: reinterpret_cast<__fp16*>(mb.data())[i] = static_cast<__fp16>(1.0f); break; + case Precision::INT8: reinterpret_cast(mb.data())[i] = 1; break; + default: reinterpret_cast(mb.data())[i] = 1; break; + } } + if (n < cap) std::memset(mb.data() + n * elem, 0, (cap - n) * elem); + } + + audio_encoder_->graph->execute(); + + for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { + const std::string& name = audio_encoder_->logical_outputs[i]; + const size_t node_id = static_cast(audio_encoder_->output_node_ids[i]); + const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); + void* ptr = audio_encoder_->graph->get_output(node_id); + auto& slot = media_features_[name]; + const size_t prev_bytes = slot.size(); + slot.resize(prev_bytes + desc.byte_size); + std::memcpy(slot.data() + prev_bytes, ptr, desc.byte_size); + auto shape_it = media_feature_shapes_.find(name); + if (shape_it == media_feature_shapes_.end() || shape_it->second.empty()) { + media_feature_shapes_[name] = desc.shape; + } else if (desc.shape.size() >= 2 && shape_it->second.size() == desc.shape.size()) { + shape_it->second[shape_it->second.size() - 2] += desc.shape[desc.shape.size() - 2]; + } + media_feature_precisions_[name] = desc.precision; } - if (n < cap) std::memset(mb.data() + n * elem, 0, (cap - n) * elem); } - audio_encoder_->graph->execute(); - for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { - const std::string& name = audio_encoder_->logical_outputs[i]; - size_t node_id = static_cast(audio_encoder_->output_node_ids[i]); - const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); - void* ptr = audio_encoder_->graph->get_output(node_id); - auto& slot = media_features_[name]; - slot.assign(desc.byte_size, 0); - std::memcpy(slot.data(), ptr, desc.byte_size); - media_feature_shapes_[name] = desc.shape; - media_feature_precisions_[name] = desc.precision; - } - audio_encoder_->graph->release_runtime_buffers(); - audio_encoder_->graph->release_all_weight_pages(); - unload_component_graph(*audio_encoder_); } uint32_t Model::argmax_last_logits() { @@ -1076,9 +1117,9 @@ void Model::prefill_with_images(const std::vector& tokens, } void Model::prefill_with_audio(const std::vector& tokens, - const std::vector& audio_features, + const std::vector>& audio_features_per_message, const std::string& profile_file) { - prefill_with_media(tokens, {}, audio_features, profile_file); + prefill_with_media(tokens, {}, audio_features_per_message, profile_file); } namespace { @@ -1373,15 +1414,22 @@ bool Model::build_lm_encoder_outputs_dynamic_gemma4( bool Model::run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features) { + const std::vector>& audio_features_per_message) { const bool have_images = !image_paths.empty() && vision_encoder_ != nullptr; - const bool have_audio = !audio_features.empty() && audio_encoder_ != nullptr; + bool any_audio = false; + for (const auto& mel : audio_features_per_message) { if (!mel.empty()) { any_audio = true; break; } } + const bool have_audio = any_audio && audio_encoder_ != nullptr; std::vector qwen_images; if (have_images) { if (!load_component_graph(*vision_encoder_)) { throw std::runtime_error("failed to load vision_encoder"); } + for (const std::string& logical : vision_encoder_->logical_outputs) { + media_features_.erase(logical); + media_feature_shapes_.erase(logical); + media_feature_precisions_.erase(logical); + } for (const auto& path : image_paths) { if (family_ == "lfm2_vl") { Lfm2VlImagePreprocessed prep = preprocess_lfm2_vl_image(path, config_); @@ -1471,9 +1519,15 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, const auto& desc = vision_encoder_->graph->get_output_buffer(node_id); void* ptr = vision_encoder_->graph->get_output(node_id); auto& slot = media_features_[name]; - slot.assign(desc.byte_size, 0); - std::memcpy(slot.data(), ptr, desc.byte_size); - media_feature_shapes_[name] = desc.shape; + const size_t prev_bytes = slot.size(); + slot.resize(prev_bytes + desc.byte_size); + std::memcpy(slot.data() + prev_bytes, ptr, desc.byte_size); + auto shape_it = media_feature_shapes_.find(name); + if (shape_it == media_feature_shapes_.end() || shape_it->second.empty()) { + media_feature_shapes_[name] = desc.shape; + } else if (desc.shape.size() >= 2 && shape_it->second.size() == desc.shape.size()) { + shape_it->second[shape_it->second.size() - 2] += desc.shape[desc.shape.size() - 2]; + } media_feature_precisions_[name] = desc.precision; } vision_encoder_->graph->release_runtime_buffers(); @@ -1483,7 +1537,7 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, } if (have_audio) { - run_audio_encoder(audio_features); + run_audio_encoder_messages(audio_features_per_message); } std::map> store_bytes; @@ -1660,17 +1714,19 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, void Model::prefill_with_media(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features, + const std::vector>& audio_features_per_message, const std::string& profile_file) { if (tokens.empty()) return; if (!image_paths.empty() && vision_encoder_ == nullptr) { throw std::runtime_error("Model bundle does not include a vision_encoder for image input"); } - if (!audio_features.empty() && audio_encoder_ == nullptr) { + bool any_audio = false; + for (const auto& mel : audio_features_per_message) { if (!mel.empty()) { any_audio = true; break; } } + if (any_audio && audio_encoder_ == nullptr) { throw std::runtime_error("Model bundle does not include an audio_encoder for audio input"); } const bool have_images = !image_paths.empty(); - const bool have_audio = !audio_features.empty(); + const bool have_audio = any_audio; if (!have_images && !have_audio) { prefill(tokens, get_prefill_chunk_size(), profile_file); return; @@ -1680,7 +1736,7 @@ void Model::prefill_with_media(const std::vector& tokens, lm_encoder_ != nullptr && decoder_prefill_chunk_ != nullptr && (vision_encoder_ != nullptr || audio_encoder_ != nullptr); if (can_chunk_prefill) { - if (run_chunk_prefill_path(tokens, image_paths, audio_features)) { + if (run_chunk_prefill_path(tokens, image_paths, audio_features_per_message)) { (void)profile_file; return; } @@ -1697,7 +1753,7 @@ void Model::prefill_with_media(const std::vector& tokens, } } if (have_audio) { - run_audio_encoder(audio_features); + run_audio_encoder_messages(audio_features_per_message); } std::string image_feature_name; @@ -1806,7 +1862,8 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* return result; } -uint32_t Model::decode_with_audio(const std::vector& tokens, const std::vector& /*mel_bins*/, +uint32_t Model::decode_with_audio(const std::vector& tokens, + const std::vector>& /*audio_features_per_message*/, float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, float min_p, float repetition_penalty, float* /*out_token_time_start*/, float* /*out_token_time_end*/) { diff --git a/cactus-engine/src/transcribe.cpp b/cactus-engine/src/transcribe.cpp index f64370d5d..b9be2e0ac 100644 --- a/cactus-engine/src/transcribe.cpp +++ b/cactus-engine/src/transcribe.cpp @@ -290,7 +290,7 @@ int cactus_transcribe( float tok_time_start = 0.0f; float tok_time_end = 0.0f; uint32_t next_token = handle->model->decode_with_audio( - tokens, audio_features, + tokens, std::vector>{audio_features}, options.temperature, options.top_p, options.top_k, "", &token_entropy, options.min_p, options.repetition_penalty, diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 42001d8e9..25736f5f6 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -416,16 +416,6 @@ inline AudioPreprocessResult preprocess_audio_for_gemma4( AudioPreprocessResult result; if (audio_samples.empty()) return result; - size_t max_soft = model_config.audio_soft_tokens; - if (max_soft > 0) { - size_t max_frames = (max_soft * 2 - 1) * 2 - 1; - size_t hop_length = 160; - size_t max_samples = max_frames * hop_length; - if (audio_samples.size() > max_samples) { - audio_samples.resize(max_samples); - } - } - size_t pad_amt = 320 - (audio_samples.size() % 320); if (pad_amt < 320) audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); @@ -442,8 +432,15 @@ inline AudioPreprocessResult preprocess_audio_for_gemma4( result.num_frames = mel.size() / mel_bins; result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); - size_t after_stage1 = (result.num_frames + 1) / 2; - result.num_soft_tokens = (after_stage1 + 1) / 2; + size_t max_soft = model_config.audio_soft_tokens; + size_t max_frames_per_chunk = max_soft > 0 ? (max_soft * 2 - 1) * 2 - 1 : 0; + if (max_soft > 0 && result.num_frames > max_frames_per_chunk) { + size_t num_chunks = (result.num_frames + max_frames_per_chunk - 1) / max_frames_per_chunk; + result.num_soft_tokens = num_chunks * max_soft; + } else { + size_t after_stage1 = (result.num_frames + 1) / 2; + result.num_soft_tokens = (after_stage1 + 1) / 2; + } return result; } diff --git a/local_integration_tests/native_runner.cpp b/local_integration_tests/native_runner.cpp index fc1cd5d96..8cba0ce08 100644 --- a/local_integration_tests/native_runner.cpp +++ b/local_integration_tests/native_runner.cpp @@ -929,7 +929,9 @@ int run_gemma_first_token(const Args& args) { if (tokens.empty()) throw std::runtime_error("first token probe produced empty prompt"); std::vector prefill_tokens(tokens.begin(), tokens.end() - 1); if (!prefill_tokens.empty()) { - model->prefill_with_media(prefill_tokens, args.image.empty() ? std::vector{} : std::vector{args.image}, audio_features); + std::vector> audio_messages; + if (!audio_features.empty()) audio_messages.push_back(std::move(audio_features)); + model->prefill_with_media(prefill_tokens, args.image.empty() ? std::vector{} : std::vector{args.image}, audio_messages); } uint32_t first = model->decode({tokens.back()}, 0.0f, 1.0f, 1); std::string text = tokenizer->decode({first}); From ef0ba0f24032f362b7df2c0e815135e0a3b06326 Mon Sep 17 00:00:00 2001 From: justinl66 Date: Thu, 21 May 2026 00:14:17 -0700 Subject: [PATCH 5/8] Transcribe ram improvement + vision optimization Signed-off-by: justinl66 --- cactus-engine/src/complete.cpp | 11 ++++------- cactus-engine/src/model.cpp | 1 + cactus-engine/tests/asr.cpp | 2 +- cactus-engine/tests/test_stt.cpp | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 3021facb4..555a40276 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -544,14 +544,11 @@ PrefillResult do_prefill( auto slice_delta_audio = [&]() -> std::vector> { if (!result.was_prefix) return prompt.audio_features; - const size_t cached_user_messages = handle->processed_images.size(); + const size_t cached_msg_count = handle->processed_images.size(); size_t cached_audio_count = 0; - size_t seen_user = 0; - for (const auto& msg : prompt.messages) { - if (msg.role != "user") continue; - if (seen_user >= cached_user_messages) break; - if (!msg.audio.empty()) cached_audio_count++; - seen_user++; + for (size_t i = 0; i < cached_msg_count && i < prompt.messages.size(); ++i) { + const auto& msg = prompt.messages[i]; + if (msg.role == "user" && !msg.audio.empty()) cached_audio_count++; } cached_audio_count = std::min(cached_audio_count, prompt.audio_features.size()); return std::vector>( diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index cf6a318e3..e3e1702d5 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -1415,6 +1415,7 @@ bool Model::build_lm_encoder_outputs_dynamic_gemma4( bool Model::run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, const std::vector>& audio_features_per_message) { + if (cache_total_seq_len_ > 0) return false; const bool have_images = !image_paths.empty() && vision_encoder_ != nullptr; bool any_audio = false; for (const auto& mel : audio_features_per_message) { if (!mel.empty()) { any_audio = true; break; } } diff --git a/cactus-engine/tests/asr.cpp b/cactus-engine/tests/asr.cpp index 494a62e1d..7cc10455a 100644 --- a/cactus-engine/tests/asr.cpp +++ b/cactus-engine/tests/asr.cpp @@ -52,7 +52,7 @@ int main(int argc, char** argv) { std::cout << "Model loaded.\n"; std::ostringstream opts; - opts << "{\"language\":\"" << language << "\",\"telemetry_enabled\":false}"; + opts << "{\"language\":\"" << language << "\",\"telemetry_enabled\":false,\"auto_handoff\":false,\"max_tokens\":512}"; constexpr size_t kBufSize = 1 << 16; std::string response(kBufSize, '\0'); diff --git a/cactus-engine/tests/test_stt.cpp b/cactus-engine/tests/test_stt.cpp index d22396a6e..55f9b2ca0 100644 --- a/cactus-engine/tests/test_stt.cpp +++ b/cactus-engine/tests/test_stt.cpp @@ -35,7 +35,7 @@ bool test_transcription() { Timer timer; int rc = cactus_transcribe(model, audio_path.c_str(), nullptr, response, sizeof(response), - R"({"max_tokens": 200, "telemetry_enabled": false})", + R"({"max_tokens": 200, "telemetry_enabled": false, "auto_handoff": false})", nullptr, nullptr, nullptr, 0); double elapsed = timer.elapsed_ms(); @@ -105,7 +105,7 @@ bool test_transcription_pcm() { Timer timer; int rc = cactus_transcribe(model, nullptr, nullptr, response, sizeof(response), - R"({"max_tokens": 200, "telemetry_enabled": false})", + R"({"max_tokens": 200, "telemetry_enabled": false, "auto_handoff": false})", nullptr, nullptr, pcm_data.data(), pcm_data.size()); double elapsed = timer.elapsed_ms(); From 765eeec193a7b62a85bbf63c0b5532cc0ac2458b Mon Sep 17 00:00:00 2001 From: justinl66 Date: Thu, 21 May 2026 09:16:28 -0700 Subject: [PATCH 6/8] Cleanup Test Files Signed-off-by: justinl66 --- .gitignore | 6 - local_integration_tests/CMakeLists.txt | 62 - local_integration_tests/README.md | 43 - local_integration_tests/conftest.py | 156 --- local_integration_tests/native_runner.cpp | 1110 ----------------- local_integration_tests/test_modalities.py | 210 ---- .../gemma4_multimodal/BACKGROUND.md | 53 - .../gemma4_multimodal/EXPERIMENTS.md | 80 -- .../gemma4_multimodal/FIXES.md | 52 - .../gemma4_multimodal/HYPOTHESES.md | 71 -- .../gemma4_multimodal/PLAN.md | 29 - .../gemma4_multimodal/README.md | 23 - .../gemma4_multimodal/SUMMARY.md | 9 - .../gemma4_multimodal_v2/00_INDEX.md | 48 - .../gemma4_multimodal_v2/01_BACKGROUND.md | 48 - .../gemma4_multimodal_v2/02_QUESTIONS.md | 41 - .../gemma4_multimodal_v2/03_HYPOTHESES.md | 66 - .../gemma4_multimodal_v2/04_EXPERIMENTS.md | 356 ------ .../gemma4_multimodal_v2/05_DECISIONS.md | 118 -- .../gemma4_multimodal_v2/06_FIX_LOG.md | 156 --- .../gemma4_multimodal_v2/07_ACCEPTANCE.md | 50 - .../gemma4_multimodal_v2/08_HANDOFF.md | 44 - .../09_STATUS_FOR_NEXT_AGENT.md | 74 -- .../10_CONTINUATION_PLAN.md | 58 - .../gemma4_multimodal_v2/scripts/run_probe.py | 785 ------------ 25 files changed, 3748 deletions(-) delete mode 100644 local_integration_tests/CMakeLists.txt delete mode 100644 local_integration_tests/README.md delete mode 100644 local_integration_tests/conftest.py delete mode 100644 local_integration_tests/native_runner.cpp delete mode 100644 local_integration_tests/test_modalities.py delete mode 100644 local_investigations/gemma4_multimodal/BACKGROUND.md delete mode 100644 local_investigations/gemma4_multimodal/EXPERIMENTS.md delete mode 100644 local_investigations/gemma4_multimodal/FIXES.md delete mode 100644 local_investigations/gemma4_multimodal/HYPOTHESES.md delete mode 100644 local_investigations/gemma4_multimodal/PLAN.md delete mode 100644 local_investigations/gemma4_multimodal/README.md delete mode 100644 local_investigations/gemma4_multimodal/SUMMARY.md delete mode 100644 local_investigations/gemma4_multimodal_v2/00_INDEX.md delete mode 100644 local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md delete mode 100644 local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md delete mode 100644 local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md delete mode 100644 local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md delete mode 100644 local_investigations/gemma4_multimodal_v2/05_DECISIONS.md delete mode 100644 local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md delete mode 100644 local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md delete mode 100644 local_investigations/gemma4_multimodal_v2/08_HANDOFF.md delete mode 100644 local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md delete mode 100644 local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md delete mode 100644 local_investigations/gemma4_multimodal_v2/scripts/run_probe.py diff --git a/.gitignore b/.gitignore index 7795b9ebe..e18280031 100644 --- a/.gitignore +++ b/.gitignore @@ -47,9 +47,3 @@ vision_activations.json gemma-4-eap-extras/ .githooks/ ruy - -# Local integration/investigation outputs -local_integration_tests/.tmp/ -local_integration_tests/__pycache__/ -local_investigations/**/__pycache__/ -local_investigations/**/artifacts/ diff --git a/local_integration_tests/CMakeLists.txt b/local_integration_tests/CMakeLists.txt deleted file mode 100644 index 317318503..000000000 --- a/local_integration_tests/CMakeLists.txt +++ /dev/null @@ -1,62 +0,0 @@ -cmake_minimum_required(VERSION 3.16) -project(CactusLocalIntegrationTests LANGUAGES CXX) - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED True) - -if(NOT CACTUS_REPO_ROOT) - get_filename_component(CACTUS_REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.." ABSOLUTE) -endif() - -set(CACTUS_LIB "${CACTUS_REPO_ROOT}/cactus/build/libcactus.a") -if(NOT EXISTS "${CACTUS_LIB}") - message(FATAL_ERROR "Missing ${CACTUS_LIB}; run ./cactus/build.sh first") -endif() - -add_executable(integration_runner native_runner.cpp) -target_link_libraries(integration_runner PRIVATE "${CACTUS_LIB}") -target_include_directories(integration_runner PRIVATE - "${CACTUS_REPO_ROOT}/cactus-engine" - "${CACTUS_REPO_ROOT}/cactus-engine/src" - "${CACTUS_REPO_ROOT}/cactus-graph" - "${CACTUS_REPO_ROOT}/cactus-kernels" - "${CACTUS_REPO_ROOT}/cactus-kernels/src" - "${CACTUS_REPO_ROOT}/cactus-kernels/libs" -) - -find_package(Threads REQUIRED) -target_link_libraries(integration_runner PRIVATE Threads::Threads) - -if(APPLE) - find_library(COREML_FRAMEWORK CoreML REQUIRED) - find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) - find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) - find_library(SECURITY_FRAMEWORK Security REQUIRED) - find_library(SYSTEMCONFIGURATION_FRAMEWORK SystemConfiguration REQUIRED) - find_library(CFNETWORK_FRAMEWORK CFNetwork REQUIRED) - target_link_libraries(integration_runner PRIVATE - "${ACCELERATE_FRAMEWORK}" - "${COREML_FRAMEWORK}" - "${FOUNDATION_FRAMEWORK}" - "${SECURITY_FRAMEWORK}" - "${SYSTEMCONFIGURATION_FRAMEWORK}" - "${CFNETWORK_FRAMEWORK}" - ) - set(CACTUS_CURL_ROOT "${CACTUS_REPO_ROOT}/cactus-engine/libs/curl") - set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") - if(EXISTS "${CACTUS_CURL_LIBRARY_MACOS}") - target_link_libraries(integration_runner PRIVATE "${CACTUS_CURL_LIBRARY_MACOS}") - target_include_directories(integration_runner PRIVATE "${CACTUS_CURL_ROOT}/include") - else() - find_library(CURL_LIBRARY NAMES curl) - if(CURL_LIBRARY) - target_link_libraries(integration_runner PRIVATE "${CURL_LIBRARY}") - endif() - endif() -else() - find_package(CURL) - if(CURL_FOUND) - target_link_libraries(integration_runner PRIVATE ${CURL_LIBRARIES}) - target_include_directories(integration_runner PRIVATE ${CURL_INCLUDE_DIRS}) - endif() -endif() diff --git a/local_integration_tests/README.md b/local_integration_tests/README.md deleted file mode 100644 index c6442606a..000000000 --- a/local_integration_tests/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Local integration tests - -These tests are intentionally local and uncommitted until the suite is ready to become part of the repository. - -## Requirements - -- Run from the repository root. -- Use existing converted model artifacts only. -- Set `CACTUS_INTEGRATION_WEIGHTS` to override model discovery. -- Default discovery checks `../cactus/weights`, then `./weights`. - -## Commands - -```bash -PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -m smoke -PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -m full -PYTHONPATH=python ./venv/bin/python -m pytest local_integration_tests -k parakeet -``` - -The pytest session builds `./cactus/build.sh` once and compiles `local_integration_tests/.tmp/build/integration_runner`. - -## Coverage - -- Gemma4 E2B: text and multiturn reuse with the matrix Cactus artifact, `gemma-4-e2b-it-sharedkv`. -- Gemma4 E2B: image, audio, and image+audio combined with the latest local multimodal artifact, preferring `gemma-4-e2b-it-alt-fixed-rowtuned-mm`. -- Qwen3-VL: text and long-prompt chunked prefill smoke. -- Qwen3-VL: image with the matrix Cactus artifact, `qwen3-vl-2b-instruct-reconvert`. -- LFM2-VL: text, image, long-prompt chunked prefill smoke. -- Parakeet TDT: file and PCM transcription. -- Whisper small: full-mode file and PCM transcription when a component-backed local bundle is available. - -## Current local result - -```bash -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s -# 7 passed, 2 deselected - -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -k parakeet -q -s -# 1 passed, 8 deselected - -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m full -q -s -# 2 skipped, 7 deselected -``` diff --git a/local_integration_tests/conftest.py b/local_integration_tests/conftest.py deleted file mode 100644 index adcfe06a4..000000000 --- a/local_integration_tests/conftest.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -import json -import os -import subprocess -from pathlib import Path -from typing import Any - -import pytest - - -REPO_ROOT = Path(__file__).resolve().parents[1] -TMP_DIR = REPO_ROOT / "local_integration_tests" / ".tmp" -ASSET_DIR = REPO_ROOT / "cactus-engine" / "tests" / "assets" - - -def pytest_configure(config: pytest.Config) -> None: - config.addinivalue_line("markers", "smoke: fast local integration coverage") - config.addinivalue_line("markers", "full: slower coverage for every available key model") - - -def _weights_root() -> Path: - explicit = os.environ.get("CACTUS_INTEGRATION_WEIGHTS") - candidates = [explicit] if explicit else [] - candidates.extend([str(REPO_ROOT.parent / "cactus" / "weights"), str(REPO_ROOT / "weights")]) - for candidate in candidates: - if not candidate: - continue - path = Path(candidate).expanduser().resolve() - if path.exists(): - return path - return Path(candidates[0]).expanduser().resolve() - - -@pytest.fixture(scope="session") -def weights_root() -> Path: - root = _weights_root() - if not root.exists(): - pytest.fail(f"weights root does not exist: {root}") - return root - - -@pytest.fixture(scope="session") -def assets() -> dict[str, Path]: - result = { - "audio": ASSET_DIR / "test.wav", - "image_monkey": ASSET_DIR / "test_monkey.png", - "image_thing": ASSET_DIR / "test_thing.png", - } - missing = [str(path) for path in result.values() if not path.exists()] - if missing: - pytest.fail("missing integration fixtures: " + ", ".join(missing)) - return result - - -@pytest.fixture(scope="session") -def runner() -> Path: - env = os.environ.copy() - env["CACTUS_NO_CLOUD_TELE"] = "1" - subprocess.run([str(REPO_ROOT / "cactus" / "build.sh")], cwd=REPO_ROOT, env=env, check=True) - - build_dir = TMP_DIR / "build" - build_dir.mkdir(parents=True, exist_ok=True) - subprocess.run( - [ - "cmake", - "-S", - str(REPO_ROOT / "local_integration_tests"), - "-B", - str(build_dir), - f"-DCACTUS_REPO_ROOT={REPO_ROOT}", - "-DCMAKE_BUILD_TYPE=Release", - ], - cwd=REPO_ROOT, - check=True, - ) - subprocess.run( - ["cmake", "--build", str(build_dir), "--target", "integration_runner", "-j", str(os.cpu_count() or 4)], - cwd=REPO_ROOT, - check=True, - ) - binary = build_dir / "integration_runner" - if not binary.exists(): - pytest.fail(f"integration runner was not built: {binary}") - return binary - - -def _is_transpiled_bundle(path: Path) -> bool: - return (path / "components" / "manifest.json").exists() or (path / "manifest.json").exists() - - -def model_path( - weights_root: Path, - *candidates: str, - required: bool = True, - transpiled: bool = True, - native_config: bool = False, -) -> Path | None: - for name in candidates: - path = weights_root / name - if not path.exists(): - continue - if transpiled and not _is_transpiled_bundle(path): - continue - if native_config and not (path / "config.txt").exists(): - continue - if path.exists(): - return path - if required: - suffixes = [] - if transpiled: - suffixes.append("components/manifest.json") - if native_config: - suffixes.append("config.txt") - suffix = " with " + " and ".join(suffixes) if suffixes else "" - pytest.fail("missing required model artifact" + suffix + "; tried: " + ", ".join(candidates)) - pytest.skip("optional model artifact not available; tried: " + ", ".join(candidates)) - return None - - -def run_runner(runner: Path, *args: object, timeout: int = 180) -> dict[str, Any]: - env = os.environ.copy() - env["CACTUS_NO_CLOUD_TELE"] = "1" - env.pop("CACTUS_KV_CACHE_FP16", None) - completed = subprocess.run( - [str(runner), *(str(arg) for arg in args)], - cwd=REPO_ROOT, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - check=False, - ) - if completed.returncode != 0: - pytest.fail( - f"integration runner failed with {completed.returncode}\n" - f"stdout:\n{completed.stdout}\n" - f"stderr:\n{completed.stderr}" - ) - try: - return json.loads(completed.stdout) - except json.JSONDecodeError as exc: - pytest.fail(f"runner did not return JSON: {exc}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}") - - -def assert_completion(result: dict[str, Any]) -> None: - response = str(result.get("response") or result.get("text") or "").strip() - token_count = int(result.get("completion_tokens") or result.get("tokens_predicted") or result.get("decode_tokens") or 0) - assert response or token_count > 0, result - - -def assert_transcript(result: dict[str, Any], fragments: tuple[str, ...] = ("hello", "testing", "voice")) -> None: - transcript = str(result.get("response") or result.get("transcript") or "").strip().lower() - assert len(transcript) > 5, result - assert any(fragment in transcript for fragment in fragments), transcript diff --git a/local_integration_tests/native_runner.cpp b/local_integration_tests/native_runner.cpp deleted file mode 100644 index 8cba0ce08..000000000 --- a/local_integration_tests/native_runner.cpp +++ /dev/null @@ -1,1110 +0,0 @@ -#include "cactus_engine.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define private public -#include "engine.h" -#undef private -#include "cactus_kernels.h" -#include "stb_image_resize2.h" -#include "utils.h" -#include "wav.h" - -namespace { - -constexpr size_t kResponseBufferSize = 1 << 20; -constexpr uint64_t kFnvOffset = 1469598103934665603ull; -constexpr uint64_t kFnvPrime = 1099511628211ull; - -struct Args { - std::string command; - std::string model; - std::string prompt; - std::string image; - std::string audio; - std::string messages_json; - int max_tokens = 32; -}; - -std::string arg_value(int& i, int argc, char** argv) { - if (i + 1 >= argc) throw std::runtime_error(std::string("missing value for ") + argv[i]); - return argv[++i]; -} - -Args parse_args(int argc, char** argv) { - if (argc < 2) throw std::runtime_error("usage: integration_runner [options]"); - Args args; - args.command = argv[1]; - for (int i = 2; i < argc; ++i) { - std::string flag = argv[i]; - if (flag == "--model") args.model = arg_value(i, argc, argv); - else if (flag == "--prompt") args.prompt = arg_value(i, argc, argv); - else if (flag == "--image") args.image = arg_value(i, argc, argv); - else if (flag == "--audio") args.audio = arg_value(i, argc, argv); - else if (flag == "--messages-json") args.messages_json = arg_value(i, argc, argv); - else if (flag == "--max-tokens") args.max_tokens = std::max(0, std::stoi(arg_value(i, argc, argv))); - else throw std::runtime_error("unknown flag: " + flag); - } - if (args.model.empty()) throw std::runtime_error("--model is required"); - return args; -} - -std::string escape_json(const std::string& value) { - std::ostringstream out; - for (char ch : value) { - switch (ch) { - case '\\': out << "\\\\"; break; - case '"': out << "\\\""; break; - case '\n': out << "\\n"; break; - case '\r': out << "\\r"; break; - case '\t': out << "\\t"; break; - default: - if (static_cast(ch) < 0x20) { - out << "\\u00"; - const char* hex = "0123456789abcdef"; - out << hex[(ch >> 4) & 0x0f] << hex[ch & 0x0f]; - } else { - out << ch; - } - } - } - return out.str(); -} - -template -std::string json_array(const std::vector& values, size_t limit = std::numeric_limits::max()) { - std::ostringstream out; - out << "["; - const size_t n = std::min(values.size(), limit); - for (size_t i = 0; i < n; ++i) { - if (i) out << ","; - out << values[i]; - } - out << "]"; - return out.str(); -} - -std::string json_float_array(const std::vector& values, size_t limit = std::numeric_limits::max()) { - std::ostringstream out; - out << "["; - const size_t n = std::min(values.size(), limit); - for (size_t i = 0; i < n; ++i) { - if (i) out << ","; - if (std::isfinite(values[i])) out << values[i]; - else out << "null"; - } - out << "]"; - return out.str(); -} - -uint64_t fnv1a_bytes(const void* data, size_t size) { - const auto* bytes = static_cast(data); - uint64_t hash = kFnvOffset; - for (size_t i = 0; i < size; ++i) { - hash ^= static_cast(bytes[i]); - hash *= kFnvPrime; - } - return hash; -} - -std::string hex64(uint64_t value) { - std::ostringstream out; - out << std::hex << std::setw(16) << std::setfill('0') << value; - return out.str(); -} - -template -std::vector first_values(const std::vector& values, size_t n) { - return std::vector(values.begin(), values.begin() + std::min(values.size(), n)); -} - -std::vector last_values(const std::vector& values, size_t n) { - if (values.size() <= n) return values; - return std::vector(values.end() - static_cast(n), values.end()); -} - -int run_e002_tokens(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for token probe: " + args.model); - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() ? "Respond briefly." : args.prompt; - message.images.push_back(args.image); - std::vector messages{message}; - - const std::string full_prompt = tokenizer->format_chat_prompt(messages, true); - const std::string no_generation_prompt = tokenizer->format_chat_prompt(messages, false); - const std::vector tokens = tokenizer->encode(full_prompt); - const std::vector prefix_tokens = tokenizer->encode(no_generation_prompt); - const uint32_t image_token_id = model->get_config().image_token_id ? model->get_config().image_token_id : tokenizer->get_image_token_id(); - - size_t image_start = tokens.size(); - size_t image_count = 0; - for (size_t i = 0; i < tokens.size(); ++i) { - if (tokens[i] == image_token_id) { - if (image_count == 0) image_start = i; - ++image_count; - } - } - - std::ostringstream out; - out << "{" - << "\"token_count\":" << tokens.size() << "," - << "\"assistant_generation_start\":" << prefix_tokens.size() << "," - << "\"image_token_id\":" << image_token_id << "," - << "\"image_token_start\":" << (image_count ? static_cast(image_start) : -1) << "," - << "\"image_token_count\":" << image_count << "," - << "\"first_20\":" << json_array(first_values(tokens, 20)) << "," - << "\"last_20\":" << json_array(last_values(tokens, 20)) << "," - << "\"input_ids\":" << json_array(tokens) << "," - << "\"prompt_prefix\":\"" << escape_json(full_prompt.substr(0, 320)) << "\"" - << "}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_preprocess(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - cactus::engine::Config config; - if (!config.from_json(args.model + "/config.txt")) { - throw std::runtime_error("failed to read config.txt for preprocess probe: " + args.model); - } - auto prep = cactus::engine::preprocess_gemma4_image(args.image, config); - double sum = 0.0; - float min_v = prep.pixel_values.empty() ? 0.0f : prep.pixel_values[0]; - float max_v = min_v; - for (float v : prep.pixel_values) { - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - size_t valid_positions = 0; - for (size_t i = 0; i + 1 < prep.pixel_position_ids.size(); i += 2) { - if (prep.pixel_position_ids[i] >= 0 && prep.pixel_position_ids[i + 1] >= 0) ++valid_positions; - } - std::vector first_positions; - const size_t position_rows = std::min(16, prep.pixel_position_ids.size() / 2); - first_positions.reserve(position_rows * 2); - for (size_t i = 0; i < position_rows * 2; ++i) first_positions.push_back(prep.pixel_position_ids[i]); - - std::ostringstream out; - out << std::setprecision(9) - << "{" - << "\"pixel_values_shape\":[1," << prep.max_patches << "," << prep.patch_dim << "]," - << "\"pixel_position_ids_shape\":[1," << prep.max_patches << ",2]," - << "\"num_patches\":" << prep.num_patches << "," - << "\"max_patches\":" << prep.max_patches << "," - << "\"patch_dim\":" << prep.patch_dim << "," - << "\"pixel_values_min\":" << min_v << "," - << "\"pixel_values_max\":" << max_v << "," - << "\"pixel_values_mean\":" << (prep.pixel_values.empty() ? 0.0 : sum / static_cast(prep.pixel_values.size())) << "," - << "\"pixel_values_hash\":\"" << hex64(fnv1a_bytes(prep.pixel_values.data(), prep.pixel_values.size() * sizeof(float))) << "\"," - << "\"pixel_position_ids_hash\":\"" << hex64(fnv1a_bytes(prep.pixel_position_ids.data(), prep.pixel_position_ids.size() * sizeof(int64_t))) << "\"," - << "\"valid_position_count\":" << valid_positions << "," - << "\"first_pixel_values\":" << json_array(first_values(prep.pixel_values, 32)) << "," - << "\"first_position_rows_flat\":" << json_array(first_positions) << "," - << "\"pixel_values\":" << json_array(prep.pixel_values) << "," - << "\"pixel_position_ids\":" << json_array(prep.pixel_position_ids) - << "}\n"; - std::cout << out.str(); - return 0; -} - -struct E003Variant { - std::string name; - std::vector resized; - std::vector pixel_values; -}; - -void resize_float_default(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { - std::vector src_float(static_cast(width) * height * 3); - for (size_t i = 0; i < src_float.size(); ++i) src_float[i] = static_cast(raw[i]); - if (!stbir_resize_float_linear(src_float.data(), width, height, 0, resized.data(), target_w, target_h, 0, STBIR_RGB)) { - throw std::runtime_error("stbir_resize_float_linear failed"); - } -} - -void resize_uint8_default(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { - std::vector tmp(static_cast(target_w) * target_h * 3); - if (!stbir_resize_uint8_linear(raw, width, height, 0, tmp.data(), target_w, target_h, 0, STBIR_RGB)) { - throw std::runtime_error("stbir_resize_uint8_linear failed"); - } - for (size_t i = 0; i < tmp.size(); ++i) resized[i] = static_cast(tmp[i]); -} - -void resize_float_triangle(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { - std::vector src_float(static_cast(width) * height * 3); - for (size_t i = 0; i < src_float.size(); ++i) src_float[i] = static_cast(raw[i]); - STBIR_RESIZE resize; - stbir_resize_init(&resize, src_float.data(), width, height, 0, resized.data(), target_w, target_h, 0, STBIR_RGB, STBIR_TYPE_FLOAT); - stbir_set_filters(&resize, STBIR_FILTER_TRIANGLE, STBIR_FILTER_TRIANGLE); - if (!stbir_resize_extended(&resize)) throw std::runtime_error("triangle float resize failed"); -} - -void resize_uint8_triangle(const unsigned char* raw, int width, int height, std::vector& resized, int target_w, int target_h) { - std::vector tmp(static_cast(target_w) * target_h * 3); - STBIR_RESIZE resize; - stbir_resize_init(&resize, raw, width, height, 0, tmp.data(), target_w, target_h, 0, STBIR_RGB, STBIR_TYPE_UINT8); - stbir_set_filters(&resize, STBIR_FILTER_TRIANGLE, STBIR_FILTER_TRIANGLE); - if (!stbir_resize_extended(&resize)) throw std::runtime_error("triangle uint8 resize failed"); - for (size_t i = 0; i < tmp.size(); ++i) resized[i] = static_cast(tmp[i]); -} - -std::vector pack_gemma4_patches(const std::vector& resized, int target_w, int target_h, int patch_size, - size_t max_patches, size_t patch_dim, float rescale_factor) { - const int patch_h = target_h / patch_size; - const int patch_w = target_w / patch_size; - std::vector pixel_values(max_patches * patch_dim, 0.0f); - for (int py = 0; py < patch_h; ++py) { - for (int px = 0; px < patch_w; ++px) { - const size_t patch_idx = static_cast(py) * patch_w + px; - float* dst = pixel_values.data() + patch_idx * patch_dim; - for (int y = 0; y < patch_size; ++y) { - const int img_y = py * patch_size + y; - for (int x = 0; x < patch_size; ++x) { - const int img_x = px * patch_size + x; - const size_t src_off = (static_cast(img_y) * target_w + img_x) * 3; - const size_t dst_off = (static_cast(y) * patch_size + x) * 3; - dst[dst_off + 0] = resized[src_off + 0] * rescale_factor; - dst[dst_off + 1] = resized[src_off + 1] * rescale_factor; - dst[dst_off + 2] = resized[src_off + 2] * rescale_factor; - } - } - } - } - return pixel_values; -} - -void write_variant_json(std::ostream& out, const E003Variant& variant, bool comma) { - double sum = 0.0; - float min_v = variant.pixel_values.empty() ? 0.0f : variant.pixel_values[0]; - float max_v = min_v; - for (float v : variant.pixel_values) { - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - float resized_min = variant.resized.empty() ? 0.0f : variant.resized[0]; - float resized_max = resized_min; - for (float v : variant.resized) { - resized_min = std::min(resized_min, v); - resized_max = std::max(resized_max, v); - } - if (comma) out << ","; - out << "\"" << variant.name << "\":{" - << "\"resized_min\":" << resized_min << "," - << "\"resized_max\":" << resized_max << "," - << "\"first_resized_values\":" << json_array(first_values(variant.resized, 32)) << "," - << "\"pixel_values_min\":" << min_v << "," - << "\"pixel_values_max\":" << max_v << "," - << "\"pixel_values_mean\":" << (variant.pixel_values.empty() ? 0.0 : sum / static_cast(variant.pixel_values.size())) << "," - << "\"first_pixel_values\":" << json_array(first_values(variant.pixel_values, 32)) << "," - << "\"pixel_values\":" << json_array(variant.pixel_values) - << "}"; -} - -std::string precision_name(Precision precision) { - switch (precision) { - case Precision::FP32: return "FP32"; - case Precision::FP16: return "FP16"; - case Precision::INT8: return "INT8"; - default: return "UNKNOWN"; - } -} - -std::string op_type_name(OpType op) { - switch (op) { - case OpType::INPUT: return "INPUT"; - case OpType::PRECISION_CAST: return "PRECISION_CAST"; - case OpType::ADD: return "ADD"; - case OpType::ADD_CLIPPED: return "ADD_CLIPPED"; - case OpType::SUBTRACT: return "SUBTRACT"; - case OpType::MULTIPLY: return "MULTIPLY"; - case OpType::DIVIDE: return "DIVIDE"; - case OpType::ABS: return "ABS"; - case OpType::POW: return "POW"; - case OpType::FLATTEN: return "FLATTEN"; - case OpType::VIEW: return "VIEW"; - case OpType::MATMUL: return "MATMUL"; - case OpType::TRANSPOSE: return "TRANSPOSE"; - case OpType::RESHAPE: return "RESHAPE"; - case OpType::SLICE: return "SLICE"; - case OpType::GATHER: return "GATHER"; - case OpType::EMBEDDING: return "EMBEDDING"; - case OpType::BILINEAR_INTERPOLATION: return "BILINEAR_INTERPOLATION"; - case OpType::SUM: return "SUM"; - case OpType::MEAN: return "MEAN"; - case OpType::VARIANCE: return "VARIANCE"; - case OpType::MIN: return "MIN"; - case OpType::MAX: return "MAX"; - case OpType::CUMSUM: return "CUMSUM"; - case OpType::RMS_NORM: return "RMS_NORM"; - case OpType::ROPE: return "ROPE"; - case OpType::ROPE_GPTJ: return "ROPE_GPTJ"; - case OpType::SOFTMAX: return "SOFTMAX"; - case OpType::ATTENTION: return "ATTENTION"; - case OpType::ATTENTION_INT8_HYBRID: return "ATTENTION_INT8_HYBRID"; - case OpType::REL_POS_BIAS: return "REL_POS_BIAS"; - case OpType::CONV1D_CAUSAL: return "CONV1D_CAUSAL"; - case OpType::CONV1D_K3: return "CONV1D_K3"; - case OpType::CONV1D_K7S3: return "CONV1D_K7S3"; - case OpType::CONV1D: return "CONV1D"; - case OpType::CONV1D_SAME_DEPTHWISE_K9: return "CONV1D_SAME_DEPTHWISE_K9"; - case OpType::CONV1D_POINTWISE: return "CONV1D_POINTWISE"; - case OpType::CONV2D_K3S2P1: return "CONV2D_K3S2P1"; - case OpType::CONV2D_DEPTHWISE_K3S2P1: return "CONV2D_DEPTHWISE_K3S2P1"; - case OpType::CONV2D_POINTWISE_1X1: return "CONV2D_POINTWISE_1X1"; - case OpType::GLU: return "GLU"; - case OpType::BATCHNORM: return "BATCHNORM"; - case OpType::SCALAR_ADD: return "SCALAR_ADD"; - case OpType::SCALAR_SUBTRACT: return "SCALAR_SUBTRACT"; - case OpType::SCALAR_MULTIPLY: return "SCALAR_MULTIPLY"; - case OpType::SCALAR_DIVIDE: return "SCALAR_DIVIDE"; - case OpType::SCALAR_EXP: return "SCALAR_EXP"; - case OpType::SCALAR_SQRT: return "SCALAR_SQRT"; - case OpType::SCALAR_COS: return "SCALAR_COS"; - case OpType::SCALAR_SIN: return "SCALAR_SIN"; - case OpType::SCALAR_LOG: return "SCALAR_LOG"; - case OpType::RELU: return "RELU"; - case OpType::SILU: return "SILU"; - case OpType::GELU: return "GELU"; - case OpType::GELU_ERF: return "GELU_ERF"; - case OpType::SIGMOID: return "SIGMOID"; - case OpType::TANH: return "TANH"; - case OpType::SAMPLE: return "SAMPLE"; - case OpType::CONCAT: return "CONCAT"; - case OpType::CAT: return "CAT"; - case OpType::SCATTER_TOPK: return "SCATTER_TOPK"; - case OpType::TOPK: return "TOPK"; - case OpType::LAYERNORM: return "LAYERNORM"; - case OpType::GROUPNORM: return "GROUPNORM"; - case OpType::MOE_LAYER: return "MOE_LAYER"; - case OpType::INDEX: return "INDEX"; - case OpType::PERSISTENT: return "PERSISTENT"; - case OpType::LSTM_CELL: return "LSTM_CELL"; - case OpType::GATED_DELTANET_DECODE: return "GATED_DELTANET_DECODE"; - case OpType::GATED_DELTANET_PREFILL: return "GATED_DELTANET_PREFILL"; - case OpType::STFT: return "STFT"; - case OpType::ALTUP_PREDICT: return "ALTUP_PREDICT"; - case OpType::ALTUP_CORRECT: return "ALTUP_CORRECT"; - case OpType::GAUSSIAN_TOPK: return "GAUSSIAN_TOPK"; - case OpType::MAXPOOL1D: return "MAXPOOL1D"; - case OpType::BILSTM_SEQUENCE: return "BILSTM_SEQUENCE"; - case OpType::LEAKY_RELU: return "LEAKY_RELU"; - case OpType::CONV2D_K3S1P1: return "CONV2D_K3S1P1"; - case OpType::STATS_POOL: return "STATS_POOL"; - case OpType::WEIGHTED_STATS_POOL: return "WEIGHTED_STATS_POOL"; - case OpType::KV_CACHE_STATE: return "KV_CACHE_STATE"; - case OpType::KV_CACHE_APPEND: return "KV_CACHE_APPEND"; - case OpType::ATTENTION_CACHED: return "ATTENTION_CACHED"; - case OpType::CONV_CACHE_STATE: return "CONV_CACHE_STATE"; - case OpType::CONV_CACHE_APPEND: return "CONV_CACHE_APPEND"; - case OpType::RFFT: return "RFFT"; - case OpType::IRFFT: return "IRFFT"; - case OpType::MEL_FILTER_BANK: return "MEL_FILTER_BANK"; - case OpType::SPECTROGRAM: return "SPECTROGRAM"; - case OpType::IMAGE_PREPROCESS: return "IMAGE_PREPROCESS"; - case OpType::CLAMP: return "CLAMP"; - case OpType::DENSE_MLP_TQ_FUSED: return "DENSE_MLP_TQ_FUSED"; - case OpType::NOT_EQUAL: return "NOT_EQUAL"; - case OpType::SCALAR_NOT_EQUAL: return "SCALAR_NOT_EQUAL"; - } - return "UNKNOWN"; -} - -std::vector bytes_to_floats(const std::vector& bytes, Precision precision) { - std::vector values; - if (precision == Precision::FP32) { - const size_t count = bytes.size() / sizeof(float); - values.resize(count); - const auto* src = reinterpret_cast(bytes.data()); - for (size_t i = 0; i < count; ++i) values[i] = src[i]; - } else if (precision == Precision::FP16) { - const size_t count = bytes.size() / sizeof(__fp16); - values.resize(count); - const auto* src = reinterpret_cast(bytes.data()); - for (size_t i = 0; i < count; ++i) values[i] = static_cast(src[i]); - } else if (precision == Precision::INT8) { - const size_t count = bytes.size(); - values.resize(count); - const auto* src = reinterpret_cast(bytes.data()); - for (size_t i = 0; i < count; ++i) values[i] = static_cast(src[i]); - } - return values; -} - -std::vector buffer_to_floats(const BufferDesc& desc) { - std::vector values; - const void* data = desc.get_data(); - if (!data) return values; - if (desc.precision == Precision::FP32) { - values.resize(desc.byte_size / sizeof(float)); - const auto* src = static_cast(data); - for (size_t i = 0; i < values.size(); ++i) values[i] = src[i]; - } else if (desc.precision == Precision::FP16) { - values.resize(desc.byte_size / sizeof(__fp16)); - const auto* src = static_cast(data); - for (size_t i = 0; i < values.size(); ++i) values[i] = static_cast(src[i]); - } else if (desc.precision == Precision::INT8) { - values.resize(desc.byte_size); - const auto* src = static_cast(data); - for (size_t i = 0; i < values.size(); ++i) values[i] = static_cast(src[i]); - } - return values; -} - -void write_float_feature_json(std::ostream& out, - const std::string& name, - const std::vector& values, - const std::vector& shape, - Precision precision, - bool comma) { - double sum = 0.0; - float min_v = values.empty() ? 0.0f : values[0]; - float max_v = min_v; - double l2 = 0.0; - size_t finite_count = 0; - size_t nonfinite_count = 0; - for (float v : values) { - if (!std::isfinite(v)) { - ++nonfinite_count; - continue; - } - if (finite_count == 0) { - min_v = v; - max_v = v; - } - ++finite_count; - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - l2 += static_cast(v) * static_cast(v); - } - if (comma) out << ","; - out << "\"" << escape_json(name) << "\":{" - << "\"shape\":" << json_array(shape) << "," - << "\"precision\":\"" << precision_name(precision) << "\"," - << "\"count\":" << values.size() << "," - << "\"finite_count\":" << finite_count << "," - << "\"nonfinite_count\":" << nonfinite_count << "," - << "\"min\":" << min_v << "," - << "\"max\":" << max_v << "," - << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," - << "\"l2\":" << std::sqrt(l2) << "," - << "\"first_values\":" << json_float_array(values, 32) << "," - << "\"values\":" << json_float_array(values) - << "}"; -} - -int run_e003_preprocess_variants(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - cactus::engine::Config config; - if (!config.from_json(args.model + "/config.txt")) { - throw std::runtime_error("failed to read config.txt for E003 probe: " + args.model); - } - - const int patch_size = static_cast(config.vision_patch_size ? config.vision_patch_size : 16); - const int pooling_k = static_cast(config.vision_pooling_kernel_size ? config.vision_pooling_kernel_size : 3); - const size_t max_patches = static_cast(config.vision_default_output_length ? config.vision_default_output_length : 280) * pooling_k * pooling_k; - const int side_multiple = pooling_k * patch_size; - const size_t patch_dim = static_cast(3) * patch_size * patch_size; - const float rescale_factor = config.rescale_factor > 0.0f ? config.rescale_factor : (1.0f / 255.0f); - - int width = 0, height = 0, channels = 0; - unsigned char* raw = cactus_image_load(args.image.c_str(), &width, &height, &channels, 3); - if (!raw) throw std::runtime_error("failed to load image for E003 probe: " + args.image); - - const double target_pixels = static_cast(max_patches) * patch_size * patch_size; - const double pixel_count = std::max(1.0, static_cast(width) * static_cast(height)); - const double factor = std::sqrt(target_pixels / pixel_count); - int target_h = static_cast(std::floor(factor * height / side_multiple)) * side_multiple; - int target_w = static_cast(std::floor(factor * width / side_multiple)) * side_multiple; - if (target_h == 0) target_h = side_multiple; - if (target_w == 0) target_w = side_multiple; - - std::vector variants; - for (const std::string& name : {"stb_default_float", "stb_default_uint8", "stb_triangle_float", "stb_triangle_uint8"}) { - E003Variant variant; - variant.name = name; - variant.resized.assign(static_cast(target_w) * target_h * 3, 0.0f); - if (name == "stb_default_float") resize_float_default(raw, width, height, variant.resized, target_w, target_h); - else if (name == "stb_default_uint8") resize_uint8_default(raw, width, height, variant.resized, target_w, target_h); - else if (name == "stb_triangle_float") resize_float_triangle(raw, width, height, variant.resized, target_w, target_h); - else resize_uint8_triangle(raw, width, height, variant.resized, target_w, target_h); - variant.pixel_values = pack_gemma4_patches(variant.resized, target_w, target_h, patch_size, max_patches, patch_dim, rescale_factor); - variants.push_back(std::move(variant)); - } - cactus_image_free(raw); - - std::ostringstream out; - out << std::setprecision(9) - << "{" - << "\"source_width\":" << width << "," - << "\"source_height\":" << height << "," - << "\"target_width\":" << target_w << "," - << "\"target_height\":" << target_h << "," - << "\"patch_size\":" << patch_size << "," - << "\"pooling_kernel_size\":" << pooling_k << "," - << "\"max_patches\":" << max_patches << "," - << "\"patch_dim\":" << patch_dim << "," - << "\"rescale_factor\":" << rescale_factor << "," - << "\"variants\":{"; - for (size_t i = 0; i < variants.size(); ++i) { - write_variant_json(out, variants[i], i > 0); - } - out << "}}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_vision(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for vision probe: " + args.model); - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() - ? "What animal is in this image? Reply in one complete short sentence." - : args.prompt; - message.images.push_back(args.image); - std::vector messages{message}; - std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); - if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { - throw std::runtime_error("chunk prefill path failed for vision probe"); - } - - std::ostringstream out; - out << std::setprecision(9) << "{\"inputs\":{"; - if (model->vision_encoder_) { - for (size_t i = 0; i < model->vision_encoder_->logical_inputs.size(); ++i) { - if (i) out << ","; - const std::string& input_name = model->vision_encoder_->logical_inputs[i]; - const size_t node_id = static_cast(model->vision_encoder_->runtime_input_node_ids[i]); - const auto& desc = model->vision_encoder_->graph->get_output_buffer(node_id); - out << "\"" << escape_json(input_name) << "\":{" - << "\"node_id\":" << node_id << "," - << "\"shape\":" << json_array(desc.shape) << "," - << "\"precision\":\"" << precision_name(desc.precision) << "\"," - << "\"byte_size\":" << desc.byte_size - << "}"; - } - } - out << "},\"features\":{"; - bool first = true; - for (const auto& kv : model->media_features_) { - const std::string& name = kv.first; - Precision precision = model->media_feature_precisions_[name]; - std::vector values = bytes_to_floats(kv.second, precision); - write_float_feature_json(out, name, values, model->media_feature_shapes_[name], precision, !first); - first = false; - } - out << "}}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_vision_node_summary(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for vision node summary: " + args.model); - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() - ? "What animal is in this image? Reply in one complete short sentence." - : args.prompt; - message.images.push_back(args.image); - std::vector messages{message}; - std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); - if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { - throw std::runtime_error("chunk prefill path failed for vision node summary"); - } - if (!model->vision_encoder_ || !model->vision_encoder_->graph) { - throw std::runtime_error("model has no vision encoder graph"); - } - - const auto& graph = *model->vision_encoder_->graph; - std::ostringstream out; - out << std::setprecision(9) << "{\"nodes\":["; - bool first = true; - for (const auto& node_ptr : graph.nodes_) { - const GraphNode& node = *node_ptr; - const auto& desc = node.output_buffer; - const bool interesting = - node.id < 25 || - (node.id >= 660 && node.id <= 710) || - desc.shape == std::vector{1, 2520} || - desc.shape == std::vector{1, 2520, 1} || - desc.shape == std::vector{1, 2520, 2} || - desc.shape == std::vector{1, 2520, 768} || - desc.shape == std::vector{1, 256, 768} || - desc.shape == std::vector{256, 768} || - desc.shape == std::vector{256, 1536}; - if (!interesting) continue; - std::vector values = buffer_to_floats(desc); - double sum = 0.0; - float min_v = values.empty() ? 0.0f : values[0]; - float max_v = min_v; - size_t finite_count = 0; - size_t nonfinite_count = 0; - for (float v : values) { - if (!std::isfinite(v)) { - ++nonfinite_count; - continue; - } - if (finite_count == 0) { - min_v = v; - max_v = v; - } - ++finite_count; - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - if (!first) out << ","; - first = false; - out << "{" - << "\"node_id\":" << node.id << "," - << "\"op\":\"" << op_type_name(node.op_type) << "\"," - << "\"inputs\":" << json_array(node.input_ids) << "," - << "\"shape\":" << json_array(desc.shape) << "," - << "\"precision\":\"" << precision_name(desc.precision) << "\"," - << "\"byte_size\":" << desc.byte_size << "," - << "\"value_count\":" << values.size() << "," - << "\"finite_count\":" << finite_count << "," - << "\"nonfinite_count\":" << nonfinite_count << "," - << "\"min\":" << min_v << "," - << "\"max\":" << max_v << "," - << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," - << "\"first_values\":" << json_float_array(values, 32) << "," - << "\"values\":" << json_float_array(values) - << "}"; - } - out << "]}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_vision_selected_nodes(const Args& args) { - if (args.image.empty()) throw std::runtime_error("--image is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for selected vision nodes: " + args.model); - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() - ? "What animal is in this image? Reply in one complete short sentence." - : args.prompt; - message.images.push_back(args.image); - std::vector messages{message}; - std::vector tokens = tokenizer->encode(tokenizer->format_chat_prompt(messages, true)); - if (!model->run_chunk_prefill_path(tokens, {args.image}, {})) { - throw std::runtime_error("chunk prefill path failed for selected vision nodes"); - } - if (!model->vision_encoder_ || !model->vision_encoder_->graph) { - throw std::runtime_error("model has no vision encoder graph"); - } - - const std::vector selected = { - 672, 675, 699, 700, - 752, 753, 756, 757, 789, 792, 793, 825, 828, 829, 846, 847, 874, 875, - 2757, 2758, 2804, 2805, 2806, 2807, 2812, 2813, 2814, 2815, 2816, - }; - const auto& graph = *model->vision_encoder_->graph; - std::ostringstream out; - out << std::setprecision(9) << "{\"nodes\":{"; - bool first = true; - for (size_t node_id : selected) { - auto it = graph.node_index_map_.find(node_id); - if (it == graph.node_index_map_.end()) continue; - const GraphNode& node = *graph.nodes_[it->second]; - const auto& desc = node.output_buffer; - std::vector values = buffer_to_floats(desc); - if (!first) out << ","; - first = false; - out << "\"" << node_id << "\":{" - << "\"node_id\":" << node.id << "," - << "\"op\":\"" << op_type_name(node.op_type) << "\"," - << "\"inputs\":" << json_array(node.input_ids) << "," - << "\"shape\":" << json_array(desc.shape) << "," - << "\"precision\":\"" << precision_name(desc.precision) << "\"," - << "\"values\":" << json_float_array(values) - << "}"; - } - out << "}}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_audio_summary(const Args& args) { - if (args.audio.empty()) throw std::runtime_error("--audio is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for audio summary: " + args.model); - } - AudioFP32 wav = load_wav(args.audio); - std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); - auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); - model->run_audio_encoder(prep.features); - - std::ostringstream out; - out << std::setprecision(9) << "{" - << "\"num_frames\":" << prep.num_frames << "," - << "\"num_soft_tokens\":" << prep.num_soft_tokens << "," - << "\"feature_count\":" << prep.features.size() << "," - << "\"feature_first_values\":" << json_float_array(prep.features, 32) << "," - << "\"media_features\":{"; - bool first_feature = true; - for (const auto& kv : model->media_features_) { - const std::string& name = kv.first; - Precision precision = model->media_feature_precisions_[name]; - std::vector values = bytes_to_floats(kv.second, precision); - double sum = 0.0; - float min_v = values.empty() ? 0.0f : values[0]; - float max_v = min_v; - size_t finite_count = 0; - size_t nonfinite_count = 0; - for (float v : values) { - if (!std::isfinite(v)) { - ++nonfinite_count; - continue; - } - if (finite_count == 0) { - min_v = v; - max_v = v; - } - ++finite_count; - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - if (!first_feature) out << ","; - first_feature = false; - out << "\"" << name << "\":{" - << "\"shape\":" << json_array(model->media_feature_shapes_[name]) << "," - << "\"precision\":\"" << precision_name(precision) << "\"," - << "\"value_count\":" << values.size() << "," - << "\"finite_count\":" << finite_count << "," - << "\"nonfinite_count\":" << nonfinite_count << "," - << "\"min\":" << min_v << "," - << "\"max\":" << max_v << "," - << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," - << "\"first_values\":" << json_float_array(values, 32) << "," - << "\"values\":" << json_float_array(values) - << "}"; - } - out << "}}\n"; - std::cout << out.str(); - return 0; -} - -int run_e002_audio_node_summary(const Args& args) { - if (args.audio.empty()) throw std::runtime_error("--audio is required"); - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for audio node summary: " + args.model); - } - AudioFP32 wav = load_wav(args.audio); - std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); - auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); - model->run_audio_encoder(prep.features); - if (!model->audio_encoder_ || !model->audio_encoder_->graph) { - throw std::runtime_error("model has no audio encoder graph"); - } - - const auto& graph = *model->audio_encoder_->graph; - std::ostringstream out; - out << std::setprecision(9) << "{\"nodes\":["; - bool first = true; - for (size_t order = 0; order < graph.nodes_.size(); ++order) { - const GraphNode& node = *graph.nodes_[order]; - const auto& desc = node.output_buffer; - const auto& shape = desc.shape; - bool interesting = node.op_type != OpType::INPUT; - if (shape.empty()) interesting = false; - if (!interesting) continue; - - std::vector values = buffer_to_floats(desc); - double sum = 0.0; - float min_v = values.empty() ? 0.0f : values[0]; - float max_v = min_v; - size_t finite_count = 0; - size_t nonfinite_count = 0; - for (float v : values) { - if (!std::isfinite(v)) { - ++nonfinite_count; - continue; - } - if (finite_count == 0) { - min_v = v; - max_v = v; - } - ++finite_count; - min_v = std::min(min_v, v); - max_v = std::max(max_v, v); - sum += static_cast(v); - } - if (!first) out << ","; - first = false; - out << "{" - << "\"order\":" << order << "," - << "\"node_id\":" << node.id << "," - << "\"op\":\"" << op_type_name(node.op_type) << "\"," - << "\"inputs\":" << json_array(node.input_ids) << "," - << "\"shape\":" << json_array(shape) << "," - << "\"precision\":\"" << precision_name(desc.precision) << "\"," - << "\"value_count\":" << values.size() << "," - << "\"finite_count\":" << finite_count << "," - << "\"nonfinite_count\":" << nonfinite_count << "," - << "\"min\":" << min_v << "," - << "\"max\":" << max_v << "," - << "\"mean\":" << (finite_count == 0 ? 0.0 : sum / static_cast(finite_count)) << "," - << "\"first_values\":" << json_float_array(values, 16) - << "}"; - } - out << "]}\n"; - std::cout << out.str(); - return 0; -} - -int run_gemma_first_token(const Args& args) { - auto model = cactus::engine::create_model(args.model); - if (!model || !model->init(args.model, 4096, "", false)) { - throw std::runtime_error("failed to initialize model for first token probe: " + args.model); - } - if (args.command == "gemma-first-token-media-step") { - model->lm_encoder_ = nullptr; - } - auto* tokenizer = model->get_tokenizer(); - if (!tokenizer) throw std::runtime_error("model has no tokenizer"); - - cactus::engine::ChatMessage message; - message.role = "user"; - message.content = args.prompt.empty() ? "Respond briefly." : args.prompt; - if (!args.image.empty()) message.images.push_back(args.image); - - std::vector audio_features; - size_t audio_num_frames = 0; - if (!args.audio.empty()) { - message.audio.push_back(args.audio); - AudioFP32 wav = load_wav(args.audio); - std::vector waveform_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); - auto prep = cactus::audio::preprocess_audio_for_gemma4(std::move(waveform_16k), model->get_config()); - audio_features = std::move(prep.features); - audio_num_frames = prep.num_frames; - message.audio_soft_token_count = prep.num_soft_tokens; - } - - std::vector messages{message}; - std::string prompt = tokenizer->format_chat_prompt(messages, true); - std::vector tokens = tokenizer->encode(prompt); - if (tokens.empty()) throw std::runtime_error("first token probe produced empty prompt"); - std::vector prefill_tokens(tokens.begin(), tokens.end() - 1); - if (!prefill_tokens.empty()) { - std::vector> audio_messages; - if (!audio_features.empty()) audio_messages.push_back(std::move(audio_features)); - model->prefill_with_media(prefill_tokens, args.image.empty() ? std::vector{} : std::vector{args.image}, audio_messages); - } - uint32_t first = model->decode({tokens.back()}, 0.0f, 1.0f, 1); - std::string text = tokenizer->decode({first}); - std::vector> top_logits; - if (model->decoder_ && !model->decoder_->output_node_ids.empty()) { - size_t out_node = static_cast(model->decoder_->output_node_ids[0]); - const auto& desc = model->decoder_->graph->get_output_buffer(out_node); - std::vector logits = buffer_to_floats(desc); - if (!desc.shape.empty() && !logits.empty()) { - size_t vocab = desc.shape.back(); - size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; - size_t row = seq > 0 ? seq - 1 : 0; - size_t start = row * vocab; - size_t end = std::min(start + vocab, logits.size()); - for (size_t i = start; i < end; ++i) top_logits.emplace_back(i - start, logits[i]); - size_t keep = std::min(10, top_logits.size()); - std::partial_sort( - top_logits.begin(), - top_logits.begin() + keep, - top_logits.end(), - [](const auto& a, const auto& b) { return a.second > b.second; }); - top_logits.resize(keep); - } - } - - const uint32_t image_token_id = model->get_config().image_token_id ? model->get_config().image_token_id : tokenizer->get_image_token_id(); - const uint32_t audio_token_id = model->get_config().audio_token_id; - size_t image_token_count = 0; - size_t audio_token_count = 0; - for (uint32_t token : tokens) { - if (image_token_id != 0 && token == image_token_id) ++image_token_count; - if (audio_token_id != 0 && token == audio_token_id) ++audio_token_count; - } - - std::ostringstream out; - out << "{" - << "\"prompt_token_count\":" << tokens.size() << "," - << "\"prefill_token_count\":" << prefill_tokens.size() << "," - << "\"last_prompt_token\":" << tokens.back() << "," - << "\"first_token\":" << first << "," - << "\"first_text\":\"" << escape_json(text) << "\"," - << "\"eos_token\":" << tokenizer->get_eos_token() << "," - << "\"is_eos\":" << (first == tokenizer->get_eos_token() ? "true" : "false") << "," - << "\"image_token_count\":" << image_token_count << "," - << "\"audio_token_id\":" << audio_token_id << "," - << "\"audio_token_count\":" << audio_token_count << "," - << "\"audio_num_frames\":" << audio_num_frames << "," - << "\"top_logits\":["; - for (size_t i = 0; i < top_logits.size(); ++i) { - if (i) out << ","; - out << "{\"token\":" << top_logits[i].first - << ",\"score\":" << top_logits[i].second - << ",\"text\":\"" << escape_json(tokenizer->decode({static_cast(top_logits[i].first)})) << "\"}"; - } - out << "]," - << "\"prompt_prefix\":\"" << escape_json(prompt.substr(0, 320)) << "\"" - << "}\n"; - std::cout << out.str(); - return 0; -} - -std::string build_messages(const Args& args) { - if (!args.messages_json.empty()) return args.messages_json; - std::ostringstream out; - out << "[{\"role\":\"user\",\"content\":\"" << escape_json(args.prompt.empty() ? "Respond briefly." : args.prompt) << "\""; - if (!args.image.empty()) out << ",\"images\":[\"" << escape_json(args.image) << "\"]"; - if (!args.audio.empty()) out << ",\"audio\":[\"" << escape_json(args.audio) << "\"]"; - out << "}]"; - return out.str(); -} - -std::string complete_options(int max_tokens) { - std::ostringstream out; - out << "{" - << "\"max_tokens\":" << max_tokens << "," - << "\"temperature\":0.0," - << "\"top_p\":1.0," - << "\"top_k\":1," - << "\"telemetry_enabled\":false," - << "\"auto_handoff\":false," - << "\"confidence_threshold\":0.0," - << "\"stop_sequences\":[\"<|im_end|>\",\"\"]" - << "}"; - return out.str(); -} - -std::vector read_wav_pcm16(const std::string& path) { - std::ifstream in(path, std::ios::binary); - if (!in) throw std::runtime_error("failed to open wav: " + path); - in.seekg(0, std::ios::end); - std::streamoff size = in.tellg(); - if (size <= 44) throw std::runtime_error("wav file too small: " + path); - in.seekg(44, std::ios::beg); - std::vector pcm(static_cast(size - 44)); - in.read(reinterpret_cast(pcm.data()), static_cast(pcm.size())); - if (!in) throw std::runtime_error("failed to read wav pcm: " + path); - return pcm; -} - -int run_complete(const Args& args) { - cactus_model_t model = cactus_init(args.model.c_str(), nullptr, false); - if (!model) throw std::runtime_error("cactus_init failed for " + args.model); - std::vector response(kResponseBufferSize, 0); - std::string messages = build_messages(args); - std::string options = complete_options(args.max_tokens); - int rc = cactus_complete( - model, - messages.c_str(), - response.data(), - response.size(), - options.c_str(), - nullptr, - nullptr, - nullptr, - nullptr, - 0); - cactus_destroy(model); - std::cout << response.data() << "\n"; - return rc > 0 ? 0 : 2; -} - -int run_transcribe(const Args& args, bool pcm) { - if (args.audio.empty()) throw std::runtime_error("--audio is required"); - cactus_model_t model = cactus_init(args.model.c_str(), nullptr, false); - if (!model) throw std::runtime_error("cactus_init failed for " + args.model); - std::vector response(kResponseBufferSize, 0); - std::vector pcm_data; - const char* audio_path = args.audio.c_str(); - const uint8_t* pcm_buffer = nullptr; - size_t pcm_size = 0; - if (pcm) { - pcm_data = read_wav_pcm16(args.audio); - audio_path = nullptr; - pcm_buffer = pcm_data.data(); - pcm_size = pcm_data.size(); - } - int rc = cactus_transcribe( - model, - audio_path, - nullptr, - response.data(), - response.size(), - "{\"max_tokens\":200,\"telemetry_enabled\":false,\"auto_handoff\":false}", - nullptr, - nullptr, - pcm_buffer, - pcm_size); - cactus_destroy(model); - std::cout << response.data() << "\n"; - return rc > 0 ? 0 : 2; -} - -} // namespace - -int main(int argc, char** argv) { - try { - Args args = parse_args(argc, argv); - if (args.command == "e002-tokens") return run_e002_tokens(args); - if (args.command == "e002-preprocess") return run_e002_preprocess(args); - if (args.command == "e002-vision") return run_e002_vision(args); - if (args.command == "e002-vision-node-summary") return run_e002_vision_node_summary(args); - if (args.command == "e002-vision-selected-nodes") return run_e002_vision_selected_nodes(args); - if (args.command == "e002-audio-summary") return run_e002_audio_summary(args); - if (args.command == "e002-audio-node-summary") return run_e002_audio_node_summary(args); - if (args.command == "gemma-first-token") return run_gemma_first_token(args); - if (args.command == "gemma-first-token-media-step") return run_gemma_first_token(args); - if (args.command == "e003-preprocess-variants") return run_e003_preprocess_variants(args); - if (args.command == "complete") return run_complete(args); - if (args.command == "transcribe") return run_transcribe(args, false); - if (args.command == "transcribe-pcm") return run_transcribe(args, true); - throw std::runtime_error("unknown command: " + args.command); - } catch (const std::exception& exc) { - std::cerr << exc.what() << "\n"; - return 1; - } -} diff --git a/local_integration_tests/test_modalities.py b/local_integration_tests/test_modalities.py deleted file mode 100644 index 62251f3e2..000000000 --- a/local_integration_tests/test_modalities.py +++ /dev/null @@ -1,210 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from conftest import assert_completion, assert_transcript, model_path, run_runner - - -def long_prompt() -> str: - sentence = "Summarize this integration test context in one concise sentence. " - return sentence * 40 - - -@pytest.mark.smoke -def test_gemma4_e2b_text_image_and_audio(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - text_model = model_path( - weights_root, - "gemma-4-e2b-it-sharedkv", - "gemma-4-e2b-it-sharedkv-c128", - "gemma-4-e2b-it", - native_config=True, - ) - media_model = model_path( - weights_root, - "gemma-4-e2b-it-alt-fixed-rowtuned-mm", - "gemma-4-e2b-it", - native_config=True, - ) - cases = [ - [text_model, "--prompt", "Reply with one short sentence about local inference."], - [media_model, "--prompt", "Describe this image in one short sentence.", "--image", assets["image_monkey"]], - [media_model, "--prompt", "Transcribe or summarize this audio in one short sentence.", "--audio", assets["audio"]], - ] - for case in cases: - model, *args = case - result = run_runner(runner, "complete", "--model", model, "--max-tokens", 24, *args, timeout=240) - assert_completion(result) - - -@pytest.mark.smoke -def test_gemma4_e2b_combined_image_audio( - runner: Path, - weights_root: Path, - assets: dict[str, Path], -) -> None: - model = model_path( - weights_root, - "gemma-4-e2b-it-alt-fixed-rowtuned-mm", - "gemma-4-e2b-it", - native_config=True, - ) - result = run_runner( - runner, - "complete", - "--model", - model, - "--prompt", - "Describe the image and summarize the audio in one short sentence.", - "--image", - assets["image_thing"], - "--audio", - assets["audio"], - "--max-tokens", - 24, - timeout=240, - ) - assert_completion(result) - - -@pytest.mark.smoke -def test_gemma4_e2b_multiturn_text_reuse(runner: Path, weights_root: Path) -> None: - model = model_path( - weights_root, - "gemma-4-e2b-it-sharedkv", - "gemma-4-e2b-it-sharedkv-c128", - "gemma-4-e2b-it", - native_config=True, - ) - messages = [ - {"role": "system", "content": "Be brief."}, - {"role": "user", "content": "Remember the name Henry."}, - {"role": "assistant", "content": "I will remember the name Henry."}, - {"role": "user", "content": "What name should you remember?"}, - ] - result = run_runner( - runner, - "complete", - "--model", - model, - "--messages-json", - json.dumps(messages), - "--max-tokens", - 24, - timeout=240, - ) - assert_completion(result) - - -@pytest.mark.full -def test_gemma4_e4b_all_modalities_when_available(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - model = model_path(weights_root, "gemma-4-e4b-it", required=False, native_config=True) - result = run_runner( - runner, - "complete", - "--model", - model, - "--prompt", - "Describe the image and audio briefly.", - "--image", - assets["image_monkey"], - "--audio", - assets["audio"], - "--max-tokens", - 24, - timeout=360, - ) - assert_completion(result) - - -@pytest.mark.smoke -def test_qwen3_vl_text_and_chunked_prefill(runner: Path, weights_root: Path) -> None: - model = model_path( - weights_root, - "qwen3-vl-2b-instruct-reconvert", - "qwen3-vl-2b-instruct-reconvert-c256", - "qwen3-vl-2b-instruct-reconvert-chunks", - "qwen3-vl-2b-instruct", - "qwen3-vl-2b", - native_config=True, - ) - for prompt in ("Reply with one short sentence about cactus.", long_prompt()): - result = run_runner(runner, "complete", "--model", model, "--prompt", prompt, "--max-tokens", 16, timeout=300) - assert_completion(result) - - -@pytest.mark.smoke -def test_qwen3_vl_image(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - model = model_path( - weights_root, - "qwen3-vl-2b-instruct-reconvert", - "qwen3-vl-2b-instruct-reconvert-c256", - "qwen3-vl-2b-instruct-reconvert-c512", - native_config=True, - ) - result = run_runner( - runner, - "complete", - "--model", - model, - "--prompt", - "Describe this image in one short sentence.", - "--image", - assets["image_monkey"], - "--max-tokens", - 16, - timeout=300, - ) - assert_completion(result) - - -@pytest.mark.smoke -def test_lfm2_vl_text_image_and_chunked_prefill(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - model = model_path( - weights_root, - "lfm2.5-vl-1.6b-retranspile-4096", - "lfm2.5-vl-1.6b-rechunk-c256", - "lfm2.5-vl-1.6b", - "lfm2.5-vl-1.6b-rechunk-c128", - "lfm2-vl-450m", - native_config=True, - ) - for prompt, image in ( - ("Reply with one short sentence about local model testing.", None), - ("Describe this image in one short sentence.", assets["image_thing"]), - (long_prompt(), None), - ): - args: list[object] = ["complete", "--model", model, "--prompt", prompt, "--max-tokens", 16] - if image is not None: - args.extend(["--image", image]) - result = run_runner(runner, *args, timeout=300) - assert_completion(result) - - -@pytest.mark.smoke -def test_parakeet_tdt_file_and_pcm_transcription(runner: Path, weights_root: Path, assets: dict[str, Path]) -> None: - model = model_path( - weights_root, - "parakeet-tdt-0.6b-v3-transpiled", - "parakeet-tdt-0.6b-v3", - native_config=True, - ) - file_result = run_runner(runner, "transcribe", "--model", model, "--audio", assets["audio"], timeout=240) - pcm_result = run_runner(runner, "transcribe-pcm", "--model", model, "--audio", assets["audio"], timeout=240) - assert_transcript(file_result) - assert_transcript(pcm_result) - - -@pytest.mark.full -def test_whisper_small_file_and_pcm_transcription_when_available( - runner: Path, - weights_root: Path, - assets: dict[str, Path], -) -> None: - model = model_path(weights_root, "whisper-small", required=False, native_config=True) - file_result = run_runner(runner, "transcribe", "--model", model, "--audio", assets["audio"], timeout=300) - pcm_result = run_runner(runner, "transcribe-pcm", "--model", model, "--audio", assets["audio"], timeout=300) - assert_transcript(file_result) - assert_transcript(pcm_result) diff --git a/local_investigations/gemma4_multimodal/BACKGROUND.md b/local_investigations/gemma4_multimodal/BACKGROUND.md deleted file mode 100644 index 8a7cf2031..000000000 --- a/local_investigations/gemma4_multimodal/BACKGROUND.md +++ /dev/null @@ -1,53 +0,0 @@ -# Background - -## Local artifact inventory - -Multimodal Gemma candidates: -- `gemma-4-e2b-it` -- `gemma-4-e2b-it-alt-fixed-rowtuned-mm` -- `gemma-4-e2b-it-fresh-mm` - -Text-only/shared-KV Gemma candidates: -- `gemma-4-e2b-it-sharedkv` -- `gemma-4-e2b-it-sharedkv-c128` -- `gemma-4-e2b-it-sharedkv-c256` -- `gemma-4-e2b-it-sharedkv-c512` - -The shared-KV bundles do not include `vision_encoder` or `audio_encoder`, so they are not valid vision/audio quality candidates. - -## Multimodal component shape facts - -Both local multimodal Gemma E2B bundles point at `google/gemma-4-E2B-it` and use the same local HF snapshot. - -The two older multimodal bundles point at missing HF snapshot `b324173c7d5721c2baba7f3b17b3b9b3d34ab1e9`. - -The fresh bundle points at current cached HF snapshot `905e84b50c4d2a365ebde34e685027578e6728db`. - -Component order: -- `audio_encoder` -- `vision_encoder` -- `lm_encoder` -- `decoder_prefill_chunk` -- `lm_encoder_step` -- `lm_encoder_media_step` -- `decoder_step` - -Important logical inputs: -- `vision_encoder`: `pixel_values`, `pixel_position_ids` -- `audio_encoder`: `input_features`, `input_features_mask` -- `lm_encoder`: `input_ids`, `image_features`, `audio_features` -- `lm_encoder_media_step`: `inputs_embeds`, `input_ids`, `position_ids` - -## Controls - -The paired audio created from `../cactus/what_is_in_this_image.mp3` was converted to RIFF/WAV because `cactus_complete` currently rejects MP3 with `Not RIFF`. - -Parakeet transcribes the converted WAV as: - -```text -What is in this image? -``` - -Qwen3-VL identifies `test_monkey.png` as a proboscis monkey, so the asset itself is a good vision control. - -HF reference on the current cached snapshot identifies the same image with the same general prompt as an orangutan. HF also works when using the Cactus-native expanded image prompt/tensors after renaming `pixel_position_ids` to HF's `image_position_ids`. diff --git a/local_investigations/gemma4_multimodal/EXPERIMENTS.md b/local_investigations/gemma4_multimodal/EXPERIMENTS.md deleted file mode 100644 index 0e5f36f6d..000000000 --- a/local_investigations/gemma4_multimodal/EXPERIMENTS.md +++ /dev/null @@ -1,80 +0,0 @@ -# Experiments - -## E1: Paired audio validation - -Purpose: -- Verify that the new audio fixture says what the user intended before using it for Gemma mixed testing. - -Inputs: -- `../cactus/what_is_in_this_image.mp3` -- Converted WAV: `local_integration_tests/.tmp/what_is_in_this_image.wav` -- Parakeet model: `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` - -Observed: - -```text -What is in this image? -``` - -Interpretation: -- The audio fixture is good. -- The native completion path currently requires WAV input, not MP3. - -## E2: Gemma and Qwen image controls - -Purpose: -- Confirm whether the monkey image is a valid vision test and whether the Gemma failure is model/path-specific. - -Observed: -- Qwen3-VL identifies the image as a proboscis monkey. -- Gemma multimodal artifacts fail to identify the monkey in image-only prompts. - -Interpretation: -- The asset is a valid vision control. -- The failure is specific to Gemma artifact/code path behavior, not the image itself. - -## E2b: HF reference with Cactus-native prompt style - -Purpose: -- Check whether Cactus-native expanded media prompts and native image tensors are themselves valid. - -Observed: -- HF current snapshot with normal processor chat template answered: `The animal in this image is an orangutan.` -- HF current snapshot with Cactus-native expanded prompt/tensors also answered: `The animal in this image is an orangutan.` - -Interpretation: -- Prompt style and native image tensor preparation are not the high-level cause. -- The remaining fault is in the transpiled/C++ component execution path or artifact lowering/runtime behavior. - -## E3: Force Gemma media through media-step path - -Purpose: -- Test whether Gemma image-only quality fails because arbitrary prompts are incorrectly routed through the static `lm_encoder` chunk path. - -Change tested: -- Temporarily disabled chunk media prefill for Gemma so `prefill_with_media` used the `lm_encoder_media_step` loop. - -Observed: -- Gemma image-only monkey prompt produced malformed/gibberish output instead of identifying the image. -- Repeating the same temporary routing change with `gemma-4-e2b-it-fresh-mm` also produced malformed/gibberish output. - -Interpretation: -- The simple routing fallback is not valid. -- The investigation should compare actual component contracts and traced prompt assumptions before changing routing further. - -## E4: Fresh reconvert artifact - -Purpose: -- Determine whether stale artifacts caused the Gemma vision failure. - -Artifact: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -Observed: -- Fresh artifact points at current cached snapshot `905e84b50c4d2a365ebde34e685027578e6728db`. -- Image-only monkey prompt produced: `The image shows a close-up of a human nose.` -- Similar prompt produced: `The image shows a close-up of a fawn's nose.` - -Interpretation: -- Reconvert was necessary to remove stale-artifact uncertainty. -- Reconvert alone does not solve Gemma native vision quality. diff --git a/local_investigations/gemma4_multimodal/FIXES.md b/local_investigations/gemma4_multimodal/FIXES.md deleted file mode 100644 index 854813740..000000000 --- a/local_investigations/gemma4_multimodal/FIXES.md +++ /dev/null @@ -1,52 +0,0 @@ -# Fixes - -## F1: Audio incremental decode prompt replay - -Status: applied before this investigation workspace was created - -Rationale: -- `cactus_complete` passed the entire processed prompt into `decode_with_audio` on each generated token. -- `decode_with_audio` currently delegates to incremental `decode`, so this replayed the prompt into KV cache repeatedly. - -Observed effect: -- Mixed-media generation speed improved. -- Gemma audio semantics remained poor, so this was not the full root cause. - -## F2: Audio encoder mask valid-frame handling - -Status: applied before this investigation workspace was created - -Rationale: -- The audio frontend pads feature buffers to graph capacity. -- The mask should mark only valid frames, not all padded frames. - -Observed effect: -- Build and smoke tests still pass. -- Gemma audio semantics remained poor, so this was not the full root cause. - -## F3: Force Gemma media-step fallback - -Status: reverted - -Rationale: -- Tested whether the static chunk `lm_encoder` path was the immediate cause of Gemma image-only failure. - -Observed effect: -- Output became gibberish. -- The change was reverted. -- The same experiment was repeated with the fresh artifact and was still gibberish. - -## F4: Fresh Gemma4 multimodal reconvert - -Status: created for investigation - -Artifact: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -Rationale: -- Existing multimodal manifests referenced a missing older HF snapshot. -- The current cached HF snapshot produces a valid HF reference answer. - -Observed effect: -- Fresh artifact no longer depends on the missing snapshot. -- Native Cactus vision quality is still wrong, so further runtime/component investigation is required. diff --git a/local_investigations/gemma4_multimodal/HYPOTHESES.md b/local_investigations/gemma4_multimodal/HYPOTHESES.md deleted file mode 100644 index c90541186..000000000 --- a/local_investigations/gemma4_multimodal/HYPOTHESES.md +++ /dev/null @@ -1,71 +0,0 @@ -# Hypotheses - -## H1: Current multimodal artifact is stale or malformed - -Status: supported but not sufficient - -Evidence so far: -- Both local multimodal Gemma E2B bundles have full multimodal components. -- Both reference the same HF snapshot. -- Shared-KV artifacts are text-only and should not be used for vision/audio quality. -- The older multimodal bundles reference a missing local HF snapshot. -- A fresh bundle was reconverted from the current cached HF snapshot. -- The fresh bundle improves the failure mode but still does not match HF vision quality. - -Next evidence needed: -- Keep using `gemma-4-e2b-it-fresh-mm` as the best Cactus artifact for further debugging. -- Do not assume reconversion alone fixes the native runtime path. - -## H2: Gemma native image preprocessing diverges from Python/HF preprocessing - -Status: open - -Evidence so far: -- C++ and Python both appear to use Gemma4-style resizing, patching, and position IDs. -- No tensor-level diff has been run yet. - -Next evidence needed: -- Compare `pixel_values` and `pixel_position_ids` between C++ and Python for the same image. - -## H3: Gemma multimodal chunk prefill uses a static traced merge plan that is invalid for arbitrary prompts - -Status: partially supported, simple fallback refuted - -Evidence so far: -- The `lm_encoder` component was traced with a fixed processor prompt embedded in the graph metadata. -- The Python adapter builds a native merge plan during transpile from the example `input_ids`. -- C++ currently sends arbitrary prompt tokens through this `lm_encoder` for media prefill. -- Gemma image-only fails even though Qwen identifies the same image. - -Experiment: -- Route Gemma media prefill through `lm_encoder_media_step` instead of the chunk `lm_encoder` path, then re-run image-only quality probes. - -Result: -- The simple dynamic fallback produced obvious gibberish for image-only monkey prompts. -- The same result reproduced with the fresh artifact. -- This suggests `lm_encoder_media_step` is not a drop-in prefill replacement as currently wired, or it needs additional exact contract handling. -- The static-merge concern remains plausible, but the fix is not simply disabling chunk prefill for all Gemma media. - -## H5: HF/native prompt and preprocessing are sound; the divergence is in transpiled/C++ execution - -Status: supported - -Evidence so far: -- HF with the normal processor chat template identifies the monkey image as an orangutan. -- HF with Cactus-native expanded prompt/tensors also identifies the image as an orangutan when `pixel_position_ids` are passed as HF `image_position_ids`. -- Fresh Cactus native with the same image says the image is a human/fawn nose. - -Next evidence needed: -- Compare component-level outputs or implement/test the intended dynamic Gemma media chunk path using `lm_encoder_text_chunk` and `lm_encoder_media_chunk`. - -## H4: Audio frontend or audio mask is wrong - -Status: open - -Evidence so far: -- The completion decode loop previously replayed the whole prompt for audio generation; this was fixed. -- The audio encoder mask was changed to mark only valid frames. -- Audio-only still does not show evidence of understanding the spoken content. - -Next evidence needed: -- Work on audio only after vision-only is explained or fixed. diff --git a/local_investigations/gemma4_multimodal/PLAN.md b/local_investigations/gemma4_multimodal/PLAN.md deleted file mode 100644 index 5fc3bdad7..000000000 --- a/local_investigations/gemma4_multimodal/PLAN.md +++ /dev/null @@ -1,29 +0,0 @@ -# Plan - -## Current phase - -Isolate Gemma vision-only correctness before treating mixed image+audio as a product behavior. - -## Method - -Use a hypothesis-driven loop: -1. Record a hypothesis in `HYPOTHESES.md`. -2. Run the smallest experiment that can support or refute it. -3. Record exact output and interpretation in `EXPERIMENTS.md`. -4. If supported, make the smallest fix and document it in `FIXES.md`. -5. Re-run the focused quality probe and the nearest regression suite. - -## Layer order - -1. Artifact validity and freshness. -2. Prompt/token construction. -3. Image preprocessing. -4. Vision encoder output. -5. LM media merge behavior. -6. Decoder/cache behavior. -7. Audio preprocessing and encoder behavior. -8. Mixed image+audio behavior. - -## Current decision - -Investigate whether Gemma multimodal prefill is incorrectly using the static chunk `lm_encoder` path for arbitrary prompts instead of the dynamic `lm_encoder_media_step` path. diff --git a/local_investigations/gemma4_multimodal/README.md b/local_investigations/gemma4_multimodal/README.md deleted file mode 100644 index 0cde142cb..000000000 --- a/local_investigations/gemma4_multimodal/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Gemma4 multimodal investigation - -Local, uncommitted workspace for investigating Gemma4 native vision/audio quality. - -Current status: -- Vision-only Gemma native does not reliably identify `cactus-engine/tests/assets/test_monkey.png`. -- Audio-only Gemma native does not yet show evidence that it is using the audio content. -- Mixed image+audio with `what_is_in_this_image.wav` currently fails because the underlying unimodal paths are not known-good. -- Qwen3-VL and Parakeet are useful controls: Qwen identifies the monkey, and Parakeet transcribes the paired audio. - -Primary artifacts: -- `../cactus/weights/gemma-4-e2b-it` -- `../cactus/weights/gemma-4-e2b-it-alt-fixed-rowtuned-mm` -- `../cactus/weights/qwen3-vl-2b-instruct-reconvert` -- `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` - -Read in order: -1. `BACKGROUND.md` -2. `PLAN.md` -3. `HYPOTHESES.md` -4. `EXPERIMENTS.md` -5. `FIXES.md` -6. `SUMMARY.md` diff --git a/local_investigations/gemma4_multimodal/SUMMARY.md b/local_investigations/gemma4_multimodal/SUMMARY.md deleted file mode 100644 index e39aa24e0..000000000 --- a/local_investigations/gemma4_multimodal/SUMMARY.md +++ /dev/null @@ -1,9 +0,0 @@ -# Summary - -Investigation is in progress. - -Current leading hypothesis: -- Gemma media prefill has a component-contract mismatch or lowering/runtime mismatch after HF-native prompt/preprocessing, likely around transpiled media encoder/LM merge execution. - -Current next step: -- Compare component-level outputs or implement/test the intended dynamic Gemma media chunk path using `lm_encoder_text_chunk` and `lm_encoder_media_chunk`. diff --git a/local_investigations/gemma4_multimodal_v2/00_INDEX.md b/local_investigations/gemma4_multimodal_v2/00_INDEX.md deleted file mode 100644 index 71ed21eb7..000000000 --- a/local_investigations/gemma4_multimodal_v2/00_INDEX.md +++ /dev/null @@ -1,48 +0,0 @@ -# Gemma4 Multimodal V2 Index - -Status: Gemma4 image-only, audio-only, and mixed image+audio now pass local end-to-end probes after the dynamic audio merge fix; Qwen, Parakeet, policy, and smoke controls pass - -Active phase: post-fix review / audit. Do not commit or push until the user reviews. - -Default artifact under investigation: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -Key controls: -- HF reference snapshot: `905e84b50c4d2a365ebde34e685027578e6728db` -- Qwen image control: `../cactus/weights/qwen3-vl-2b-instruct-reconvert` -- Parakeet audio control: `../cactus/weights/parakeet-tdt-0.6b-v3-transpiled` - -Read order: -1. `09_STATUS_FOR_NEXT_AGENT.md` -2. `10_CONTINUATION_PLAN.md` -3. `01_BACKGROUND.md` -4. `02_QUESTIONS.md` -5. `03_HYPOTHESES.md` -6. `04_EXPERIMENTS.md` -7. `05_DECISIONS.md` -8. `06_FIX_LOG.md` -9. `07_ACCEPTANCE.md` -10. `08_HANDOFF.md` - -Latest proven artifacts: - -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json`: image preprocessing parity recorded as `status: pass`, max abs diff `4.987010955859184e-07`. -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json`: latest recorded vision encoder output is FP16-close to the Python/HF reference for `pooler_output`, cosine `0.9999955754245847`, mean abs diff `0.0015996834738693023`. -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json`: image-only Cactus Gemma answered `The animal in the image is a golden monkey.` -- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json`: after the Gemma4 audio bias FP16 fix, first 64 audio rows are FP16-close to Python, cosine `0.9978210926`, mean abs diff `0.00645508`. -- `artifacts/E029_first_token_after_dynamic_merge/`: dynamic merge first-token probes now produce normal text starts, audio `Please`, mixed `This`. -- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json`: audio-only Cactus Gemma responds coherently: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json`: mixed Cactus Gemma answers: `This image is of a golden monkey.` -- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json`: image-only still answers: `The animal in the image is a golden monkey.` -- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json`: Qwen image control identifies a proboscis monkey. -- `artifacts/E034_parakeet_regression_after_dynamic_merge/`: Parakeet file and PCM controls both transcribe `What is in this image?` -- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json`: full emitted audio tensor matches the E023 post-bias tensor exactly, max diff `0`. - -Latest test commands: -- `source ../cactus/venv/bin/activate && ./cactus/build.sh` -- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q` -> `16 passed` -- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s` -> `7 passed, 2 deselected` - -Continuation contract: -- Follow `10_CONTINUATION_PLAN.md`. -- The acceptance probes now pass locally. Next work is review, cleanup, and any additional audit the user requests. diff --git a/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md b/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md deleted file mode 100644 index a4336c5f5..000000000 --- a/local_investigations/gemma4_multimodal_v2/01_BACKGROUND.md +++ /dev/null @@ -1,48 +0,0 @@ -# Background - -## Stable facts copied from V1 - -The older local Gemma multimodal bundles are: -- `../cactus/weights/gemma-4-e2b-it` -- `../cactus/weights/gemma-4-e2b-it-alt-fixed-rowtuned-mm` - -Both older bundles reference missing HF snapshot: -- `b324173c7d5721c2baba7f3b17b3b9b3d34ab1e9` - -The fresh investigation bundle is: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -The fresh bundle references current cached HF snapshot: -- `905e84b50c4d2a365ebde34e685027578e6728db` - -The shared-KV Gemma artifacts are text-only for this investigation because they do not include `vision_encoder` or `audio_encoder`. - -## Confirmed prior observations - -- HF reference identifies `test_monkey.png` as an orangutan/monkey-class animal. -- HF still works when using Cactus-native expanded image prompt/tensors with `pixel_position_ids` passed as HF `image_position_ids`. -- Fresh Cactus native Gemma originally failed image quality in E001, describing the monkey image as a human/fawn nose. -- Later current artifacts record image preprocessing parity and image-only generation success: - - `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` - - `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` -- Later artifacts record audio-only and mixed image+audio passing after the audio bias FP16 fix and dynamic Gemma4 audio merge: - - `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` - - `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` -- Qwen3-VL identifies the same image as a proboscis monkey. -- Parakeet transcribes the paired audio as: `What is in this image?` -- Direct MP3 input to native completion fails with `Not RIFF`; the paired audio must be converted to WAV. - -## Local paths - -Image: -- `cactus-engine/tests/assets/test_monkey.png` - -Audio: -- Source MP3: `../cactus/what_is_in_this_image.mp3` -- Converted WAV when needed: `local_integration_tests/.tmp/what_is_in_this_image.wav` - -Harness: -- `local_investigations/gemma4_multimodal_v2/scripts/run_probe.py` - -Artifacts: -- `local_investigations/gemma4_multimodal_v2/artifacts/` diff --git a/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md b/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md deleted file mode 100644 index 46965b759..000000000 --- a/local_investigations/gemma4_multimodal_v2/02_QUESTIONS.md +++ /dev/null @@ -1,41 +0,0 @@ -# Questions - -## Q1: Where is the first bad boundary? - -Status: answered for vision and audio in the current artifact set - -Candidate boundaries: -- Prompt/token IDs and media spans. -- Image preprocessing tensors. -- Vision encoder output. -- LM media merge output. -- Decoder first-token logits. -- Autoregressive decode loop. - -Resolution: -- For vision, E002/E003/E014/E015 have current artifacts and image-only is recorded as passing. -- For audio, E022/E023 localized and fixed the audio encoder bias issue, and E027/E029 localized and fixed the static media merge issue. - -## Q2: Is the fresh Gemma artifact good enough to be the default investigation artifact? - -Status: yes for current investigation - -Known evidence: -- It points at the current cached HF snapshot. -- E001 confirmed HF on the same snapshot worked while Cactus image-only originally did not. -- Later artifacts record Cactus image-only success on the fresh artifact. -- E030/E031 record audio-only and mixed image+audio passing on the fresh artifact. - -Next falsifier: -- If a fresh rerun contradicts E030/E031, inspect the changed artifact/runtime state before changing prompts. - -## Q3: Is the previous media-step fallback worth pursuing? - -Status: no as a wholesale fallback; dynamic media-step use inside chunked prefill is accepted. - -Known evidence: -- Forcing all Gemma media through the old step fallback produced gibberish on mixed prompts. -- The accepted fix uses `lm_encoder_media_step` only to assemble real prompt-position outputs before chunked decoder prefill. - -Next falsifier: -- Only revisit after proving the correct `lm_encoder_media_chunk`/step contract and how cache copies should work. diff --git a/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md b/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md deleted file mode 100644 index 46eacf99b..000000000 --- a/local_investigations/gemma4_multimodal_v2/03_HYPOTHESES.md +++ /dev/null @@ -1,66 +0,0 @@ -# Hypotheses - -## H1: Artifact staleness caused the original failure - -Status: supported but not sufficient; not the active boundary - -Evidence: -- Older multimodal artifacts reference a missing HF snapshot. -- Fresh artifact was reconverted from current cached snapshot. -- Fresh artifact originally failed native Gemma vision quality in E001. -- Later artifacts show image-only now passes after targeted fixes. - -Decision: -- Use the fresh artifact for V2 unless a stronger artifact is produced for a specific experiment. - -## H2: Prompt style or native image preprocessing is invalid - -Status: refuted for current image-only path - -Evidence: -- HF works with the normal processor prompt. -- HF also works with Cactus-native expanded prompt/tensors. -- E001 stores both records under `artifacts/E001/`. -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records image preprocessing parity as `status: pass`. -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records image-only output `The animal in the image is a golden monkey.` - -Resolution: -- Audio and mixed image+audio now pass in E030/E031; do not reopen image preprocessing without contradictory fresh evidence. - -## H3: The transpiled/C++ component path diverges after preprocessing - -Status: historically true for vision and audio; currently fixed in local artifacts - -Evidence: -- Vision had later graph/runtime/conversion fixes recorded in `06_FIX_LOG.md`. -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` records FP16-close vision output agreement. -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` records a substantial emitted `audio_features` mismatch against the Python native-like reference. -- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` records audio features as FP16-close after the audio bias fix. - -Next test: -- Rerun E030/E031 if the artifact or runtime changes. - -## H4: The C++ media merge/cache path is using the wrong component contract - -Status: supported and fixed for Gemma4 prompts containing audio - -Evidence: -- Simple `lm_encoder_media_step` fallback produced gibberish. -- Fresh artifact includes `lm_encoder_text_chunk`, `lm_encoder_media_step`, and `lm_encoder_media_chunk`, but C++ currently uses the monolithic `lm_encoder` chunk path for media. -- E025/E026 previously returned empty responses after the audio encoder fix, which made media merge/cache the next boundary. -- After the audio encoder fix, E027 showed the static `lm_encoder` chunk path selected `` for audio/mixed first tokens. -- E029/E030/E031 show dynamic prompt-position media merge fixes first-token and generation behavior. - -Next test: -- Keep dynamic merge active for Gemma4 prompts with audio and rerun regressions after runtime changes. - -## H5: Audio should remain blocked until vision is explained - -Status: retired - -Evidence: -- Vision-only has a recorded passing artifact in `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json`. -- Audio-only and mixed image+audio pass in E030/E031. - -Next test: -- None unless a fresh rerun contradicts the passing artifacts. diff --git a/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md b/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md deleted file mode 100644 index 37ef0129d..000000000 --- a/local_investigations/gemma4_multimodal_v2/04_EXPERIMENTS.md +++ /dev/null @@ -1,356 +0,0 @@ -# Experiments - -## Experiment format - -Each experiment must include: -- ID -- Question -- Setup -- Command -- Raw result path -- Interpretation -- Next decision - -## E001: Baseline ledger - -Question: -- Can the harness reproduce the known HF/Cactus/control outputs and store them in a comparable format? - -Setup: -- Fresh Gemma artifact: `../cactus/weights/gemma-4-e2b-it-fresh-mm` -- Image: `cactus-engine/tests/assets/test_monkey.png` -- Audio: converted `what_is_in_this_image.wav` - -Command: - -```bash -python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case all-baseline --experiment E001 -``` - -Raw result path: -- `artifacts/E001/` - -Interpretation: -- Completed. -- HF Gemma normal prompt: `The animal in this image is an orangutan.` -- HF Gemma Cactus-native expanded prompt/tensors: `The animal in this image is an orangutan.` -- Cactus fresh Gemma native: `The image shows a close-up of a human nose.` -- Cactus Qwen control identifies the proboscis monkey. -- Parakeet file and PCM controls both transcribe: `What is in this image?` - -Next decision: -- Move to E002 component-boundary comparison. -- The next probe must localize the first bad boundary after HF-native prompt/preprocessing, not attempt another runtime fix. - -## E002: Component-boundary comparison - -Question: -- At which component boundary does fresh Cactus native first diverge from the HF/Python reference? - -Setup: -- Same artifact, prompt, and image as E001. -- Compare in this order: input IDs/media spans, image tensors, vision encoder output, LM merge output, decoder first-token logits. - -Command: -```bash -source ../cactus/venv/bin/activate -./cactus/build.sh -python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case e002-boundary --experiment E002 -``` - -Raw result path: -- `artifacts/E002/` -- `artifacts/E002/token_spans.json` -- `artifacts/E002/image_preprocess_compare.json` -- `artifacts/E002/hf_native_inputs.json` -- `artifacts/E002/cactus_token_probe.json` -- `artifacts/E002/cactus_preprocess.json` - -Interpretation: -- Completed 2026-05-20. -- Gate A passed: HF/Python native and C++ Cactus native `input_ids` match exactly. -- Gate A image span also matches exactly: token count `282`, assistant generation start `279`, image token id `258880`, image start `5`, image token count `256`. -- Gate B failed at `pixel_values`. -- `pixel_values` shape matches: `[1, 2520, 768]`. -- `pixel_position_ids` shape and values match, with valid position count `2304`. -- `pixel_values` differ immediately at element `0`; max absolute difference is `0.1298471341896057`, mean absolute difference is `0.002023158910580417`. -- HF/Python pixel range is `[0.0, 1.0]`; C++ pixel range is `[-0.0213023126, 1.00900519]`. -- The first divergent boundary is Gemma4 native image preprocessing, before the vision encoder. - -Next decision: -- Stop E002 at Gate B for now, then continue with E003. Do not run vision encoder, LM merge, or decoder comparison until preprocessing parity is fixed or a stricter preprocessing reference proves the current C++ tensor is acceptable. -- The next fix should be scoped to `preprocess_gemma4_image` parity with `_prepare_gemma4_native_image_tensors`, including resize/interpolation and numeric range behavior. -- This is not final acceptance; the mission continues until Gemma image-only, audio-only, and mixed image+audio pass end to end. - -Superseded by later artifacts below: -- E003 / `E002_after_E003_pillow_resize` records preprocessing parity as passing. -- E015 records image-only generation as passing. -- Audio-only and mixed image+audio are now the active failures. - -## Current status correction: later artifacts after E002 - -This section records the newer artifacts currently present in the worktree. It does not erase E001/E002 history; it updates the active boundary for future agents. - -## E003 / E002_after_E003_pillow_resize: Image preprocessing parity - -Question: -- Does the C++ Gemma4 image preprocessing output match the Python/HF native reference after the Pillow-compatible resize path? - -Raw result paths: -- `artifacts/E003/preprocess_operation_compare.json` -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` - -Interpretation: -- Completed in current artifacts. -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. -- `pixel_values` max abs diff is `4.987010955859184e-07`. -- `pixel_values` mean abs diff is `2.2151478535820782e-07`. -- HF/Python range is `[0.0, 1.0]`; C++ range is `[0, 1]`. - -Next decision: -- Treat image preprocessing as fixed unless a fresh rerun contradicts this artifact. -- Continue to the vision encoder boundary. - -## E014: Latest recorded vision encoder comparison - -Question: -- Does the Cactus `vision_encoder` output match the Python/HF reference closely enough after the recorded graph/runtime/conversion fixes? - -Raw result path: -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` - -Interpretation: -- The latest recorded `pooler_output` comparison has matching shape `[256, 1536]`. -- Cosine similarity is `0.9999955754245847`. -- Max abs diff is `0.03515437499999985`. -- Mean abs diff is `0.0015996834738693023`. -- This is recorded as FP16-close. The artifact does not by itself prove every later LM merge/logit boundary, so do not overstate it as full multimodal acceptance. - -Next decision: -- Use image-only generation quality as the next acceptance check for the vision path. - -## E015: Gemma image-only quality check after vision fixes - -Question: -- Does fresh Cactus Gemma image-only generation identify `test_monkey.png` as a monkey-class animal? - -Raw result path: -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` - -Interpretation: -- Cactus returned success. -- Response: `The animal in the image is a golden monkey.` -- `prefill_tokens: 282`. -- `decode_tokens: 10`. -- This satisfies the image-only semantic acceptance class recorded in `07_ACCEPTANCE.md`. - -Next decision: -- Treat Gemma image-only as passing in the current artifact set. -- Continue to Gemma audio-only and mixed image+audio. - -## E016 / E020: Gemma audio-only quality checks - -Question: -- Does fresh Cactus Gemma audio-only generation produce coherent text using the paired audio prompt? - -Raw result paths: -- `artifacts/E016_gemma_audio_after_vision_gate/cactus_gemma_audio.json` -- `artifacts/E020_gemma_audio_after_audio_retranspile/cactus_gemma_audio.json` - -Interpretation: -- Both recorded Cactus runs returned success but no generated text. -- E020 response is empty: `response: ""`. -- E020 has `prefill_tokens: 83` and `decode_tokens: 0`. -- Retranspiling the audio encoder in the recorded E019/E020 sequence did not resolve audio-only generation. - -Next decision: -- Audio-only remains an active failure. -- Continue with audio boundary comparison before changing prompts or decoder settings. - -## E017: Gemma mixed image+audio quality check - -Question: -- Does fresh Cactus Gemma mixed image+audio generation answer the spoken question about `test_monkey.png`? - -Raw result path: -- `artifacts/E017_gemma_mixed_after_vision_gate/cactus_gemma_mixed.json` - -Interpretation: -- Cactus returned success but no generated text. -- Response is empty: `response: ""`. -- `prefill_tokens: 349`. -- `decode_tokens: 0`. - -Next decision: -- Mixed image+audio remains an active failure. -- Do not treat image-only success as full acceptance. - -## E018 / E019: Audio summary before and after audio retranspile - -Question: -- Do the recorded Cactus audio preprocessing and emitted audio feature summaries change after retranspiling the audio encoder? - -Raw result paths: -- `artifacts/E018_audio_summary/audio_summary.json` -- `artifacts/E019_audio_summary_after_retranspile/audio_summary.json` - -Interpretation: -- Both artifacts record `num_frames: 255`, `num_soft_tokens: 64`, and `feature_count: 32640`. -- Both artifacts record emitted `audio_features` shape `[1, 753, 1536]` with precision `FP16`. -- The first emitted values are unchanged across E018 and E019. -- These summaries are not enough to prove audio preprocessing parity because they only include prefixes and aggregate statistics. - -Next decision: -- Use full-tensor comparison before selecting an audio fix. - -## E021: Full Cactus-vs-Python audio feature comparison - -Question: -- Do the Cactus emitted `audio_features` match the Python native-like audio feature reference? - -Raw result paths: -- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` - -Interpretation: -- `cactus_audio_summary.json` now includes full `audio_features.values`. -- Cactus and Python output shapes match: `[1, 753, 1536]`. -- Python prepared feature shape is `[1, 3012, 128]`; Python input feature mask sum is `255`. -- For the first 64 rows, cosine similarity is `0.36957991123199463`, max abs diff is `2.796870708465576`, and mean abs diff is `0.16708998382091522`. -- For all 753 rows, cosine similarity is `0.7203282713890076`, max abs diff is `2.796870708465576`, and mean abs diff is `0.10658295452594757`. -- The `python_audio_token_count` field in this artifact is not used for conclusions because the comparison script counted a token id that is not validated here. - -Next decision: -- The current best proven active boundary is the audio encoder output / emitted audio feature boundary. -- Next work should localize the first divergent internal audio encoder operation or weight. Do not infer a specific internal cause from E021 alone. - -## E022 / E023 / E024: Audio encoder localization and FP16 bias fix - -Question: -- What caused the emitted Gemma4 `audio_features` mismatch from E021? - -Raw result paths: -- `artifacts/E022_audio_node_localization/` -- `artifacts/E023_audio_after_fp16_bias/` -- `artifacts/E024_retranspile_audio_fp16_bias/` - -Interpretation: -- E022 localized the first important divergence to the audio output projection bias handling. -- The active artifact had stored `audio_output_proj.bias` as an INT8 fallback, which made Cactus add raw values like `[1, -2, 0, 70]` instead of HF FP16-like bias values. -- After changing Gemma4 audio bias policy to FP16 and updating the active artifact, E023 records first 64 audio rows as FP16-close to Python: cosine `0.9978210926`, mean abs diff `0.00645508`. - -Next decision: -- Keep Gemma4 audio bias tensors as FP16. -- Continue to generation and first-token probes because audio-only and mixed still returned empty strings after E023. - -## E025 / E026 / E027: Post-audio-encoder generation and first-token failure - -Question: -- After fixing the audio encoder boundary, why do audio-only and mixed generation still fail? - -Raw result paths: -- `artifacts/E025_gemma_audio_after_fp16_bias/cactus_gemma_audio.json` -- `artifacts/E026_gemma_mixed_after_fp16_bias/cactus_gemma_mixed.json` -- `artifacts/E027_first_token_after_audio_encoder_fix/` - -Interpretation: -- Audio-only and mixed still returned empty responses after the audio encoder fix. -- First-token probes showed the static chunk path selected `` for both audio-only and mixed prompts, unlike HF references (`The` / `This` starts). -- Forcing the old media-step path improved audio-only first token to `Please`, but mixed produced whitespace, so wholesale fallback was not acceptable. -- The static `lm_encoder` graph was captured with stale media span assumptions and could insert audio at the wrong positions/counts for the current prompt. - -Next decision: -- Use a Gemma4-audio-only dynamic media merge inside the chunked prefill path instead of relying on stale static `lm_encoder` merge captures. - -## E029: First-token probes after dynamic merge - -Question: -- Does the dynamic Gemma4 audio merge fix first-token behavior for audio-only and mixed prompts? - -Raw result path: -- `artifacts/E029_first_token_after_dynamic_merge/` - -Interpretation: -- Audio-only first token is `Please`, with normal text alternatives in top logits. -- Mixed image+audio first token is `This`, matching the HF-style first-token direction. -- Neither probe selects ``, ``, or whitespace-only as the top token. - -Next decision: -- Continue to full generation quality checks. - -## E030: Gemma audio-only after dynamic merge - -Question: -- Does audio-only Gemma use the paired spoken prompt coherently? - -Raw result path: -- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` - -Interpretation: -- Cactus generated: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- This is coherent with the audio prompt saying `what is in this image` and no image being supplied. -- `decode_tokens: 21`. - -Next decision: -- Audio-only quality gate passes. Continue to mixed image+audio. - -## E031: Gemma mixed image+audio after dynamic merge - -Question: -- Does mixed Gemma answer the spoken question about `test_monkey.png`? - -Raw result path: -- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` - -Interpretation: -- Cactus generated: `This image is of a golden monkey.` -- This satisfies the mixed acceptance class. -- `decode_tokens: 8`. - -Next decision: -- Mixed gate passes. Run regressions. - -## E032 / E033 / E034 / E035: Regression checks after dynamic merge - -Question: -- Did the dynamic merge fix regress image-only Gemma, Qwen, Parakeet, or the audio encoder boundary? - -Raw result paths: -- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` -- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` -- `artifacts/E034_parakeet_regression_after_dynamic_merge/` -- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` - -Interpretation: -- Gemma image-only still generated: `The animal in the image is a golden monkey.` -- Qwen image control generated a proboscis-monkey answer. -- Parakeet file and PCM controls both transcribed: `What is in this image?` -- E035 shows the emitted Cactus audio tensor is unchanged from E023, with full-tensor max diff `0`. -- Policy tests passed: `16 passed`. -- Smoke tests passed: `7 passed, 2 deselected`. - -Next decision: -- The planned local acceptance gates pass. Keep the working tree uncommitted for user audit. - -## E036 / E037 / E038 / E039 / E040: Cleanup verification probes - -Question: -- After ignoring/removing generated local outputs, do the required quality probes still regenerate needed files and pass? - -Raw result paths: -- `artifacts/E036_cleanup_gemma_image/cactus_gemma_image.json` -- `artifacts/E037_cleanup_gemma_audio/cactus_gemma_audio.json` -- `artifacts/E038_cleanup_gemma_mixed/cactus_gemma_mixed.json` -- `artifacts/E039_cleanup_qwen_image/cactus_qwen_image.json` -- `artifacts/E040_cleanup_parakeet/` - -Interpretation: -- Gemma image-only still generated: `The animal in the image is a golden monkey.` -- Gemma audio-only still generated: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- Gemma mixed still generated: `This image is of a golden monkey.` -- Qwen image still generated a proboscis-monkey answer. -- Parakeet file and PCM still transcribed: `What is in this image?` -- The ignored `.tmp` directory was recreated successfully by the harness. - -Next decision: -- Cleanup did not regress the acceptance probes or remove benchmark/test harness source. diff --git a/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md b/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md deleted file mode 100644 index 922eef32b..000000000 --- a/local_investigations/gemma4_multimodal_v2/05_DECISIONS.md +++ /dev/null @@ -1,118 +0,0 @@ -# Decisions - -## D1: Do not use older multimodal Gemma artifacts as the default - -Decision: -- Use `gemma-4-e2b-it-fresh-mm` for V2 baseline work. - -Reason: -- Older artifacts reference a missing HF snapshot. - -## D2: Do not continue the simple media-step fallback - -Decision: -- Do not disable Gemma chunk prefill wholesale to force `lm_encoder_media_step`. - -Reason: -- It produced gibberish on old and fresh artifacts. - -## D3: Treat quality probes as acceptance, not root-cause tools - -Decision: -- Use quality probes to confirm behavior, but debug via component boundary comparison. - -Reason: -- Human text output identified the issue but did not localize the first bad boundary. - -## D4: First bad boundary is Gemma4 image preprocessing - -Decision: -- Treat E002 Gate B as the first proven divergence. -- Do not debug vision encoder, LM merge, or decoder logits until Gemma4 native preprocessing matches the HF/Python reference or a stronger reference refutes the HF/Python preprocessing tensor as the correct target. - -Reason: -- E002 Gate A proves prompt `input_ids` and image token spans match exactly between HF/Python native and C++ Cactus native. -- E002 Gate B proves `pixel_position_ids` match, but `pixel_values` differ immediately and outside tiny tolerance. -- The mismatch exists before any graph runtime, media merge, cache, or decoder behavior is involved. - -## D5: Localization is not acceptance - -Decision: -- Boundary localization is now treated only as a checkpoint. -- The investigation remains active until Gemma4 native works end to end across image-only, audio-only, and mixed image+audio, with controls and smoke still passing. - -Reason: -- The user explicitly raised the acceptance bar from identifying the cause to making the model fully work end to end. - -## D6: Image preprocessing is no longer the active boundary - -Decision: -- Treat Gemma4 image preprocessing parity as fixed in the current artifact set. - -Reason: -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. -- The recorded `pixel_values` max abs diff is `4.987010955859184e-07`, and mean abs diff is `2.2151478535820782e-07`. - -## D7: Image-only generation is currently accepted, but full multimodal is not - -Decision: -- Treat Gemma image-only as passing in the current artifact set. -- Do not treat the investigation as complete. - -Reason: -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records the response `The animal in the image is a golden monkey.` -- `07_ACCEPTANCE.md` allows monkey/orangutan/proboscis monkey class answers for image-only. -- Audio-only and mixed image+audio still return empty responses in the latest recorded artifacts. - -## D8: Superseded historical audio / mixed failure - -Decision: -- This was the correct decision at E021, but it is superseded by D10 and D11 after E023/E029/E030/E031. -- Do not return to prompt tweaks or decoder parameters unless a fresh rerun contradicts the passing artifacts. - -Reason: -- `artifacts/E020_gemma_audio_after_audio_retranspile/cactus_gemma_audio.json` records `response: ""` and `decode_tokens: 0`. -- `artifacts/E017_gemma_mixed_after_vision_gate/cactus_gemma_mixed.json` records `response: ""` and `decode_tokens: 0`. -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` records matching Cactus/Python audio feature output shapes but poor first-64-row agreement: cosine `0.36957991123199463`, mean abs diff `0.16708998382091522`. - -## D9: E021 localizes a boundary, not an internal cause - -Decision: -- Use E021 only to justify investigating inside the audio encoder path. -- Do not claim a specific audio op, weight, runtime kernel, or conversion policy is wrong until a smaller comparison identifies it. - -Reason: -- E021 compares full emitted audio features against a Python native-like reference and proves a mismatch at that boundary. -- It does not identify the first divergent node inside the audio encoder. - -## D10: Gemma4 audio bias tensors must stay FP16 - -Decision: -- Keep Gemma4 audio component bias tensors as FP16 fallback tensors. - -Reason: -- E022 localized the audio encoder divergence to the output projection bias handling. -- E023 shows the FP16 bias policy brings the emitted audio features close to the Python reference. -- Storing these biases as INT8 caused Cactus to add raw integer-like values instead of the expected FP16 bias values. - -## D11: Use dynamic Gemma4 merge for prompts with audio - -Decision: -- For Gemma4 prompts with audio, do not use the static full `lm_encoder` merge graph. -- Dynamically walk the actual prompt tokens, route text through `lm_encoder_step`, route image/audio token rows through `lm_encoder_media_step`, then feed the assembled tensors to `decoder_prefill_chunk`. - -Reason: -- E027 showed the static chunk path selected `` for audio-only and mixed prompts after the audio encoder boundary was fixed. -- The static `lm_encoder` graph can encode stale media span assumptions from capture time. -- E029/E030/E031 show the dynamic merge produces normal first tokens and passing end-to-end audio/mixed outputs. - -## D12: Do not carry vision changes from `justin/v2-cpp-fix` - -Decision: -- Treat `origin/justin/v2-cpp-fix` as already absorbed baseline infrastructure. -- Carry over only the later dynamic Gemma4 audio merge idea from `77f808ed`, adapted to the current branch structure. - -Reason: -- The latest `justin/v2-cpp-fix` commit is already in this branch line. -- Vision already has passing local artifacts. -- The relevant audio fix pattern is the dynamic media walk, not a vision-path change. diff --git a/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md b/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md deleted file mode 100644 index 5f2ec53d7..000000000 --- a/local_investigations/gemma4_multimodal_v2/06_FIX_LOG.md +++ /dev/null @@ -1,156 +0,0 @@ -# Fix Log - -Current status: V2 has recorded fixes in the worktree. Do not use the old `No V2 fixes yet` state. - -Rules: -- A code change must cite a supported hypothesis and experiment ID. -- A failed fix must be reverted or explicitly documented as intentionally retained. -- Each fix record must include before/after quality and nearest regression result. - -Historical fixes from before V2: -- Audio incremental decode changed to feed only the last token. -- Audio encoder mask changed to mark valid frames only. -- Fresh Gemma artifact reconverted as `gemma-4-e2b-it-fresh-mm`. -- Simple media-step fallback was tested and reverted. - -## Local investigation tooling - -2026-05-20: -- Added local-only E002 probe commands to `local_integration_tests/native_runner.cpp`: - - `e002-tokens` - - `e002-preprocess` -- Updated `local_integration_tests/CMakeLists.txt` so the local runner can include internal engine headers. -- Extended `scripts/run_probe.py` with `--case e002-boundary`. -- These are investigation artifacts only, not runtime fixes. - -## F1: Gemma4 image preprocessing parity - -Supported by: -- `artifacts/E003/preprocess_operation_compare.json` -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` - -Change: -- `cactus-engine/src/engine_image.cpp` now has a Pillow-compatible uint8 bilinear resize path used by `preprocess_gemma4_image`. - -Before: -- `artifacts/E002/image_preprocess_compare.json` recorded `pixel_values` max abs diff `0.1298471341896057` and mean abs diff `0.002023158910580417`. -- C++ pixel range extended outside `[0, 1]`. - -After: -- `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` records `status: pass`. -- `pixel_values` max abs diff is `4.987010955859184e-07`. -- `pixel_values` mean abs diff is `2.2151478535820782e-07`. - -## F2: Vision graph/runtime/transpile fixes recorded by later artifacts - -Supported by: -- `artifacts/E006_gateC_after_fp16_embedding_indices/vision_encoder_compare.json` -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` - -Changes visible in the worktree: -- `cactus-graph/src/ops_tensor.cpp` handles FP16 embedding indices in `compute_embedding_node`. -- `cactus-graph/src/ops_math.cpp` adds FP32 binary and reduction support used by scalar/math lowering paths. -- `python/cactus/transpile/lower.py` preserves tensor dtype in `_ensure_scalar_math_tensor`. -- `python/cactus/transpile/runtime_compat.py` preserves tensor dtype for scalar ops, `abs`, and `pow`. -- `python/cactus/transpile/model_adapters.py` updates Gemma4 native-like vision pooling/projection behavior. - -Evidence: -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` records FP16-close `pooler_output` agreement: cosine `0.9999955754245847`, mean abs diff `0.0015996834738693023`. -- `artifacts/E015_gemma_image_after_vision_gate/cactus_gemma_image.json` records image-only output: `The animal in the image is a golden monkey.` - -Limit: -- This does not prove audio-only or mixed image+audio acceptance. - -## F3: Gemma4 vision projection conversion policy and active artifact adjustment - -Supported by: -- `artifacts/E013_gateC_after_fp16_projection_graph/vision_encoder_compare.json` -- `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` - -Changes visible in the worktree: -- `python/cactus/convert/model_adapters/policy.py` makes `model.embed_vision.embedding_projection.weight` an FP16 fallback with reason `vision embedding projection scale-sensitive`. -- `python/cactus/convert/model_adapters/adapters.py` no longer overrides that policy back to CQ. -- `python/cactus/convert/model_adapters/naming.py` and `python/cactus/convert/cactus_adapters/tensor_io.py` remove `embed_vision_proj` from Gemma4 divide-scale handling. -- `python/cactus/convert/tests/test_policy.py` now expects FP16 for the Gemma4 vision projection policy. - -Evidence: -- The latest recorded vision comparison improves to FP16-close agreement in `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json`. - -## F4: Audio full-tensor diagnostic instrumentation - -Supported by: -- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` - -Change: -- `local_integration_tests/native_runner.cpp` now emits full `audio_features.values` for `e002-audio-summary`. - -Evidence: -- `artifacts/E021_audio_full_compare/cactus_audio_summary.json` contains full emitted `audio_features.values`. -- `artifacts/E021_audio_full_compare/audio_encoder_compare.json` compares those values against Python native-like audio features. - -Limit: -- This is diagnostic instrumentation only. -- It proves the emitted audio feature boundary mismatches, but not which internal audio encoder operation or weight is the first cause. - -## F5: Gemma4 audio bias FP16 policy and artifact adjustment - -Supported by: -- `artifacts/E022_audio_node_localization/` -- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` -- `artifacts/E024_retranspile_audio_fp16_bias/` - -Changes: -- `python/cactus/convert/model_adapters/policy.py` keeps Gemma4 audio component bias tensors in FP16. -- `python/cactus/convert/tests/test_policy.py` expects FP16 for Gemma4 audio bias policy. -- The active fresh Gemma artifact has an FP16 `audio_output_proj.bias` and regenerated `audio_encoder` graph artifacts. - -Before: -- E021 first 64 emitted audio rows had cosine `0.3695799112` and mean abs diff `0.16708998` versus Python. - -After: -- E023 first 64 emitted audio rows have cosine `0.9978210926` and mean abs diff `0.00645508` versus Python. - -## F6: Dynamic Gemma4 audio/media merge for chunked prefill - -Supported by: -- `artifacts/E027_first_token_after_audio_encoder_fix/` -- `artifacts/E029_first_token_after_dynamic_merge/` -- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` -- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` - -Changes: -- `cactus-engine/src/model.cpp` now builds Gemma4 LM encoder outputs dynamically for prompts containing audio. -- The dynamic path walks actual prompt tokens, uses `lm_encoder_step` for text, uses `lm_encoder_media_step` for image/audio token rows, and feeds the assembled outputs into `decoder_prefill_chunk`. -- `cactus-engine/src/engine.h` declares the dynamic helper. -- The existing static `lm_encoder` merge path remains for non-audio cases. - -Before: -- E025/E026 still returned empty audio-only and mixed responses after the audio encoder fix. -- E027 first-token probes selected `` for both audio-only and mixed prompts on the static chunk path. - -After: -- E029 first-token probes produce audio `Please` and mixed `This`. -- E030 audio-only produces coherent text responding to the spoken image question. -- E031 mixed image+audio produces `This image is of a golden monkey.` -- E032/E033/E034/E035 and smoke/policy tests record no local regression. - -## F7: Cleanup of generated local outputs - -Supported by: -- `artifacts/E036_cleanup_gemma_image/cactus_gemma_image.json` -- `artifacts/E037_cleanup_gemma_audio/cactus_gemma_audio.json` -- `artifacts/E038_cleanup_gemma_mixed/cactus_gemma_mixed.json` -- `artifacts/E039_cleanup_qwen_image/cactus_qwen_image.json` -- `artifacts/E040_cleanup_parakeet/` - -Changes: -- `.gitignore` now ignores local integration `.tmp` output, local Python bytecode, and local investigation artifact directories. -- Removed rebuildable local `.tmp` and `__pycache__` directories. -- Benchmark source/results under `benchmarks/` were not changed. - -Evidence: -- Policy tests passed after cleanup: `16 passed`. -- Smoke tests passed after cleanup: `7 passed, 2 deselected`. -- E036-E040 quality probes passed after cleanup. diff --git a/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md b/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md deleted file mode 100644 index 3e6887129..000000000 --- a/local_investigations/gemma4_multimodal_v2/07_ACCEPTANCE.md +++ /dev/null @@ -1,50 +0,0 @@ -# Acceptance - -## End-to-end gate - -Gemma4 native multimodal work is accepted only when the full user path works: -- Image-only Gemma correctly identifies `test_monkey.png` as a monkey/orangutan/proboscis monkey class animal. -- Audio-only Gemma shows it uses the paired spoken prompt coherently. -- Mixed image+audio Gemma answers the spoken question about `test_monkey.png` correctly. -- Any component boundary divergence found during debugging has a before/after artifact showing the fix. -- Qwen image control still works. -- Parakeet file and PCM controls still transcribe coherently. -- Smoke tests pass. - -Localization is not acceptance. It is a checkpoint that determines the next fix. - -## Vision gate - -Gemma native vision is accepted only when: -- HF and Cactus use the same image and semantically equivalent prompt. -- The first bad component boundary has been identified and fixed or ruled out. -- Cactus output identifies the monkey image as a monkey/orangutan/proboscis monkey class animal. -- Qwen image control still works. - -## Audio gate - -Gemma audio work starts after vision has a known-good path. A documented vision boundary failure is not enough to accept the mission. - -Audio is accepted only when: -- The paired WAV is verified by Parakeet. -- Gemma audio preprocessing and audio encoder behavior are compared against the Python/HF reference. -- Cactus output shows evidence of using the spoken content. - -## Mixed gate - -Mixed image+audio is accepted only after vision and audio have each passed their gates and the combined path works. - -Mixed image+audio must use the actual paired test assets: -- Image: `cactus-engine/tests/assets/test_monkey.png` -- Audio prompt: `what is in this image` -- Expected answer: a short coherent answer identifying the monkey/orangutan/proboscis monkey class. - -## Regression gate - -After any code fix: - -```bash -source ../cactus/venv/bin/activate -./cactus/build.sh -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s -``` diff --git a/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md b/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md deleted file mode 100644 index 9a27aa908..000000000 --- a/local_investigations/gemma4_multimodal_v2/08_HANDOFF.md +++ /dev/null @@ -1,44 +0,0 @@ -# Handoff - -Status: local acceptance probes pass; leave uncommitted for user audit. - -Current proven state: -- Gemma4 image preprocessing parity is fixed. -- Gemma4 vision/image-only generation passes. -- Gemma4 audio encoder output is FP16-close to Python after the audio bias FP16 fix. -- Gemma4 audio-only generation is coherent. -- Gemma4 mixed image+audio answers the spoken question about `test_monkey.png` correctly. -- Qwen image, Parakeet file/PCM, policy tests, and smoke tests pass. - -## Key artifacts - -- Image preprocessing: `artifacts/E002_after_E003_pillow_resize/image_preprocess_compare.json` -- Vision encoder: `artifacts/E014_gateC_after_unscaled_fp16_projection/vision_encoder_compare.json` -- Audio encoder after bias fix: `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` -- First-token dynamic merge: `artifacts/E029_first_token_after_dynamic_merge/` -- Gemma audio-only: `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` -- Gemma mixed: `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` -- Gemma image regression: `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` -- Qwen regression: `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` -- Parakeet regression: `artifacts/E034_parakeet_regression_after_dynamic_merge/` -- Audio encoder unchanged after dynamic merge: `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` - -## Latest outputs - -- Gemma image-only: `The animal in the image is a golden monkey.` -- Gemma audio-only: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- Gemma mixed: `This image is of a golden monkey.` -- Qwen image: identifies a proboscis monkey. -- Parakeet file and PCM: `What is in this image?` - -## Tests - -- Build completed with `source ../cactus/venv/bin/activate && ./cactus/build.sh`. -- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q` -> `16 passed`. -- `PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s` -> `7 passed, 2 deselected`. - -## Review focus - -- Runtime: dynamic Gemma4 audio merge in `cactus-engine/src/model.cpp`. -- Policy: Gemma4 audio bias FP16 handling in `python/cactus/convert/model_adapters/policy.py`. -- Artifact changes: active fresh Gemma audio encoder graph/bias files were adjusted during the investigation. diff --git a/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md b/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md deleted file mode 100644 index da4b6605a..000000000 --- a/local_investigations/gemma4_multimodal_v2/09_STATUS_FOR_NEXT_AGENT.md +++ /dev/null @@ -1,74 +0,0 @@ -# Status For Next Agent - -## One-line status - -Gemma4 image-only, audio-only, and mixed image+audio now pass local end-to-end probes after the audio bias FP16 fix and dynamic Gemma4 audio merge. Qwen, Parakeet, policy, and smoke controls also pass. - -## Current working state - -Repo: -- `/Users/noahcylich/Documents/Desert/cactus-v2-chunked-prefill` - -Branch: -- `v2-chunked-prefill` - -Do not commit or push unless the user explicitly asks. The worktree contains unrelated and related tracked changes plus local investigation artifacts. - -Default Gemma artifact: -- `../cactus/weights/gemma-4-e2b-it-fresh-mm` - -HF reference snapshot: -- `/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db` - -Assets: -- Image: `cactus-engine/tests/assets/test_monkey.png` -- Audio WAV: `local_integration_tests/.tmp/what_is_in_this_image.wav` -- Source MP3: `../cactus/what_is_in_this_image.mp3` - -## Fixes applied - -- Gemma4 audio component bias tensors are kept FP16. E023 shows the audio encoder boundary became FP16-close to Python after this. -- Gemma4 prompts containing audio now bypass the stale static full `lm_encoder` merge graph. The runtime dynamically walks the actual token sequence, routes text through `lm_encoder_step`, routes image/audio token rows through `lm_encoder_media_step`, assembles the LM encoder outputs, and feeds `decoder_prefill_chunk`. -- This keeps the existing static path for image-only and non-audio cases. - -## Latest proof artifacts - -- `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json`: first 64 audio rows cosine `0.9978210926`, mean abs diff `0.00645508` versus Python. -- `artifacts/E029_first_token_after_dynamic_merge/`: audio first token `Please`, mixed first token `This`; no `` or immediate stop. -- `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json`: audio-only response is coherent with the spoken prompt: `Please provide the image you are referring to. I need an image to tell you what is in it.` -- `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json`: mixed response: `This image is of a golden monkey.` -- `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json`: image-only response: `The animal in the image is a golden monkey.` -- `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json`: Qwen identifies a proboscis monkey. -- `artifacts/E034_parakeet_regression_after_dynamic_merge/`: Parakeet file and PCM both transcribe `What is in this image?` -- `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json`: full emitted audio tensor matches E023 exactly, max diff `0`. - -## Latest commands run - -Required build: - -```bash -source ../cactus/venv/bin/activate -./cactus/build.sh -``` - -Policy: - -```bash -PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q -``` - -Result: -- `16 passed` - -Smoke: - -```bash -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s -``` - -Result: -- `7 passed, 2 deselected` - -## Remaining work - -No known local acceptance failure remains from the planned test matrix. Next step is user audit/review of the uncommitted changes. diff --git a/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md b/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md deleted file mode 100644 index 643082b7b..000000000 --- a/local_investigations/gemma4_multimodal_v2/10_CONTINUATION_PLAN.md +++ /dev/null @@ -1,58 +0,0 @@ -# Continuation Plan - -## Goal - -Keep fresh Gemma4 Cactus native working end to end for image-only, audio-only, and mixed image+audio, while preserving Qwen, Parakeet, and smoke controls. - -The local acceptance plan now passes. This document is for review and any follow-up hardening, not for continuing a known active failure. - -## Current local result - -- Gemma image-only: passes, identifies `test_monkey.png` as a golden monkey. -- Gemma audio-only: passes, responds coherently to the spoken `what is in this image` prompt when no image is supplied. -- Gemma mixed image+audio: passes, answers `This image is of a golden monkey.` -- Qwen image control: passes, identifies a proboscis monkey. -- Parakeet file and PCM controls: pass, both transcribe `What is in this image?` -- Policy tests: pass, `16 passed`. -- Smoke tests: pass, `7 passed, 2 deselected`. - -## Important implementation decisions - -- Keep Gemma4 audio bias tensors FP16. -- For Gemma4 prompts with audio, use the dynamic media merge path rather than the static full `lm_encoder` merge graph. -- Do not carry additional vision behavior from `justin/v2-cpp-fix`; that branch is already in the ancestry and vision already passes. -- Keep non-audio/image-only behavior on the existing static chunk path. - -## Verification artifacts - -- Audio encoder boundary: `artifacts/E023_audio_after_fp16_bias/audio_encoder_compare.json` -- First-token dynamic merge: `artifacts/E029_first_token_after_dynamic_merge/` -- Gemma audio-only: `artifacts/E030_gemma_audio_after_dynamic_merge/cactus_gemma_audio.json` -- Gemma mixed: `artifacts/E031_gemma_mixed_after_dynamic_merge/cactus_gemma_mixed.json` -- Gemma image regression: `artifacts/E032_gemma_image_regression_after_dynamic_merge/cactus_gemma_image.json` -- Qwen regression: `artifacts/E033_qwen_image_regression_after_dynamic_merge/cactus_qwen_image.json` -- Parakeet regression: `artifacts/E034_parakeet_regression_after_dynamic_merge/` -- Audio encoder unchanged after dynamic merge: `artifacts/E035_audio_encoder_regression_after_dynamic_merge/audio_encoder_regression_compare.json` - -## If further testing is requested - -Run the same sequence under the required build rule: - -```bash -source ../cactus/venv/bin/activate -./cactus/build.sh -``` - -Then rerun: - -```bash -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-audio --experiment -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-mixed --experiment -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-gemma-image --experiment -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case cactus-qwen-image --experiment -PYTHONPATH=python ../cactus/venv/bin/python local_investigations/gemma4_multimodal_v2/scripts/run_probe.py --case parakeet-audio --experiment -PYTHONPATH=python ../cactus/venv/bin/python -m pytest python/cactus/convert/tests/test_policy.py -q -PYTHONPATH=python ../cactus/venv/bin/python -m pytest local_integration_tests -m smoke -q -s -``` - -Do not commit or push until the user audits the changes. diff --git a/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py b/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py deleted file mode 100644 index cb12337b1..000000000 --- a/local_investigations/gemma4_multimodal_v2/scripts/run_probe.py +++ /dev/null @@ -1,785 +0,0 @@ -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import subprocess -import sys -import time -from pathlib import Path -from typing import Any - - -REPO_ROOT = Path(__file__).resolve().parents[3] -INVESTIGATION_ROOT = Path(__file__).resolve().parents[1] -DEFAULT_WEIGHTS_ROOT = REPO_ROOT.parent / "cactus" / "weights" -ASSET_DIR = REPO_ROOT / "cactus-engine" / "tests" / "assets" -TMP_DIR = REPO_ROOT / "local_integration_tests" / ".tmp" - - -def run(cmd: list[str], *, timeout: int = 300) -> subprocess.CompletedProcess[str]: - env = os.environ.copy() - env["CACTUS_NO_CLOUD_TELE"] = "1" - return subprocess.run( - cmd, - cwd=REPO_ROOT, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - check=False, - ) - - -def write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - - -def command_record(case: str, command: list[str], completed: subprocess.CompletedProcess[str]) -> dict[str, Any]: - parsed: Any = None - if completed.stdout.strip().startswith("{"): - try: - parsed = json.loads(completed.stdout) - except json.JSONDecodeError: - parsed = None - return { - "case": case, - "command": command, - "returncode": completed.returncode, - "stdout": completed.stdout, - "stderr": completed.stderr, - "parsed_stdout": parsed, - } - - -def ensure_runner() -> Path: - build_dir = TMP_DIR / "build" - runner = build_dir / "integration_runner" - sources = [ - REPO_ROOT / "local_integration_tests" / "native_runner.cpp", - REPO_ROOT / "local_integration_tests" / "CMakeLists.txt", - REPO_ROOT / "cactus" / "build" / "libcactus.a", - ] - if runner.exists() and all(runner.stat().st_mtime >= path.stat().st_mtime for path in sources if path.exists()): - return runner - cmake = run( - [ - "cmake", - "-S", - str(REPO_ROOT / "local_integration_tests"), - "-B", - str(build_dir), - f"-DCACTUS_REPO_ROOT={REPO_ROOT}", - "-DCMAKE_BUILD_TYPE=Release", - ], - timeout=120, - ) - if cmake.returncode != 0: - raise RuntimeError(cmake.stderr or cmake.stdout) - build = run(["cmake", "--build", str(build_dir), "--target", "integration_runner", "-j", str(os.cpu_count() or 4)], timeout=120) - if build.returncode != 0: - raise RuntimeError(build.stderr or build.stdout) - if not runner.exists(): - raise RuntimeError(f"integration runner was not built: {runner}") - return runner - - -def ensure_question_wav() -> Path: - wav = TMP_DIR / "what_is_in_this_image.wav" - if wav.exists(): - return wav - mp3 = REPO_ROOT.parent / "cactus" / "what_is_in_this_image.mp3" - if not mp3.exists(): - raise RuntimeError(f"missing paired mp3: {mp3}") - wav.parent.mkdir(parents=True, exist_ok=True) - completed = run( - [ - "ffmpeg", - "-y", - "-hide_banner", - "-loglevel", - "error", - "-i", - str(mp3), - "-ac", - "1", - "-ar", - "16000", - "-sample_fmt", - "s16", - str(wav), - ], - timeout=120, - ) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - return wav - - -def cactus_complete(model: Path, prompt: str, image: Path | None = None, audio: Path | None = None, max_tokens: int = 60) -> dict[str, Any]: - runner = ensure_runner() - command: list[str] = [ - str(runner), - "complete", - "--model", - str(model), - "--prompt", - prompt, - "--max-tokens", - str(max_tokens), - ] - if image is not None: - command.extend(["--image", str(image)]) - if audio is not None: - command.extend(["--audio", str(audio)]) - return command_record("cactus_complete", command, run(command, timeout=360)) - - -def native_json(command: list[str], *, timeout: int = 360) -> dict[str, Any]: - completed = run(command, timeout=timeout) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - try: - return json.loads(completed.stdout) - except json.JSONDecodeError as exc: - raise RuntimeError(f"native command did not return JSON: {command}\n{completed.stdout[:1000]}") from exc - - -def parakeet_transcribe(model: Path, audio: Path, pcm: bool = False) -> dict[str, Any]: - runner = ensure_runner() - command = [str(runner), "transcribe-pcm" if pcm else "transcribe", "--model", str(model), "--audio", str(audio)] - return command_record("parakeet_transcribe_pcm" if pcm else "parakeet_transcribe", command, run(command, timeout=240)) - - -def hf_gemma_image(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: - script = f""" -import json -import torch -from pathlib import Path -from transformers import AutoProcessor, AutoModelForImageTextToText - -model_dir = Path({str(model_dir)!r}) -image_path = Path({str(image)!r}) -prompt = {prompt!r} -processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) -model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='auto') -messages = [{{'role': 'user', 'content': [{{'type': 'image', 'image': str(image_path)}}, {{'type': 'text', 'text': prompt}}]}}] -inputs = processor.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors='pt') -inputs = {{k: v.to(model.device) if hasattr(v, 'to') else v for k, v in inputs.items()}} -with torch.no_grad(): - out = model.generate(**inputs, max_new_tokens=40, do_sample=False) -text = processor.decode(out[0][inputs['input_ids'].shape[-1]:], skip_special_tokens=True) -print(json.dumps({{'response': text, 'input_tokens': int(inputs['input_ids'].shape[-1])}})) -""" - command = [sys.executable, "-c", script] - return command_record("hf_gemma_image", command, run(command, timeout=360)) - - -def hf_gemma_native_image(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: - script = f""" -import json -import torch -from pathlib import Path -from transformers import AutoProcessor, AutoModelForImageTextToText -from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs - -model_dir = Path({str(model_dir)!r}) -image_path = Path({str(image)!r}) -prompt = {prompt!r} -processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) -prepared = prepare_gemma4_multimodal_inputs( - processor, - prompt=prompt, - image_files=(str(image_path),), - audio_file=None, - torch_dtype=torch.float16, - use_gemma4_chat_template=True, -) -model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='auto') -inputs = {{}} -summaries = {{}} -for name, tensor in zip(prepared.names, prepared.tensors): - key = 'image_position_ids' if name == 'pixel_position_ids' else name - inputs[key] = tensor.to(model.device) - summaries[name] = {{ - 'shape': list(tensor.shape), - 'dtype': str(tensor.dtype), - 'min': float(tensor.float().min().item()) if tensor.numel() else 0.0, - 'max': float(tensor.float().max().item()) if tensor.numel() else 0.0, - 'mean': float(tensor.float().mean().item()) if tensor.numel() else 0.0, - }} -with torch.no_grad(): - out = model.generate(**inputs, max_new_tokens=40, do_sample=False) -text = processor.decode(out[0][inputs['input_ids'].shape[-1]:], skip_special_tokens=True) -input_ids = prepared.tensors[prepared.names.index('input_ids')] -print(json.dumps({{ - 'response': text, - 'input_tokens': int(input_ids.shape[-1]), - 'input_shapes': prepared.metadata.get('input_shapes'), - 'processor_prompt_prefix': str(prepared.metadata.get('processor_prompt'))[:240], - 'tensor_summaries': summaries, -}})) -""" - command = [sys.executable, "-c", script] - return command_record("hf_gemma_native_image", command, run(command, timeout=360)) - - -def hf_gemma_e002_inputs(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: - script = f""" -import hashlib -import json -import torch -from pathlib import Path -from transformers import AutoProcessor -from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs - -model_dir = Path({str(model_dir)!r}) -image_path = Path({str(image)!r}) -prompt = {prompt!r} -processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) -prepared = prepare_gemma4_multimodal_inputs( - processor, - prompt=prompt, - image_files=(str(image_path),), - audio_file=None, - torch_dtype=torch.float32, - use_gemma4_chat_template=True, -) -by_name = {{name: tensor.detach().cpu() for name, tensor in zip(prepared.names, prepared.tensors)}} -input_ids = by_name['input_ids'].reshape(-1).to(torch.int64) -pixel_values = by_name['pixel_values'].to(torch.float32).contiguous() -pixel_position_ids = by_name['pixel_position_ids'].reshape(-1).to(torch.int64).contiguous() -tokenizer = getattr(processor, 'tokenizer', processor) -image_token = getattr(processor, 'image_token', None) -image_token_id = tokenizer.convert_tokens_to_ids(image_token) if image_token is not None and hasattr(tokenizer, 'convert_tokens_to_ids') else -1 -ids = input_ids.tolist() -image_positions = [i for i, tok in enumerate(ids) if tok == image_token_id] -processor_prompt = str(prepared.metadata.get('processor_prompt')) -assistant_generation_start = len(ids) -if processor_prompt.endswith('<|turn>model\\n'): - no_generation_prompt = processor_prompt[:-len('<|turn>model\\n')] - no_gen = tokenizer(no_generation_prompt, return_tensors='pt', add_special_tokens=False)['input_ids'].reshape(-1) - assistant_generation_start = int(no_gen.numel()) -valid_positions = int(((by_name['pixel_position_ids'] != -1).any(dim=-1)).sum().item()) -pixel_np = pixel_values.numpy() -pos_np = by_name['pixel_position_ids'].to(torch.int64).contiguous().numpy() -payload = {{ - 'token_count': len(ids), - 'assistant_generation_start': assistant_generation_start, - 'image_token_id': int(image_token_id), - 'image_token_start': int(image_positions[0]) if image_positions else -1, - 'image_token_count': len(image_positions), - 'first_20': ids[:20], - 'last_20': ids[-20:], - 'input_ids': ids, - 'prompt_prefix': processor_prompt[:320], - 'pixel_values_shape': list(pixel_values.shape), - 'pixel_position_ids_shape': list(by_name['pixel_position_ids'].shape), - 'pixel_values_min': float(pixel_values.min().item()), - 'pixel_values_max': float(pixel_values.max().item()), - 'pixel_values_mean': float(pixel_values.mean().item()), - 'pixel_values_sha256': hashlib.sha256(pixel_np.tobytes()).hexdigest(), - 'pixel_position_ids_sha256': hashlib.sha256(pos_np.tobytes()).hexdigest(), - 'valid_position_count': valid_positions, - 'first_pixel_values': pixel_values.reshape(-1)[:32].tolist(), - 'first_position_rows_flat': pixel_position_ids[:32].tolist(), - 'pixel_values': pixel_values.reshape(-1).tolist(), - 'pixel_position_ids': pixel_position_ids.tolist(), -}} -print(json.dumps(payload)) -""" - command = [sys.executable, "-c", script] - completed = run(command, timeout=360) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - return json.loads(completed.stdout) - - -def e002_cactus_tokens(model: Path, image: Path, prompt: str) -> dict[str, Any]: - runner = ensure_runner() - command = [ - str(runner), - "e002-tokens", - "--model", - str(model), - "--prompt", - prompt, - "--image", - str(image), - ] - return native_json(command, timeout=360) - - -def e002_cactus_preprocess(model: Path, image: Path) -> dict[str, Any]: - runner = ensure_runner() - command = [ - str(runner), - "e002-preprocess", - "--model", - str(model), - "--image", - str(image), - ] - return native_json(command, timeout=360) - - -def e003_cactus_preprocess_variants(model: Path, image: Path) -> dict[str, Any]: - runner = ensure_runner() - command = [ - str(runner), - "e003-preprocess-variants", - "--model", - str(model), - "--image", - str(image), - ] - return native_json(command, timeout=360) - - -def e002_cactus_vision(model: Path, image: Path) -> dict[str, Any]: - runner = ensure_runner() - command = [ - str(runner), - "e002-vision", - "--model", - str(model), - "--image", - str(image), - "--prompt", - "What animal is in this image? Reply in one complete short sentence.", - ] - return native_json(command, timeout=360) - - -def hf_gemma_vision_features(model_dir: Path, image: Path, prompt: str) -> dict[str, Any]: - script = f""" -import json -import torch -from pathlib import Path -from transformers import AutoProcessor, AutoModelForImageTextToText -from cactus.transpile.multimodal_runtime import prepare_gemma4_multimodal_inputs - -model_dir = Path({str(model_dir)!r}) -image_path = Path({str(image)!r}) -prompt = {prompt!r} -processor = AutoProcessor.from_pretrained(model_dir, local_files_only=True) -prepared = prepare_gemma4_multimodal_inputs( - processor, - prompt=prompt, - image_files=(str(image_path),), - audio_file=None, - torch_dtype=torch.float16, - use_gemma4_chat_template=True, -) -by_name = {{name: tensor for name, tensor in zip(prepared.names, prepared.tensors)}} -model = AutoModelForImageTextToText.from_pretrained(model_dir, local_files_only=True, dtype=torch.float16, device_map='cpu') -pixel_values = by_name['pixel_values'].to(dtype=torch.float16) -position_ids = by_name['pixel_position_ids'].to(dtype=torch.long) -with torch.no_grad(): - outputs = model.get_image_features(pixel_values=pixel_values, image_position_ids=position_ids) -candidates = {{}} -for name, tensor in ( - ('last_hidden_state', getattr(outputs, 'last_hidden_state', None)), - ('pooler_output', getattr(outputs, 'pooler_output', None)), -): - if tensor is None: - continue - flat = tensor.detach().float().cpu().reshape(-1) - candidates[name] = {{ - 'shape': list(tensor.shape), - 'dtype': str(tensor.dtype), - 'count': int(flat.numel()), - 'min': float(flat.min().item()) if flat.numel() else 0.0, - 'max': float(flat.max().item()) if flat.numel() else 0.0, - 'mean': float(flat.mean().item()) if flat.numel() else 0.0, - 'l2': float(torch.linalg.vector_norm(flat).item()) if flat.numel() else 0.0, - 'first_values': flat[:32].tolist(), - 'values': flat.tolist(), - }} -print(json.dumps({{'features': candidates}})) -""" - completed = run([sys.executable, "-c", script], timeout=420) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - return json.loads(completed.stdout) - - -def first_mismatch(left: list[Any], right: list[Any]) -> int | None: - for idx, (lval, rval) in enumerate(zip(left, right)): - if lval != rval: - return idx - if len(left) != len(right): - return min(len(left), len(right)) - return None - - -def sha256_float_list(values: list[float]) -> str: - import array - - arr = array.array("f", values) - return hashlib.sha256(arr.tobytes()).hexdigest() - - -def run_e002_boundary(out_dir: Path, weights_root: Path) -> None: - image = ASSET_DIR / "test_monkey.png" - prompt = "What animal is in this image? Reply in one complete short sentence." - fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" - hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") - - hf = hf_gemma_e002_inputs(hf_snapshot, image, prompt) - cactus_tokens = e002_cactus_tokens(fresh_gemma, image, prompt) - write_json(out_dir / "hf_native_inputs.json", hf) - write_json(out_dir / "cactus_token_probe.json", cactus_tokens) - - token_mismatch = first_mismatch(hf["input_ids"], cactus_tokens["input_ids"]) - token_compare = { - "gate": "prompt_and_token_spans", - "status": "pass" if token_mismatch is None else "fail", - "input_ids_match": token_mismatch is None, - "first_mismatch_index": token_mismatch, - "hf": {key: hf[key] for key in ("token_count", "assistant_generation_start", "image_token_id", "image_token_start", "image_token_count", "first_20", "last_20", "prompt_prefix")}, - "cactus": {key: cactus_tokens[key] for key in ("token_count", "assistant_generation_start", "image_token_id", "image_token_start", "image_token_count", "first_20", "last_20", "prompt_prefix")}, - } - if token_mismatch is not None: - lo = max(0, token_mismatch - 8) - hi = token_mismatch + 9 - token_compare["mismatch_window"] = { - "hf": hf["input_ids"][lo:hi], - "cactus": cactus_tokens["input_ids"][lo:hi], - "start": lo, - } - write_json(out_dir / "token_spans.json", token_compare) - if token_mismatch is not None: - return - - cactus_preprocess = e002_cactus_preprocess(fresh_gemma, image) - write_json(out_dir / "cactus_preprocess.json", cactus_preprocess) - hf_values = hf["pixel_values"] - cactus_values = cactus_preprocess["pixel_values"] - hf_positions = hf["pixel_position_ids"] - cactus_positions = cactus_preprocess["pixel_position_ids"] - if len(hf_values) != len(cactus_values): - max_abs_diff = None - mean_abs_diff = None - first_pixel_mismatch = min(len(hf_values), len(cactus_values)) - else: - diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, cactus_values)] - max_abs_diff = max(diffs) if diffs else 0.0 - mean_abs_diff = sum(diffs) / len(diffs) if diffs else 0.0 - first_pixel_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-5), None) - position_mismatch = first_mismatch(hf_positions, cactus_positions) - preprocess_pass = ( - hf["pixel_values_shape"] == cactus_preprocess["pixel_values_shape"] - and hf["pixel_position_ids_shape"] == cactus_preprocess["pixel_position_ids_shape"] - and position_mismatch is None - and max_abs_diff is not None - and max_abs_diff <= 1e-5 - ) - compare = { - "gate": "image_preprocessing_tensors", - "status": "pass" if preprocess_pass else "fail", - "pixel_values_shape_match": hf["pixel_values_shape"] == cactus_preprocess["pixel_values_shape"], - "pixel_position_ids_shape_match": hf["pixel_position_ids_shape"] == cactus_preprocess["pixel_position_ids_shape"], - "pixel_position_ids_match": position_mismatch is None, - "first_position_mismatch_index": position_mismatch, - "pixel_values_max_abs_diff": max_abs_diff, - "pixel_values_mean_abs_diff": mean_abs_diff, - "first_pixel_value_mismatch_index_at_1e-5": first_pixel_mismatch, - "hf_summary": {key: hf[key] for key in ("pixel_values_shape", "pixel_position_ids_shape", "pixel_values_min", "pixel_values_max", "pixel_values_mean", "pixel_values_sha256", "pixel_position_ids_sha256", "valid_position_count", "first_pixel_values", "first_position_rows_flat")}, - "cactus_summary": {key: cactus_preprocess[key] for key in ("pixel_values_shape", "pixel_position_ids_shape", "pixel_values_min", "pixel_values_max", "pixel_values_mean", "pixel_values_hash", "pixel_position_ids_hash", "valid_position_count", "first_pixel_values", "first_position_rows_flat")}, - } - if first_pixel_mismatch is not None: - lo = max(0, first_pixel_mismatch - 8) - hi = first_pixel_mismatch + 9 - compare["pixel_mismatch_window"] = { - "start": lo, - "hf": hf_values[lo:hi], - "cactus": cactus_values[lo:hi], - } - if position_mismatch is not None: - lo = max(0, position_mismatch - 8) - hi = position_mismatch + 9 - compare["position_mismatch_window"] = { - "start": lo, - "hf": hf_positions[lo:hi], - "cactus": cactus_positions[lo:hi], - } - write_json(out_dir / "image_preprocess_compare.json", compare) - - -def run_e003_preprocess_isolation(out_dir: Path, weights_root: Path) -> None: - image = ASSET_DIR / "test_monkey.png" - prompt = "What animal is in this image? Reply in one complete short sentence." - fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" - hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") - - hf = hf_gemma_e002_inputs(hf_snapshot, image, prompt) - variants = e003_cactus_preprocess_variants(fresh_gemma, image) - write_json(out_dir / "hf_native_inputs.json", hf) - write_json(out_dir / "cactus_resize_variants.json", variants) - - hf_values = hf["pixel_values"] - comparisons: dict[str, Any] = {} - best_name: str | None = None - best_max: float | None = None - for name, variant in variants["variants"].items(): - values = variant["pixel_values"] - if len(values) != len(hf_values): - comparison = { - "status": "fail", - "length_match": False, - "hf_length": len(hf_values), - "variant_length": len(values), - } - else: - diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, values)] - max_abs = max(diffs) if diffs else 0.0 - mean_abs = sum(diffs) / len(diffs) if diffs else 0.0 - first_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-5), None) - comparison = { - "status": "pass" if max_abs <= 1e-5 else "fail", - "length_match": True, - "max_abs_diff": max_abs, - "mean_abs_diff": mean_abs, - "first_mismatch_index_at_1e-5": first_mismatch, - "variant_summary": { - key: variant[key] - for key in ( - "resized_min", - "resized_max", - "first_resized_values", - "pixel_values_min", - "pixel_values_max", - "pixel_values_mean", - "first_pixel_values", - ) - }, - } - if first_mismatch is not None: - lo = max(0, first_mismatch - 8) - hi = first_mismatch + 9 - comparison["mismatch_window"] = { - "start": lo, - "hf": hf_values[lo:hi], - "variant": values[lo:hi], - } - if best_max is None or max_abs < best_max: - best_name = name - best_max = max_abs - comparisons[name] = comparison - - write_json( - out_dir / "preprocess_operation_compare.json", - { - "gate": "e003_preprocess_operation_isolation", - "source_width": variants["source_width"], - "source_height": variants["source_height"], - "target_width": variants["target_width"], - "target_height": variants["target_height"], - "patch_size": variants["patch_size"], - "pooling_kernel_size": variants["pooling_kernel_size"], - "max_patches": variants["max_patches"], - "patch_dim": variants["patch_dim"], - "rescale_factor": variants["rescale_factor"], - "hf_summary": { - key: hf[key] - for key in ( - "pixel_values_shape", - "pixel_values_min", - "pixel_values_max", - "pixel_values_mean", - "first_pixel_values", - ) - }, - "best_variant": best_name, - "best_variant_max_abs_diff": best_max, - "comparisons": comparisons, - }, - ) - - -def compare_float_vectors(hf_values: list[float], cactus_values: list[float]) -> dict[str, Any]: - if len(hf_values) != len(cactus_values): - return { - "status": "fail", - "length_match": False, - "hf_length": len(hf_values), - "cactus_length": len(cactus_values), - } - cactus_nonfinite = sum(1 for value in cactus_values if value is None) - hf_nonfinite = sum(1 for value in hf_values if value is None) - if cactus_nonfinite or hf_nonfinite: - return { - "status": "fail", - "length_match": True, - "hf_nonfinite_count": hf_nonfinite, - "cactus_nonfinite_count": cactus_nonfinite, - "first_nonfinite_index": next( - (i for i, (a, b) in enumerate(zip(hf_values, cactus_values)) if a is None or b is None), - None, - ), - } - diffs = [abs(float(a) - float(b)) for a, b in zip(hf_values, cactus_values)] - dot = sum(float(a) * float(b) for a, b in zip(hf_values, cactus_values)) - hf_norm = sum(float(a) * float(a) for a in hf_values) ** 0.5 - cactus_norm = sum(float(b) * float(b) for b in cactus_values) ** 0.5 - max_abs = max(diffs) if diffs else 0.0 - mean_abs = sum(diffs) / len(diffs) if diffs else 0.0 - first_mismatch = next((i for i, diff in enumerate(diffs) if diff > 1e-3), None) - result: dict[str, Any] = { - "status": "pass" if max_abs <= 1e-2 and mean_abs <= 1e-3 else "fail", - "length_match": True, - "max_abs_diff": max_abs, - "mean_abs_diff": mean_abs, - "cosine_similarity": dot / (hf_norm * cactus_norm) if hf_norm and cactus_norm else 0.0, - "first_mismatch_index_at_1e-3": first_mismatch, - } - if first_mismatch is not None: - lo = max(0, first_mismatch - 8) - hi = first_mismatch + 9 - result["mismatch_window"] = { - "start": lo, - "hf": hf_values[lo:hi], - "cactus": cactus_values[lo:hi], - } - return result - - -def run_e002_vision_boundary(out_dir: Path, weights_root: Path) -> None: - image = ASSET_DIR / "test_monkey.png" - prompt = "What animal is in this image? Reply in one complete short sentence." - fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" - hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") - - hf = hf_gemma_vision_features(hf_snapshot, image, prompt) - cactus = e002_cactus_vision(fresh_gemma, image) - write_json(out_dir / "hf_vision_features.json", hf) - write_json(out_dir / "cactus_vision_features.json", cactus) - - comparisons: dict[str, Any] = {} - for cactus_name, cactus_feature in cactus["features"].items(): - comparisons[cactus_name] = {} - for hf_name, hf_feature in hf["features"].items(): - comparison = compare_float_vectors(hf_feature["values"], cactus_feature["values"]) - comparison["hf_shape"] = hf_feature["shape"] - comparison["cactus_shape"] = cactus_feature["shape"] - comparison["hf_summary"] = {key: hf_feature[key] for key in ("dtype", "count", "min", "max", "mean", "l2", "first_values")} - comparison["cactus_summary"] = {key: cactus_feature[key] for key in ("precision", "count", "finite_count", "nonfinite_count", "min", "max", "mean", "l2", "first_values")} - comparisons[cactus_name][hf_name] = comparison - - write_json( - out_dir / "vision_encoder_compare.json", - { - "gate": "vision_encoder_output", - "comparisons": comparisons, - }, - ) - - -def manifest_summary(path: Path) -> dict[str, Any]: - manifest = path / "components" / "manifest.json" - config = path / "config.txt" - result: dict[str, Any] = {"path": str(path), "manifest_exists": manifest.exists(), "config_exists": config.exists()} - if manifest.exists(): - data = json.loads(manifest.read_text()) - result.update( - { - "family": data.get("family"), - "task": data.get("task"), - "model_source": data.get("model_source"), - "component_order": data.get("component_order"), - "components": [ - { - "component": component.get("component"), - "logical_inputs": component.get("logical_inputs"), - "logical_outputs": component.get("logical_outputs"), - } - for component in data.get("components", []) - ], - } - ) - return result - - -def run_case(case: str, out_dir: Path, weights_root: Path) -> None: - image = ASSET_DIR / "test_monkey.png" - prompt = "What animal is in this image? Reply in one complete short sentence." - fresh_gemma = weights_root / "gemma-4-e2b-it-fresh-mm" - qwen = weights_root / "qwen3-vl-2b-instruct-reconvert" - parakeet = weights_root / "parakeet-tdt-0.6b-v3-transpiled" - hf_snapshot = Path("/Users/noahcylich/.cache/huggingface/hub/models--google--gemma-4-E2B-it/snapshots/905e84b50c4d2a365ebde34e685027578e6728db") - - if case == "inventory": - write_json(out_dir / "inventory.json", {"gemma_fresh": manifest_summary(fresh_gemma)}) - return - if case == "hf-gemma-image": - write_json(out_dir / "hf_gemma_image.json", hf_gemma_image(hf_snapshot, image, prompt)) - return - if case == "hf-gemma-native-image": - write_json(out_dir / "hf_gemma_native_image.json", hf_gemma_native_image(hf_snapshot, image, prompt)) - return - if case == "cactus-gemma-image": - write_json(out_dir / "cactus_gemma_image.json", cactus_complete(fresh_gemma, prompt, image=image)) - return - if case == "cactus-gemma-audio": - wav = ensure_question_wav() - audio_prompt = "Listen to the audio and answer briefly." - write_json(out_dir / "cactus_gemma_audio.json", cactus_complete(fresh_gemma, audio_prompt, audio=wav)) - return - if case == "cactus-gemma-mixed": - wav = ensure_question_wav() - mixed_prompt = "Answer the spoken question using the provided image. Reply in one short sentence." - write_json(out_dir / "cactus_gemma_mixed.json", cactus_complete(fresh_gemma, mixed_prompt, image=image, audio=wav)) - return - if case == "cactus-qwen-image": - write_json(out_dir / "cactus_qwen_image.json", cactus_complete(qwen, prompt, image=image, max_tokens=40)) - return - if case == "parakeet-audio": - wav = ensure_question_wav() - write_json(out_dir / "parakeet_audio.json", parakeet_transcribe(parakeet, wav)) - write_json(out_dir / "parakeet_audio_pcm.json", parakeet_transcribe(parakeet, wav, pcm=True)) - return - if case == "all-baseline": - for subcase in ("inventory", "hf-gemma-image", "hf-gemma-native-image", "cactus-gemma-image", "cactus-qwen-image", "parakeet-audio"): - run_case(subcase, out_dir, weights_root) - return - if case == "e002-boundary": - run_e002_boundary(out_dir, weights_root) - return - if case == "e003-preprocess-isolation": - run_e003_preprocess_isolation(out_dir, weights_root) - return - if case == "e002-vision-boundary": - run_e002_vision_boundary(out_dir, weights_root) - return - raise ValueError(f"unknown case: {case}") - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--case", required=True) - parser.add_argument("--experiment", required=True) - parser.add_argument("--weights-root", default=str(DEFAULT_WEIGHTS_ROOT)) - args = parser.parse_args() - - out_dir = INVESTIGATION_ROOT / "artifacts" / args.experiment - out_dir.mkdir(parents=True, exist_ok=True) - write_json( - out_dir / "run_metadata.json", - { - "case": args.case, - "experiment": args.experiment, - "repo_root": str(REPO_ROOT), - "weights_root": str(Path(args.weights_root).resolve()), - "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"), - }, - ) - run_case(args.case, out_dir, Path(args.weights_root).resolve()) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 0a3376eac2af1fc729b90da768885f2fad748b56 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 21 May 2026 18:26:48 -0700 Subject: [PATCH 7/8] Optimize chunked prefill first-token sampling Add a first-token path that can sample directly after cache priming, track chunked prefill cache-copy and tail metrics, and allow sufficiently large tails to run as padded chunks. When padded chunks are used for first-token sampling, rewind the copied decode cache to the final real prompt token and run that final token through the step decoder so logits are taken from real context instead of pad context. Keep LFM2-VL on scalar tails because its recurrent cached state is not safe to rewind after executing padded tokens. Signed-off-by: Noah Cylich --- cactus-engine/src/complete.cpp | 17 +++- cactus-engine/src/engine.h | 23 ++++- cactus-engine/src/model.cpp | 163 ++++++++++++++++++++++++++------- 3 files changed, 166 insertions(+), 37 deletions(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 555a40276..11adc88ca 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -711,9 +711,20 @@ int cactus_complete( } } - auto prefill_result = do_prefill(handle, prompt, prompt.tokens); - prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); - next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); + bool first_token_from_prefill = false; + if (!has_images && !has_audio && handle->processed_tokens.empty()) { + reset_cache(handle); + first_token_from_prefill = handle->model->prefill_and_sample_first_token(prompt.tokens, next_token); + if (first_token_from_prefill) { + prompt_tokens = prompt.tokens.size(); + first_token_entropy = 0.0f; + } + } + if (!first_token_from_prefill) { + auto prefill_result = do_prefill(handle, prompt, prompt.tokens); + prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); + next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); + } handle->processed_tokens = prompt.tokens; handle->processed_images = prompt.images; diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 02043a6b9..5732f44b1 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "cactus_graph.h" @@ -580,6 +581,7 @@ class Model { uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f); + bool prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token); void prefill(const std::vector& tokens, size_t chunk_size = 128, const std::string& profile_file = "", bool prepare_decode = true); @@ -639,6 +641,9 @@ class Model { bool load_npu_prefill(const std::string& model_path); bool has_npu_prefill() const { return false; } size_t get_prefill_chunk_size() const { return 128; } + double last_prefill_cache_copy_ms() const { return last_prefill_cache_copy_ms_; } + size_t last_prefill_padding_tokens() const { return last_prefill_padding_tokens_; } + size_t last_prefill_scalar_tail_tokens() const { return last_prefill_scalar_tail_tokens_; } void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} @@ -679,6 +684,14 @@ class Model { void copy_cache_state(const Component& src, Component& dst); + struct ChunkedPrefillResult { + size_t logical_tokens = 0; + size_t executed_tokens = 0; + size_t padding_tokens = 0; + size_t scalar_tail_tokens = 0; + size_t last_logit_row = 0; + }; + bool load_manifest(); bool setup_tokenizer(); bool load_components(const std::unordered_set& required_components); @@ -694,13 +707,14 @@ class Model { void copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index); void copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset); bool cache_states_compatible(const Component& source, const Component& target) const; - void copy_cache_states(const Component& source, Component& target); + void copy_cache_states(const Component& source, Component& target, size_t logical_current = std::numeric_limits::max()); void reset_component_cache_states(Component& comp); size_t component_chunk_tokens(const Component& comp, const std::string& input_name) const; size_t component_output_tokens(const Component& comp, const std::string& output_name) const; - size_t run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, - bool prepare_decode); + ChunkedPrefillResult run_chunked_prefill(const std::vector& tokens, size_t start_position, + size_t chunk_size, bool prepare_decode); void run_full_context_text(); + uint32_t argmax_component_logits(Component& comp, size_t logit_row = std::numeric_limits::max()); void write_int_input(Component& comp, const std::string& name, int64_t value); void write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value); void write_bytes_input(Component& comp, const std::string& name, const void* data, size_t byte_size); @@ -747,6 +761,9 @@ class Model { size_t cache_total_seq_len_ = 0; size_t cache_max_seq_len_ = 4096; size_t last_logit_position_ = 0; + double last_prefill_cache_copy_ms_ = 0.0; + size_t last_prefill_padding_tokens_ = 0; + size_t last_prefill_scalar_tail_tokens_ = 0; std::vector context_tokens_; static constexpr size_t MAX_TOKEN_HISTORY = 128; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 8d6a02d1c..41b94cb5a 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -640,7 +641,7 @@ bool Model::cache_states_compatible(const Component& source, const Component& ta return true; } -void Model::copy_cache_states(const Component& source, Component& target) { +void Model::copy_cache_states(const Component& source, Component& target, size_t logical_current) { if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) { throw std::runtime_error("prefill and step cache states are not compatible"); } @@ -662,7 +663,7 @@ void Model::copy_cache_states(const Component& source, Component& target) { } void* src_ptr = source.graph->get_output(static_cast(src_node)); void* dst_ptr = target.graph->get_output(static_cast(dst_node)); - if (src_desc.byte_size == dst_desc.byte_size) { + if (src_desc.byte_size == dst_desc.byte_size && logical_current == std::numeric_limits::max()) { std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); continue; } @@ -670,6 +671,7 @@ void Model::copy_cache_states(const Component& source, Component& target) { auto* src_meta = static_cast(src_ptr); auto* dst_meta = static_cast(dst_ptr); const size_t src_current = static_cast(src_meta[0]); + const size_t effective_current = std::min(src_current, logical_current); const size_t src_max = static_cast(src_meta[1]); const size_t kv_heads = static_cast(src_meta[2]); const size_t head_dim = static_cast(src_meta[3]); @@ -685,7 +687,7 @@ void Model::copy_cache_states(const Component& source, Component& target) { if (dst_max == 0) { throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); } - const size_t dst_current = std::min(src_current, dst_max); + const size_t dst_current = std::min(effective_current, dst_max); dst_meta[0] = dst_current; dst_meta[1] = dst_max; dst_meta[2] = kv_heads; @@ -710,13 +712,13 @@ void Model::copy_cache_states(const Component& source, Component& target) { src_scales + src_row * scale_stride, rows * scale_stride * sizeof(float)); }; - if (src_current <= dst_max) { - copy_rows(0, 0, src_current); + if (effective_current <= dst_max) { + copy_rows(0, 0, effective_current); } else { const size_t copied_sink = std::min(sink, dst_max); const size_t tail_rows = dst_max - copied_sink; copy_rows(0, 0, copied_sink); - if (tail_rows > 0) copy_rows(copied_sink, src_current - tail_rows, tail_rows); + if (tail_rows > 0) copy_rows(copied_sink, effective_current - tail_rows, tail_rows); } continue; } @@ -730,6 +732,7 @@ void Model::copy_cache_states(const Component& source, Component& target) { auto* src_meta = static_cast(src_ptr); auto* dst_meta = static_cast(dst_ptr); const size_t src_current = static_cast(src_meta[0]); + const size_t effective_current = std::min(src_current, logical_current); const size_t src_max = static_cast(src_meta[1]); const size_t kv_heads = static_cast(src_meta[2]); const size_t head_dim = static_cast(src_meta[3]); @@ -742,7 +745,7 @@ void Model::copy_cache_states(const Component& source, Component& target) { if (dst_max == 0) { throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); } - const size_t dst_current = std::min(src_current, dst_max); + const size_t dst_current = std::min(effective_current, dst_max); dst_meta[0] = dst_current; dst_meta[1] = dst_max; dst_meta[2] = kv_heads; @@ -751,8 +754,8 @@ void Model::copy_cache_states(const Component& source, Component& target) { std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); const auto* src_rows = static_cast(src_ptr) + 64; auto* dst_rows = static_cast(dst_ptr) + 64; - if (src_current <= dst_max) { - std::memcpy(dst_rows, src_rows, src_current * row_bytes); + if (effective_current <= dst_max) { + std::memcpy(dst_rows, src_rows, effective_current * row_bytes); } else { const size_t copied_sink = std::min(sink, dst_max); const size_t tail_rows = dst_max - copied_sink; @@ -762,7 +765,7 @@ void Model::copy_cache_states(const Component& source, Component& target) { if (tail_rows > 0) { std::memcpy( dst_rows + copied_sink * row_bytes, - src_rows + (src_current - tail_rows) * row_bytes, + src_rows + (effective_current - tail_rows) * row_bytes, tail_rows * row_bytes); } } @@ -802,17 +805,34 @@ size_t Model::component_output_tokens(const Component& comp, const std::string& return 0; } -size_t Model::run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, bool prepare_decode) { - if (decode_route_ != DecodeRoute::CACHED_STEP || !encoder_ || !decoder_ || !decoder_prefill_) return 0; - if (start_position != 0) return 0; - if (!load_component_graph(*decoder_prefill_)) return 0; - if (prefill_encoder_ && !load_component_graph(*prefill_encoder_)) return 0; - if (!cache_states_compatible(*decoder_prefill_, *decoder_)) return 0; +Model::ChunkedPrefillResult Model::run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, bool prepare_decode) { + ChunkedPrefillResult result; + last_prefill_cache_copy_ms_ = 0.0; + last_prefill_padding_tokens_ = 0; + last_prefill_scalar_tail_tokens_ = 0; + if (decode_route_ != DecodeRoute::CACHED_STEP || !encoder_ || !decoder_ || !decoder_prefill_) return result; + if (start_position != 0) return result; + if (!load_component_graph(*decoder_prefill_)) return result; + if (prefill_encoder_ && !load_component_graph(*prefill_encoder_)) return result; + if (!cache_states_compatible(*decoder_prefill_, *decoder_)) return result; size_t component_tokens = component_chunk_tokens(*decoder_prefill_, "inputs_embeds"); - if (component_tokens <= 1) return 0; + if (component_tokens <= 1) return result; size_t effective_chunk = chunk_size > 0 ? std::min(chunk_size, component_tokens) : component_tokens; if (effective_chunk != component_tokens) effective_chunk = component_tokens; - if (tokens.size() < effective_chunk) return 0; + const size_t whole_chunks_end = (tokens.size() / effective_chunk) * effective_chunk; + const size_t tail_tokens = tokens.size() - whole_chunks_end; + const size_t padding_cutoff = std::max(1, effective_chunk / 16); + const bool pad_tail = family_ != "lfm2_vl" && tail_tokens >= padding_cutoff; + const size_t executable_tokens = whole_chunks_end + (pad_tail ? effective_chunk : 0); + if (executable_tokens == 0) { + result.scalar_tail_tokens = tail_tokens; + last_prefill_scalar_tail_tokens_ = tail_tokens; + return result; + } + result.padding_tokens = executable_tokens > tokens.size() ? executable_tokens - tokens.size() : 0; + result.scalar_tail_tokens = tokens.size() - std::min(tokens.size(), executable_tokens); + last_prefill_padding_tokens_ = result.padding_tokens; + last_prefill_scalar_tail_tokens_ = result.scalar_tail_tokens; size_t encoder_chunk = 0; if (prefill_encoder_ && input_index(*prefill_encoder_, "input_ids") >= 0 && input_index(*prefill_encoder_, "position_ids") >= 0) { @@ -823,7 +843,7 @@ size_t Model::run_chunked_prefill(const std::vector& tokens, size_t st } size_t processed = 0; - while (processed + effective_chunk <= tokens.size()) { + while (processed + effective_chunk <= executable_tokens) { for (size_t i = 0; i < decoder_prefill_->input_buffers.size(); ++i) { std::fill(decoder_prefill_->input_buffers[i].begin(), decoder_prefill_->input_buffers[i].end(), 0); } @@ -833,7 +853,9 @@ size_t Model::run_chunked_prefill(const std::vector& tokens, size_t st std::fill(prefill_encoder_->input_buffers[i].begin(), prefill_encoder_->input_buffers[i].end(), 0); } for (size_t i = 0; i < encoder_chunk; ++i) { - write_int_input_at(*prefill_encoder_, "input_ids", i, static_cast(tokens[processed + chunk_offset + i])); + size_t index = processed + chunk_offset + i; + uint32_t token = index < tokens.size() ? tokens[index] : static_cast(config_.pad_token_id); + write_int_input_at(*prefill_encoder_, "input_ids", i, static_cast(token)); write_int_input_at(*prefill_encoder_, "position_ids", i, static_cast(start_position + processed + chunk_offset + i)); } prefill_encoder_->graph->execute(); @@ -841,20 +863,30 @@ size_t Model::run_chunked_prefill(const std::vector& tokens, size_t st } } else { for (size_t i = 0; i < effective_chunk; ++i) { - run_encoder_step(tokens[processed + i], start_position + processed + i); + size_t index = processed + i; + uint32_t token = index < tokens.size() ? tokens[index] : static_cast(config_.pad_token_id); + run_encoder_step(token, start_position + processed + i); copy_component_outputs_to_chunk_inputs(*encoder_, *decoder_prefill_, i); } } decoder_prefill_->graph->execute(); processed += effective_chunk; } + result.executed_tokens = processed; + result.logical_tokens = std::min(tokens.size(), processed); + if (result.logical_tokens > 0) { + result.last_logit_row = (result.logical_tokens - 1) % effective_chunk; + } if (processed > 0 && prepare_decode) { for (size_t i = 0; i < decoder_->input_buffers.size(); ++i) { std::fill(decoder_->input_buffers[i].begin(), decoder_->input_buffers[i].end(), 0); } - copy_cache_states(*decoder_prefill_, *decoder_); + auto copy_start = std::chrono::high_resolution_clock::now(); + copy_cache_states(*decoder_prefill_, *decoder_, start_position + result.logical_tokens); + auto copy_end = std::chrono::high_resolution_clock::now(); + last_prefill_cache_copy_ms_ = std::chrono::duration_cast(copy_end - copy_start).count() / 1000.0; } - return processed; + return result; } void Model::run_full_context_text() { @@ -1081,13 +1113,18 @@ void Model::run_audio_encoder(const std::vector& audio_features) { } } -uint32_t Model::argmax_last_logits() { - size_t out_node = static_cast(decoder_->output_node_ids.empty() ? 0 : decoder_->output_node_ids[0]); - const auto& desc = decoder_->graph->get_output_buffer(out_node); - void* ptr = decoder_->graph->get_output(out_node); +uint32_t Model::argmax_component_logits(Component& comp, size_t logit_row) { + size_t out_node = static_cast(comp.output_node_ids.empty() ? 0 : comp.output_node_ids[0]); + const auto& desc = comp.graph->get_output_buffer(out_node); + void* ptr = comp.graph->get_output(out_node); size_t vocab = desc.shape.empty() ? 0 : desc.shape.back(); size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; - size_t row = decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT ? std::min(last_logit_position_, seq > 0 ? seq - 1 : 0) : (seq > 0 ? seq - 1 : 0); + size_t row = seq > 0 ? seq - 1 : 0; + if (logit_row != std::numeric_limits::max()) { + row = std::min(logit_row, seq > 0 ? seq - 1 : 0); + } else if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + row = std::min(last_logit_position_, seq > 0 ? seq - 1 : 0); + } size_t row_off = row * vocab; uint32_t best = 0; float best_v = -std::numeric_limits::infinity(); @@ -1107,19 +1144,83 @@ uint32_t Model::argmax_last_logits() { return best; } +uint32_t Model::argmax_last_logits() { + return argmax_component_logits(*decoder_); +} + +bool Model::prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token) { + last_prefill_cache_copy_ms_ = 0.0; + last_prefill_padding_tokens_ = 0; + last_prefill_scalar_tail_tokens_ = 0; + if (tokens.empty() || !decoder_ || cache_total_seq_len_ != 0) { + return false; + } + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + out_token = argmax_last_logits(); + record_sampled_token(out_token); + return true; + } + if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { + for (size_t i = 0; i < tokens.size(); ++i) { + run_step(tokens[i], i, i + 1 == tokens.size()); + } + cache_total_seq_len_ = tokens.size(); + out_token = argmax_last_logits(); + record_sampled_token(out_token); + return true; + } + if (!encoder_) { + return false; + } + ChunkedPrefillResult chunked; + if (decoder_prefill_) { + chunked = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), true); + if (chunked.logical_tokens == tokens.size() && chunked.padding_tokens > 0 && tokens.size() > 0) { + copy_cache_states(*decoder_prefill_, *decoder_, tokens.size() - 1); + cache_total_seq_len_ = tokens.size() - 1; + run_step(tokens.back(), cache_total_seq_len_, true); + ++cache_total_seq_len_; + out_token = argmax_last_logits(); + record_sampled_token(out_token); + last_prefill_scalar_tail_tokens_ = 1; + return true; + } + cache_total_seq_len_ += chunked.logical_tokens; + } + for (size_t i = chunked.logical_tokens; i < tokens.size(); ++i) { + run_step(tokens[i], cache_total_seq_len_, i + 1 == tokens.size()); + ++cache_total_seq_len_; + } + last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; + if (chunked.logical_tokens == tokens.size() && chunked.logical_tokens > 0 && decoder_prefill_) { + out_token = argmax_component_logits(*decoder_prefill_, chunked.last_logit_row); + } else { + out_token = argmax_last_logits(); + } + record_sampled_token(out_token); + return true; +} + void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& /*profile_file*/, bool prepare_decode) { + last_prefill_cache_copy_ms_ = 0.0; + last_prefill_padding_tokens_ = 0; + last_prefill_scalar_tail_tokens_ = 0; if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); if (!context_tokens_.empty()) run_full_context_text(); cache_total_seq_len_ = context_tokens_.size(); return; } - size_t processed = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), prepare_decode); - cache_total_seq_len_ += processed; - for (size_t i = processed; i < tokens.size(); ++i) { + ChunkedPrefillResult chunked = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), prepare_decode); + cache_total_seq_len_ += chunked.logical_tokens; + for (size_t i = chunked.logical_tokens; i < tokens.size(); ++i) { run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false); ++cache_total_seq_len_; } + last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; } void Model::prefill_with_images(const std::vector& tokens, From 953bfaf965ba7735c085204239c645665b9856f7 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 21 May 2026 19:38:50 -0700 Subject: [PATCH 8/8] Wire chunked bundle CLI benchmark flags Signed-off-by: Noah Cylich --- cactus-engine/cactus_engine.h | 9 +++ cactus-engine/src/complete.cpp | 133 +++++++++++++++++++++++++++++++++ cactus-engine/tests/chat.cpp | 76 ++++++++++++++++++- python/cactus/cli/run.py | 9 ++- 4 files changed, 224 insertions(+), 3 deletions(-) diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index ce65fbf08..cb575ec21 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -74,6 +74,15 @@ CACTUS_FFI_EXPORT int cactus_score_window( size_t buffer_size ); +CACTUS_FFI_EXPORT int cactus_benchmark_tokens( + cactus_model_t model, + const uint32_t* prompt_tokens, + size_t prompt_token_len, + size_t decode_token_len, + char* response_buffer, + size_t buffer_size +); + CACTUS_FFI_EXPORT int cactus_transcribe( cactus_model_t model, const char* audio_file_path, // NULL if using pcm_buffer diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 11adc88ca..cae05d44f 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -4,10 +4,13 @@ #include "telemetry.h" #include "cactus_kernels.h" #include "wav.h" +#include #include +#include #include #include #include +#include #include #include @@ -21,6 +24,20 @@ namespace { std::vector> extract_schema_property_types(const std::string& schema); std::vector extract_schema_required(const std::string& schema); +void maybe_pin_benchmark_main_to_max_perf_core() { +#if defined(__ANDROID__) + const char* enabled = std::getenv("CACTUS_BENCH_PIN_MAIN_MAX_PERF"); + if (!enabled || std::string(enabled) != "1") return; + + (void)CactusThreading::get_thread_pool(); + const auto& cores = CactusThreading::CoreTopology::get().performance_cores; + if (cores.empty()) return; + int core = *std::max_element(cores.begin(), cores.end()); + bool ok = CactusThreading::pin_current_thread_to_cores({core}); + std::cerr << "CACTUS_BENCH_PIN_MAIN_MAX_PERF cpu=" << core << " ok=" << (ok ? 1 : 0) << "\n"; +#endif +} + std::string extract_last_user_query(const std::vector& messages) { for (auto it = messages.rbegin(); it != messages.rend(); ++it) { if (it->role == "user") { @@ -1130,4 +1147,120 @@ int cactus_score_window( } } +int cactus_benchmark_tokens( + cactus_model_t model, + const uint32_t* prompt_tokens, + size_t prompt_token_len, + size_t decode_token_len, + char* response_buffer, + size_t buffer_size +) { + if (!model || !prompt_tokens || prompt_token_len == 0 || !response_buffer || buffer_size == 0) { + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + + try { + auto* handle = static_cast(model); + std::vector prompt(prompt_tokens, prompt_tokens + prompt_token_len); + + auto start_time = std::chrono::high_resolution_clock::now(); + double peak_ram_usage_mb = get_ram_usage_mb(); + auto sample_peak_ram = [&]() { + peak_ram_usage_mb = std::max(peak_ram_usage_mb, get_ram_usage_mb()); + }; + handle->model->reset_cache(); + maybe_pin_benchmark_main_to_max_perf_core(); + auto prefill_start_time = std::chrono::high_resolution_clock::now(); + size_t cache_prime_tokens = 0; + uint32_t next_token = 0; + bool first_token_from_prefill = false; + if (decode_token_len == 0) { + handle->model->prefill(prompt, handle->model->get_prefill_chunk_size(), "", false); + cache_prime_tokens = prompt.size(); + } else { + first_token_from_prefill = handle->model->prefill_and_sample_first_token(prompt, next_token); + if (first_token_from_prefill) { + cache_prime_tokens = prompt.size(); + } else if (prompt.size() > 1) { + handle->model->prefill( + std::vector(prompt.begin(), prompt.end() - 1), + handle->model->get_prefill_chunk_size() + ); + cache_prime_tokens = prompt.size() - 1; + } + } + auto prefill_end_time = std::chrono::high_resolution_clock::now(); + sample_peak_ram(); + auto first_decode_start_time = std::chrono::high_resolution_clock::now(); + if (decode_token_len > 0 && !first_token_from_prefill) { + next_token = handle->model->decode({prompt.back()}, 0.0f, 1.0f, 1); + } + sample_peak_ram(); + auto first_token_time = std::chrono::high_resolution_clock::now(); + + size_t generated = decode_token_len > 0 ? 1 : 0; + while (generated < decode_token_len) { + next_token = handle->model->decode({next_token}, 0.0f, 1.0f, 1); + sample_peak_ram(); + ++generated; + } + + auto end_time = std::chrono::high_resolution_clock::now(); + double ttft_ms = std::chrono::duration_cast(first_token_time - start_time).count() / 1000.0; + double total_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + double cache_prime_ms = std::chrono::duration_cast(prefill_end_time - prefill_start_time).count() / 1000.0; + double first_decode_ms = std::chrono::duration_cast(first_token_time - first_decode_start_time).count() / 1000.0; + double cache_state_copy_ms = handle->model->last_prefill_cache_copy_ms(); + double cache_prime_compute_ms = std::max(0.0, cache_prime_ms - cache_state_copy_ms); + double decode_ms = std::max(0.0, total_ms - ttft_ms); + double prefill_prepare_tps = (cache_prime_tokens > 0 && cache_prime_ms > 0.0) + ? (static_cast(cache_prime_tokens) * 1000.0) / cache_prime_ms + : 0.0; + double prefill_tps = (cache_prime_tokens > 0 && cache_prime_compute_ms > 0.0) + ? (static_cast(cache_prime_tokens) * 1000.0) / cache_prime_compute_ms + : 0.0; + double ttft_prompt_tps = ttft_ms > 0.0 ? (static_cast(prompt_token_len) * 1000.0) / ttft_ms : 0.0; + double decode_tps = (generated > 1 && decode_ms > 0.0) ? (static_cast(generated - 1) * 1000.0) / decode_ms : 0.0; + + std::ostringstream oss; + oss << "{" + << "\"success\":true," + << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << ttft_ms << "," + << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_ms << "," + << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << "," + << "\"prefill_compute_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << "," + << "\"prefill_prepare_tps\":" << std::fixed << std::setprecision(2) << prefill_prepare_tps << "," + << "\"ttft_prompt_tps\":" << std::fixed << std::setprecision(2) << ttft_prompt_tps << "," + << "\"cache_prime_ms\":" << std::fixed << std::setprecision(2) << cache_prime_ms << "," + << "\"cache_prime_compute_ms\":" << std::fixed << std::setprecision(2) << cache_prime_compute_ms << "," + << "\"cache_state_copy_ms\":" << std::fixed << std::setprecision(2) << cache_state_copy_ms << "," + << "\"cache_prime_tokens\":" << cache_prime_tokens << "," + << "\"first_decode_ms\":" << std::fixed << std::setprecision(2) << first_decode_ms << "," + << "\"first_token_from_prefill\":" << (first_token_from_prefill ? "true" : "false") << "," + << "\"prefill_padding_tokens\":" << handle->model->last_prefill_padding_tokens() << "," + << "\"prefill_scalar_tail_tokens\":" << handle->model->last_prefill_scalar_tail_tokens() << "," + << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << "," + << "\"prompt_tokens\":" << prompt_token_len << "," + << "\"completion_tokens\":" << generated << "," + << "\"peak_ram_usage_mb\":" << std::fixed << std::setprecision(2) << peak_ram_usage_mb << "," + << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() + << "}"; + + std::string result = oss.str(); + if (result.size() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + std::strcpy(response_buffer, result.c_str()); + return static_cast(result.size()); + } catch (const std::exception& e) { + handle_error_response(e.what(), response_buffer, buffer_size); + return -1; + } catch (...) { + handle_error_response("Unknown error during token benchmark", response_buffer, buffer_size); + return -1; + } +} + } diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index 9a5eeb1b1..594c75de2 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -283,6 +283,32 @@ bool file_exists(const std::string& path) { return f.good(); } +std::vector parse_token_ids(const std::string& text) { + std::vector tokens; + std::stringstream ss(text); + std::string part; + while (std::getline(ss, part, ',')) { + size_t start = part.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) continue; + size_t end = part.find_last_not_of(" \t\r\n"); + std::string value = part.substr(start, end - start + 1); + char* parsed_end = nullptr; + unsigned long parsed = std::strtoul(value.c_str(), &parsed_end, 10); + if (!parsed_end || *parsed_end != '\0') { + throw std::runtime_error("Invalid token id: " + value); + } + tokens.push_back(static_cast(parsed)); + } + return tokens; +} + +bool write_text_file(const std::string& path, const std::string& text) { + std::ofstream out(path); + if (!out) return false; + out << text; + return out.good(); +} + std::string json_string_value(const std::string& json, const std::string& key) { std::string needle = "\"" + key + "\":\""; size_t start = json.find(needle); @@ -338,7 +364,8 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 << " [--system ] [--image ] [--audio ]" - << " [--prompt ] [--thinking]\n"; + << " [--prompt ] [--input-ids ] [--max-new-tokens ]" + << " [--result-json ] [--thinking]\n"; } } // namespace @@ -354,6 +381,9 @@ int main(int argc, char** argv) { std::string current_image; std::string current_audio; std::string initial_prompt; + std::string input_ids; + std::string result_json; + int max_new_tokens = kMaxTokens; bool thinking = false; for (int i = 2; i < argc; ++i) { @@ -366,6 +396,12 @@ int main(int argc, char** argv) { current_audio = expand_tilde(argv[++i]); } else if (arg == "--prompt" && i + 1 < argc) { initial_prompt = argv[++i]; + } else if (arg == "--input-ids" && i + 1 < argc) { + input_ids = argv[++i]; + } else if (arg == "--max-new-tokens" && i + 1 < argc) { + max_new_tokens = std::max(0, std::atoi(argv[++i])); + } else if (arg == "--result-json" && i + 1 < argc) { + result_json = argv[++i]; } else if (arg == "--thinking") { thinking = true; } @@ -387,6 +423,39 @@ int main(int argc, char** argv) { return 1; } + if (!input_ids.empty()) { + std::vector tokens; + try { + tokens = parse_token_ids(input_ids); + } catch (const std::exception& e) { + std::cerr << e.what() << "\n"; + cactus_destroy(model); + return 1; + } + if (tokens.empty()) { + std::cerr << "--input-ids did not contain any token ids\n"; + cactus_destroy(model); + return 1; + } + std::vector response(kResponseBufferSize, 0); + int rc = cactus_benchmark_tokens( + model, + tokens.data(), + tokens.size(), + static_cast(max_new_tokens), + response.data(), + response.size()); + std::string response_json(response.data()); + if (!result_json.empty() && !write_text_file(result_json, response_json)) { + std::cerr << "Failed to write result JSON: " << result_json << "\n"; + cactus_destroy(model); + return 1; + } + std::cout << response_json << "\n"; + cactus_destroy(model); + return rc < 0 ? 1 : 0; + } + std::cout << "Model loaded.\n"; std::cout << "Commands: /image [prompt], /audio [prompt], "; #ifdef HAVE_SDL2 @@ -485,7 +554,7 @@ int main(int argc, char** argv) { history.push_back({"user", input}); std::string messages = build_messages(system_prompt, history, current_image, current_audio, attach_media); std::string options = "{\"temperature\":0.7,\"top_p\":0.95,\"top_k\":40,\"max_tokens\":" - + std::to_string(kMaxTokens) + + std::to_string(max_new_tokens) + ",\"enable_thinking_if_supported\":" + (thinking ? "true" : "false") + ",\"auto_handoff\":false,\"confidence_threshold\":0.0" + ",\"stop_sequences\":[\"<|im_end|>\",\"\"]}"; @@ -512,6 +581,9 @@ int main(int argc, char** argv) { current_pcm.size()); std::string response_json(response.data()); + if (!result_json.empty() && !write_text_file(result_json, response_json)) { + std::cerr << "Failed to write result JSON: " << result_json << "\n"; + } double ram_mb = json_number_value(response_json, "ram_usage_mb"); printer.print_stats(ram_mb); std::cout << "\n"; diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 52957ac46..c4919b79a 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -43,13 +43,20 @@ def cmd_run(args): if chat is None: return 1 + image_file = next((image for image in getattr(args, 'image_file', []) if image), None) + cmd = [str(chat), str(bundle_dir)] for flag, value in (("--system", getattr(args, 'system', None)), ("--prompt", getattr(args, 'prompt', None)), - ("--image", getattr(args, 'image', None)), + ("--image", getattr(args, 'image', None) or image_file), ("--audio", getattr(args, 'audio', None) or getattr(args, 'audio_file', None))): if value: cmd.extend([flag, str(Path(value).expanduser().resolve()) if flag in ("--image", "--audio") else str(value)]) + for flag, value in (("--input-ids", getattr(args, 'input_ids', None)), + ("--max-new-tokens", getattr(args, 'max_new_tokens', None)), + ("--result-json", getattr(args, 'result_json', None))): + if value is not None: + cmd.extend([flag, str(value)]) if getattr(args, 'thinking', False): cmd.append("--thinking")