diff --git a/.gitignore b/.gitignore index 99271876e..e18280031 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,8 @@ bindings/flutter/*.xcframework .DS_Store __pycache__ +__pycache__/ +*.pyc *.egg-info venv diff --git a/android/build.sh b/android/build.sh index 9d6f2d1e2..370a3fd83 100755 --- a/android/build.sh +++ b/android/build.sh @@ -12,9 +12,16 @@ CACTUS_CURL_ROOT=${CACTUS_CURL_ROOT:-"$PROJECT_ROOT/cactus-engine/libs/curl"} if [ -z "$ANDROID_NDK_HOME" ]; then if [ -n "$ANDROID_HOME" ]; then ANDROID_NDK_HOME=$(ls -d "$ANDROID_HOME/ndk/"* 2>/dev/null | sort -V | tail -1) - elif [ -d "$HOME/Library/Android/sdk" ]; then + fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "$HOME/Library/Android/sdk" ]; then ANDROID_NDK_HOME=$(ls -d "$HOME/Library/Android/sdk/ndk/"* 2>/dev/null | sort -V | tail -1) fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/Caskroom/android-ndk" ]; then + ANDROID_NDK_HOME=$(ls -d /opt/homebrew/Caskroom/android-ndk/*/AndroidNDK*.app/Contents/NDK 2>/dev/null | sort -V | tail -1) + fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/share/android-ndk" ]; then + ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" + fi fi if [ -z "$ANDROID_NDK_HOME" ] || [ ! -d "$ANDROID_NDK_HOME" ]; then diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index dbb9d5138..29d6e4346 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -74,6 +74,15 @@ CACTUS_FFI_EXPORT int cactus_score_window( size_t buffer_size ); +CACTUS_FFI_EXPORT int cactus_benchmark_tokens( + cactus_model_t model, + const uint32_t* prompt_tokens, + size_t prompt_token_len, + size_t decode_token_len, + char* response_buffer, + size_t buffer_size +); + CACTUS_FFI_EXPORT int cactus_transcribe( cactus_model_t model, const char* audio_file_path, // NULL if using pcm_buffer diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index d355eb586..cae05d44f 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -4,10 +4,13 @@ #include "telemetry.h" #include "cactus_kernels.h" #include "wav.h" +#include #include +#include #include #include #include +#include #include #include @@ -21,6 +24,20 @@ namespace { std::vector> extract_schema_property_types(const std::string& schema); std::vector extract_schema_required(const std::string& schema); +void maybe_pin_benchmark_main_to_max_perf_core() { +#if defined(__ANDROID__) + const char* enabled = std::getenv("CACTUS_BENCH_PIN_MAIN_MAX_PERF"); + if (!enabled || std::string(enabled) != "1") return; + + (void)CactusThreading::get_thread_pool(); + const auto& cores = CactusThreading::CoreTopology::get().performance_cores; + if (cores.empty()) return; + int core = *std::max_element(cores.begin(), cores.end()); + bool ok = CactusThreading::pin_current_thread_to_cores({core}); + std::cerr << "CACTUS_BENCH_PIN_MAIN_MAX_PERF cpu=" << core << " ok=" << (ok ? 1 : 0) << "\n"; +#endif +} + std::string extract_last_user_query(const std::vector& messages) { for (auto it = messages.rbegin(); it != messages.rend(); ++it) { if (it->role == "user") { @@ -325,7 +342,7 @@ struct PreparedPrompt { size_t context_token_count = 0; std::vector> images; - std::vector audio_features; + std::vector> audio_features; size_t audio_num_frames = 0; bool has_images() const { @@ -334,7 +351,8 @@ struct PreparedPrompt { } bool has_audio() const { - return !audio_features.empty(); + return std::any_of(audio_features.begin(), audio_features.end(), + [](const auto& mel) { return !mel.empty(); }); } }; @@ -457,38 +475,36 @@ PreparedPrompt prepare_prompt( } if (prompt.model_type == Config::ModelType::GEMMA4) { - std::vector audio_samples; - if (pcm_buffer != nullptr && pcm_buffer_size > 1) { - auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); - audio_samples = resample_to_16k_fp32(waveform_fp32, 16000); - } else if (!prompt.audio_paths.empty()) { - for (auto it = prompt.messages.rbegin(); it != prompt.messages.rend(); ++it) { - if (!it->audio.empty()) { - const std::string& audio_path = it->audio.back(); - AudioFP32 wav = load_wav(audio_path); - audio_samples = resample_to_16k_fp32(wav.samples, wav.sample_rate); - break; - } - } - } std::vector user_indices; for (size_t i = 0; i < prompt.messages.size(); i++) { if (prompt.messages[i].role == "user") user_indices.push_back(i); } auto& counts = handle->user_audio_counts; - for (size_t u = 0; u + 1 < user_indices.size(); u++) { - if (u < counts.size() && counts[u] > 0) { - prompt.messages[user_indices[u]].audio_soft_token_count = counts[u]; + if (counts.size() < user_indices.size()) counts.resize(user_indices.size(), 0); + + if (pcm_buffer != nullptr && pcm_buffer_size > 1 && !user_indices.empty()) { + auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); + auto samples_16k = resample_to_16k_fp32(waveform_fp32, 16000); + if (!samples_16k.empty()) { + auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(samples_16k, handle->model->get_config()); + prompt.audio_features.push_back(std::move(audio_prep.features)); + size_t u = user_indices.size() - 1; + prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; + counts[u] = audio_prep.num_soft_tokens; + } + } else if (!prompt.audio_paths.empty()) { + for (size_t u = 0; u < user_indices.size(); u++) { + const auto& msg = prompt.messages[user_indices[u]]; + if (msg.audio.empty()) continue; + const std::string& audio_path = msg.audio.back(); + AudioFP32 wav = load_wav(audio_path); + auto samples_16k = resample_to_16k_fp32(wav.samples, wav.sample_rate); + if (samples_16k.empty()) continue; + auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(samples_16k, handle->model->get_config()); + prompt.audio_features.push_back(std::move(audio_prep.features)); + prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; + counts[u] = audio_prep.num_soft_tokens; } - } - if (!audio_samples.empty() && !user_indices.empty()) { - auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(audio_samples, handle->model->get_config()); - prompt.audio_features = std::move(audio_prep.features); - prompt.audio_num_frames = audio_prep.num_frames; - size_t u = user_indices.size() - 1; - prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; - if (counts.size() <= u) counts.resize(u + 1, 0); - counts[u] = audio_prep.num_soft_tokens; } } @@ -542,6 +558,21 @@ PrefillResult do_prefill( if (tokens_to_process.size() > 1) { std::vector prefill_tokens(tokens_to_process.begin(), tokens_to_process.end() - 1); result.prefilled_count = prefill_tokens.size(); + + auto slice_delta_audio = [&]() -> std::vector> { + if (!result.was_prefix) return prompt.audio_features; + const size_t cached_msg_count = handle->processed_images.size(); + size_t cached_audio_count = 0; + for (size_t i = 0; i < cached_msg_count && i < prompt.messages.size(); ++i) { + const auto& msg = prompt.messages[i]; + if (msg.role == "user" && !msg.audio.empty()) cached_audio_count++; + } + cached_audio_count = std::min(cached_audio_count, prompt.audio_features.size()); + return std::vector>( + prompt.audio_features.begin() + cached_audio_count, + prompt.audio_features.end()); + }; + if (has_images && has_audio) { std::vector delta_image_paths; if (result.was_prefix) { @@ -556,7 +587,7 @@ PrefillResult do_prefill( } else { delta_image_paths = prompt.image_paths; } - handle->model->prefill_with_media(prefill_tokens, delta_image_paths, prompt.audio_features); + handle->model->prefill_with_media(prefill_tokens, delta_image_paths, slice_delta_audio()); } else if (has_images) { std::vector delta_image_paths; if (result.was_prefix) { @@ -573,7 +604,7 @@ PrefillResult do_prefill( } handle->model->prefill_with_images(prefill_tokens, delta_image_paths); } else if (has_audio) { - handle->model->prefill_with_audio(prefill_tokens, prompt.audio_features); + handle->model->prefill_with_audio(prefill_tokens, slice_delta_audio()); } else { handle->model->prefill(prefill_tokens, handle->model->get_prefill_chunk_size()); } @@ -676,14 +707,6 @@ int cactus_complete( bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); - const bool has_gemma4_mixed_media = prompt.model_type == Config::ModelType::GEMMA4 && has_images && has_audio; - auto decode_gemma4_mixed_media = [&](const std::vector& tokens, float* out_entropy) -> uint32_t { - (void)tokens; - (void)out_entropy; - throw std::runtime_error( - "Gemma4 mixed-media native decode is not present in this build. " - "Use cactus run-transpiled for transpiled graph bundles."); - }; auto stop_token_sequences = build_stop_sequences(tokenizer, prompt.options.stop_sequences, prompt.model_type, !prompt.tools.empty()); @@ -693,26 +716,28 @@ int cactus_complete( uint32_t next_token; size_t prompt_tokens; - if ((has_gemma4_mixed_media || has_audio) && !handle->processed_tokens.empty()) { + if (has_audio && !handle->processed_tokens.empty()) { auto& cache = handle->processed_tokens; size_t common = 0; size_t limit = std::min(cache.size(), prompt.tokens.size()); while (common < limit && cache[common] == prompt.tokens[common]) common++; if (common < cache.size()) { CACTUS_LOG_WARN("complete", "KV cache diverges from new prompt at position " << common - << "/" << cache.size() << "; trimming and re-prefilling the divergent suffix"); - size_t kv_len = handle->model->get_cache_size(); - if (kv_len > common) { - handle->model->remove_thinking_tokens({{common, kv_len - common}}); - } - cache.resize(common); + << "/" << cache.size() << "; resetting cache for clean audio re-prefill"); + reset_cache(handle); } } - if (has_gemma4_mixed_media) { - prompt_tokens = prompt.tokens.size(); - next_token = decode_gemma4_mixed_media(prompt.tokens, &first_token_entropy); - } else { + bool first_token_from_prefill = false; + if (!has_images && !has_audio && handle->processed_tokens.empty()) { + reset_cache(handle); + first_token_from_prefill = handle->model->prefill_and_sample_first_token(prompt.tokens, next_token); + if (first_token_from_prefill) { + prompt_tokens = prompt.tokens.size(); + first_token_entropy = 0.0f; + } + } + if (!first_token_from_prefill) { auto prefill_result = do_prefill(handle, prompt, prompt.tokens); prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); @@ -784,11 +809,10 @@ int cactus_complete( if (handle->should_stop) break; float token_entropy = 0.0f; - if (has_gemma4_mixed_media) { - next_token = decode_gemma4_mixed_media(handle->processed_tokens, &token_entropy); - } else if (has_audio) { + if (has_audio) { + uint32_t last_token = handle->processed_tokens.empty() ? next_token : handle->processed_tokens.back(); next_token = handle->model->decode_with_audio( - handle->processed_tokens, prompt.audio_features, + {last_token}, prompt.audio_features, prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, "", &token_entropy, prompt.options.min_p, prompt.options.repetition_penalty); @@ -1123,4 +1147,120 @@ int cactus_score_window( } } +int cactus_benchmark_tokens( + cactus_model_t model, + const uint32_t* prompt_tokens, + size_t prompt_token_len, + size_t decode_token_len, + char* response_buffer, + size_t buffer_size +) { + if (!model || !prompt_tokens || prompt_token_len == 0 || !response_buffer || buffer_size == 0) { + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + + try { + auto* handle = static_cast(model); + std::vector prompt(prompt_tokens, prompt_tokens + prompt_token_len); + + auto start_time = std::chrono::high_resolution_clock::now(); + double peak_ram_usage_mb = get_ram_usage_mb(); + auto sample_peak_ram = [&]() { + peak_ram_usage_mb = std::max(peak_ram_usage_mb, get_ram_usage_mb()); + }; + handle->model->reset_cache(); + maybe_pin_benchmark_main_to_max_perf_core(); + auto prefill_start_time = std::chrono::high_resolution_clock::now(); + size_t cache_prime_tokens = 0; + uint32_t next_token = 0; + bool first_token_from_prefill = false; + if (decode_token_len == 0) { + handle->model->prefill(prompt, handle->model->get_prefill_chunk_size(), "", false); + cache_prime_tokens = prompt.size(); + } else { + first_token_from_prefill = handle->model->prefill_and_sample_first_token(prompt, next_token); + if (first_token_from_prefill) { + cache_prime_tokens = prompt.size(); + } else if (prompt.size() > 1) { + handle->model->prefill( + std::vector(prompt.begin(), prompt.end() - 1), + handle->model->get_prefill_chunk_size() + ); + cache_prime_tokens = prompt.size() - 1; + } + } + auto prefill_end_time = std::chrono::high_resolution_clock::now(); + sample_peak_ram(); + auto first_decode_start_time = std::chrono::high_resolution_clock::now(); + if (decode_token_len > 0 && !first_token_from_prefill) { + next_token = handle->model->decode({prompt.back()}, 0.0f, 1.0f, 1); + } + sample_peak_ram(); + auto first_token_time = std::chrono::high_resolution_clock::now(); + + size_t generated = decode_token_len > 0 ? 1 : 0; + while (generated < decode_token_len) { + next_token = handle->model->decode({next_token}, 0.0f, 1.0f, 1); + sample_peak_ram(); + ++generated; + } + + auto end_time = std::chrono::high_resolution_clock::now(); + double ttft_ms = std::chrono::duration_cast(first_token_time - start_time).count() / 1000.0; + double total_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + double cache_prime_ms = std::chrono::duration_cast(prefill_end_time - prefill_start_time).count() / 1000.0; + double first_decode_ms = std::chrono::duration_cast(first_token_time - first_decode_start_time).count() / 1000.0; + double cache_state_copy_ms = handle->model->last_prefill_cache_copy_ms(); + double cache_prime_compute_ms = std::max(0.0, cache_prime_ms - cache_state_copy_ms); + double decode_ms = std::max(0.0, total_ms - ttft_ms); + double prefill_prepare_tps = (cache_prime_tokens > 0 && cache_prime_ms > 0.0) + ? (static_cast(cache_prime_tokens) * 1000.0) / cache_prime_ms + : 0.0; + double prefill_tps = (cache_prime_tokens > 0 && cache_prime_compute_ms > 0.0) + ? (static_cast(cache_prime_tokens) * 1000.0) / cache_prime_compute_ms + : 0.0; + double ttft_prompt_tps = ttft_ms > 0.0 ? (static_cast(prompt_token_len) * 1000.0) / ttft_ms : 0.0; + double decode_tps = (generated > 1 && decode_ms > 0.0) ? (static_cast(generated - 1) * 1000.0) / decode_ms : 0.0; + + std::ostringstream oss; + oss << "{" + << "\"success\":true," + << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << ttft_ms << "," + << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_ms << "," + << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << "," + << "\"prefill_compute_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << "," + << "\"prefill_prepare_tps\":" << std::fixed << std::setprecision(2) << prefill_prepare_tps << "," + << "\"ttft_prompt_tps\":" << std::fixed << std::setprecision(2) << ttft_prompt_tps << "," + << "\"cache_prime_ms\":" << std::fixed << std::setprecision(2) << cache_prime_ms << "," + << "\"cache_prime_compute_ms\":" << std::fixed << std::setprecision(2) << cache_prime_compute_ms << "," + << "\"cache_state_copy_ms\":" << std::fixed << std::setprecision(2) << cache_state_copy_ms << "," + << "\"cache_prime_tokens\":" << cache_prime_tokens << "," + << "\"first_decode_ms\":" << std::fixed << std::setprecision(2) << first_decode_ms << "," + << "\"first_token_from_prefill\":" << (first_token_from_prefill ? "true" : "false") << "," + << "\"prefill_padding_tokens\":" << handle->model->last_prefill_padding_tokens() << "," + << "\"prefill_scalar_tail_tokens\":" << handle->model->last_prefill_scalar_tail_tokens() << "," + << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << "," + << "\"prompt_tokens\":" << prompt_token_len << "," + << "\"completion_tokens\":" << generated << "," + << "\"peak_ram_usage_mb\":" << std::fixed << std::setprecision(2) << peak_ram_usage_mb << "," + << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() + << "}"; + + std::string result = oss.str(); + if (result.size() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + std::strcpy(response_buffer, result.c_str()); + return static_cast(result.size()); + } catch (const std::exception& e) { + handle_error_response(e.what(), response_buffer, buffer_size); + return -1; + } catch (...) { + handle_error_response("Unknown error during token benchmark", response_buffer, buffer_size); + return -1; + } +} + } diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 0899634b2..3ba72c9e5 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "cactus_graph.h" @@ -580,18 +581,21 @@ class Model { uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f); + bool prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token); - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + void prefill(const std::vector& tokens, size_t chunk_size = 128, const std::string& profile_file = "", + bool prepare_decode = true); void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, const std::string& profile_file = ""); - void prefill_with_audio(const std::vector& tokens, const std::vector& audio_features, + void prefill_with_audio(const std::vector& tokens, + const std::vector>& audio_features_per_message, const std::string& profile_file = ""); void prefill_with_media(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features, + const std::vector>& audio_features_per_message, const std::string& profile_file = ""); uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, @@ -599,7 +603,8 @@ class Model { size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f); - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, + uint32_t decode_with_audio(const std::vector& tokens, + const std::vector>& audio_features_per_message, float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f, @@ -635,7 +640,10 @@ class Model { bool load_npu_prefill(const std::string& model_path); bool has_npu_prefill() const { return false; } - size_t get_prefill_chunk_size() const { return 256; } + size_t get_prefill_chunk_size() const { return 128; } + double last_prefill_cache_copy_ms() const { return last_prefill_cache_copy_ms_; } + size_t last_prefill_padding_tokens() const { return last_prefill_padding_tokens_; } + size_t last_prefill_scalar_tail_tokens() const { return last_prefill_scalar_tail_tokens_; } void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} @@ -655,7 +663,7 @@ class Model { std::string path; }; - struct CacheStateEntry { + struct CacheStateBinding { std::string layer_key; int key_node_id = -1; int value_node_id = -1; @@ -669,32 +677,56 @@ class Model { std::vector output_node_ids; std::vector logical_outputs; std::vector bindings; + std::vector cache_states; std::unique_ptr graph; std::vector> input_buffers; - std::vector cache_states; }; void copy_cache_state(const Component& src, Component& dst); - void reset_component_cache_states(Component& comp); + + struct ChunkedPrefillResult { + size_t logical_tokens = 0; + size_t executed_tokens = 0; + size_t padding_tokens = 0; + size_t scalar_tail_tokens = 0; + size_t last_logit_row = 0; + }; bool load_manifest(); bool setup_tokenizer(); - bool load_components(); + bool load_components(const std::unordered_set& required_components); + bool load_component_graph(Component& comp); + void unload_component_graph(Component& comp); bool bind_runtime_buffers(Component& comp); void run_step(uint32_t token_id, size_t position, bool read_logits); + void run_encoder_step(uint32_t token_id, size_t position); void run_media_step(size_t position, const uint8_t* feature_row, size_t feature_row_bytes, Precision feature_precision); + void copy_component_outputs_to_inputs(const Component& source, Component& target); void copy_encoder_outputs_to_decoder(const Component& enc); + void copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index); + void copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset); + bool cache_states_compatible(const Component& source, const Component& target) const; + void copy_cache_states(const Component& source, Component& target, size_t logical_current = std::numeric_limits::max()); + void reset_component_cache_states(Component& comp); + size_t component_chunk_tokens(const Component& comp, const std::string& input_name) const; + size_t component_output_tokens(const Component& comp, const std::string& output_name) const; + ChunkedPrefillResult run_chunked_prefill(const std::vector& tokens, size_t start_position, + size_t chunk_size, bool prepare_decode); + void run_full_context_text(); + uint32_t argmax_component_logits(Component& comp, size_t logit_row = std::numeric_limits::max()); void write_int_input(Component& comp, const std::string& name, int64_t value); + void write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value); void write_bytes_input(Component& comp, const std::string& name, const void* data, size_t byte_size); int input_index(const Component& comp, const std::string& name) const; int output_index(const Component& comp, const std::string& name) const; uint32_t argmax_last_logits(); void run_vision_encoder(const std::string& image_path); void run_audio_encoder(const std::vector& audio_features); + void run_audio_encoder_messages(const std::vector>& audio_features_per_message); bool run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features); + const std::vector>& audio_features_per_message); bool build_lm_encoder_outputs_dynamic_gemma4( const std::vector& tokens, std::map>& store_bytes, @@ -705,6 +737,10 @@ class Model { std::map components_; Component* encoder_ = nullptr; Component* decoder_ = nullptr; + Component* decoder_prefill_ = nullptr; + Component* prefill_encoder_ = nullptr; + enum class DecodeRoute { CACHED_STEP, DIRECT_DECODER_STEP, FULL_CONTEXT_TEXT }; + DecodeRoute decode_route_ = DecodeRoute::CACHED_STEP; Component* vision_encoder_ = nullptr; Component* audio_encoder_ = nullptr; Component* lm_encoder_media_step_ = nullptr; @@ -724,6 +760,11 @@ class Model { bool initialized_ = false; size_t cache_total_seq_len_ = 0; size_t cache_max_seq_len_ = 4096; + size_t last_logit_position_ = 0; + double last_prefill_cache_copy_ms_ = 0.0; + size_t last_prefill_padding_tokens_ = 0; + size_t last_prefill_scalar_tail_tokens_ = 0; + std::vector context_tokens_; static constexpr size_t MAX_TOKEN_HISTORY = 128; std::vector token_history_; @@ -871,6 +912,16 @@ struct Lfm2VlImagePreprocessed { Lfm2VlImagePreprocessed preprocess_lfm2_vl_image(const std::string& image_path, const Config& config); +struct Qwen3VlImagePreprocessed { + std::vector pixel_values; + size_t grid_t = 1; + size_t grid_h = 0; + size_t grid_w = 0; + size_t patch_dim = 0; +}; + +Qwen3VlImagePreprocessed preprocess_qwen3_vl_image(const std::string& image_path, const Config& config); + struct SpectrogramConfig { size_t n_fft = 400; diff --git a/cactus-engine/src/engine_image.cpp b/cactus-engine/src/engine_image.cpp index bd9ee452d..f73c2429a 100644 --- a/cactus-engine/src/engine_image.cpp +++ b/cactus-engine/src/engine_image.cpp @@ -19,6 +19,115 @@ namespace cactus { namespace engine { +namespace { + +constexpr int kPillowResizePrecisionBits = 22; + +static unsigned char pillow_clip8(int64_t value) { + int shifted = static_cast(value >> kPillowResizePrecisionBits); + if (shifted < 0) return 0; + if (shifted > 255) return 255; + return static_cast(shifted); +} + +static void precompute_pillow_bilinear_coeffs(int in_size, int out_size, + std::vector& bounds, + std::vector& coeffs, + int& ksize) { + const double scale = static_cast(in_size) / static_cast(out_size); + const double filterscale = std::max(scale, 1.0); + const double support = filterscale; + const double inv_filterscale = 1.0 / filterscale; + ksize = static_cast(std::ceil(support)) * 2 + 1; + bounds.assign(static_cast(out_size) * 2, 0); + coeffs.assign(static_cast(out_size) * ksize, 0); + + for (int out = 0; out < out_size; ++out) { + const double center = (static_cast(out) + 0.5) * scale; + int xmin = static_cast(center - support + 0.5); + if (xmin < 0) xmin = 0; + int xmax = static_cast(center + support + 0.5); + if (xmax > in_size) xmax = in_size; + xmax -= xmin; + + bounds[static_cast(out) * 2] = xmin; + bounds[static_cast(out) * 2 + 1] = xmax; + + std::vector weights(static_cast(ksize), 0.0); + double total = 0.0; + for (int i = 0; i < xmax; ++i) { + double x = (static_cast(i + xmin) - center + 0.5) * inv_filterscale; + if (x < 0.0) x = -x; + double weight = x < 1.0 ? 1.0 - x : 0.0; + weights[static_cast(i)] = weight; + total += weight; + } + for (int i = 0; i < ksize; ++i) { + double weight = total != 0.0 ? weights[static_cast(i)] / total : 0.0; + coeffs[static_cast(out) * ksize + i] = static_cast( + weight < 0.0 + ? -0.5 + weight * static_cast(1 << kPillowResizePrecisionBits) + : 0.5 + weight * static_cast(1 << kPillowResizePrecisionBits)); + } + } +} + +static std::vector resize_rgb_uint8_pillow_bilinear(const unsigned char* src, + int src_w, + int src_h, + int dst_w, + int dst_h) { + std::vector x_bounds; + std::vector y_bounds; + std::vector x_coeffs; + std::vector y_coeffs; + int x_ksize = 0; + int y_ksize = 0; + precompute_pillow_bilinear_coeffs(src_w, dst_w, x_bounds, x_coeffs, x_ksize); + precompute_pillow_bilinear_coeffs(src_h, dst_h, y_bounds, y_coeffs, y_ksize); + + const int y_first = y_bounds[0]; + const int y_last = y_bounds[static_cast(dst_h - 1) * 2] + + y_bounds[static_cast(dst_h - 1) * 2 + 1]; + const int temp_h = y_last - y_first; + std::vector temp(static_cast(dst_w) * temp_h * 3); + + for (int ty = 0; ty < temp_h; ++ty) { + const int sy = y_first + ty; + for (int dx = 0; dx < dst_w; ++dx) { + const int xmin = x_bounds[static_cast(dx) * 2]; + const int count = x_bounds[static_cast(dx) * 2 + 1]; + const int32_t* k = x_coeffs.data() + static_cast(dx) * x_ksize; + for (int c = 0; c < 3; ++c) { + int64_t sum = static_cast(1) << (kPillowResizePrecisionBits - 1); + for (int x = 0; x < count; ++x) { + sum += static_cast(src[(static_cast(sy) * src_w + xmin + x) * 3 + c]) * k[x]; + } + temp[(static_cast(ty) * dst_w + dx) * 3 + c] = pillow_clip8(sum); + } + } + } + + std::vector dst(static_cast(dst_w) * dst_h * 3); + for (int dy = 0; dy < dst_h; ++dy) { + const int ymin = y_bounds[static_cast(dy) * 2] - y_first; + const int count = y_bounds[static_cast(dy) * 2 + 1]; + const int32_t* k = y_coeffs.data() + static_cast(dy) * y_ksize; + for (int dx = 0; dx < dst_w; ++dx) { + for (int c = 0; c < 3; ++c) { + int64_t sum = static_cast(1) << (kPillowResizePrecisionBits - 1); + for (int y = 0; y < count; ++y) { + sum += static_cast(temp[(static_cast(ymin + y) * dst_w + dx) * 3 + c]) * k[y]; + } + dst[(static_cast(dy) * dst_w + dx) * 3 + c] = pillow_clip8(sum); + } + } + } + return dst; +} + +} // namespace + Siglip2Preprocessor::PreprocessedImage::~PreprocessedImage() { pixel_values.clear(); pixel_attention_mask.clear(); @@ -634,16 +743,9 @@ Gemma4ImagePreprocessed preprocess_gemma4_image(const std::string& image_path, c resized[i] = static_cast(raw[i]); } } else { - std::vector src_float(static_cast(width) * height * 3); - for (size_t i = 0; i < src_float.size(); ++i) { - src_float[i] = static_cast(raw[i]); - } - if (!stbir_resize_float_linear( - src_float.data(), width, height, 0, - resized.data(), target_w, target_h, 0, - STBIR_RGB)) { - stbi_image_free(raw); - throw std::runtime_error("Failed to resize image: " + image_path); + std::vector resized_u8 = resize_rgb_uint8_pillow_bilinear(raw, width, height, target_w, target_h); + for (size_t i = 0; i < resized.size(); ++i) { + resized[i] = static_cast(resized_u8[i]); } } stbi_image_free(raw); @@ -700,6 +802,99 @@ Gemma4ImagePreprocessed preprocess_gemma4_image(const std::string& image_path, c return result; } +Qwen3VlImagePreprocessed preprocess_qwen3_vl_image(const std::string& image_path, const Config& config) { + Qwen3VlImagePreprocessed result; + const int patch_size = static_cast(config.vision_patch_size ? config.vision_patch_size : 16); + const int temporal_patch_size = 2; + const int merge_size = 2; + const size_t image_tokens = config.image_seq_len ? static_cast(config.image_seq_len) : 64; + const size_t merged_patches = image_tokens * static_cast(merge_size * merge_size); + const int grid_side = static_cast(std::sqrt(static_cast(merged_patches))); + if (patch_size <= 0 || grid_side <= 0 || static_cast(grid_side * grid_side) != merged_patches) { + throw std::runtime_error("Qwen3-VL native image preprocessing requires a square static image grid"); + } + + int width = 0, height = 0, channels = 0; + unsigned char* raw = stbi_load(image_path.c_str(), &width, &height, &channels, 3); + if (!raw) { + throw std::runtime_error("Failed to load image: " + image_path + " - " + std::string(stbi_failure_reason())); + } + if (width <= 0 || height <= 0) { + stbi_image_free(raw); + throw std::runtime_error("Loaded image has invalid dimensions: " + image_path); + } + + const int target_h = grid_side * patch_size; + const int target_w = grid_side * patch_size; + std::vector resized(static_cast(target_w) * target_h * 3); + if (target_w == width && target_h == height) { + for (size_t i = 0; i < resized.size(); ++i) { + resized[i] = static_cast(raw[i]); + } + } else { + std::vector src_float(static_cast(width) * height * 3); + for (size_t i = 0; i < src_float.size(); ++i) { + src_float[i] = static_cast(raw[i]); + } + if (!stbir_resize_float_linear( + src_float.data(), width, height, 0, + resized.data(), target_w, target_h, 0, + STBIR_RGB)) { + stbi_image_free(raw); + throw std::runtime_error("Failed to resize image: " + image_path); + } + } + stbi_image_free(raw); + + const float rescale_factor = config.rescale_factor > 0.0f ? config.rescale_factor : (1.0f / 255.0f); + const float mean_v = config.image_mean; + const float std_v = config.image_std != 0.0f ? config.image_std : 1.0f; + for (float& v : resized) { + v = (v * rescale_factor - mean_v) / std_v; + } + + const size_t channel = 3; + const size_t grid_t = 1; + const size_t grid_h = static_cast(grid_side); + const size_t grid_w = static_cast(grid_side); + const size_t patch_dim = channel * static_cast(temporal_patch_size) * patch_size * patch_size; + result.pixel_values.assign(grid_t * grid_h * grid_w * patch_dim, 0.0f); + + for (size_t gh_major = 0; gh_major < grid_h / merge_size; ++gh_major) { + for (size_t gw_major = 0; gw_major < grid_w / merge_size; ++gw_major) { + for (size_t mh = 0; mh < static_cast(merge_size); ++mh) { + for (size_t mw = 0; mw < static_cast(merge_size); ++mw) { + const size_t patch_y = gh_major * merge_size + mh; + const size_t patch_x = gw_major * merge_size + mw; + const size_t patch_index = + (((gh_major * (grid_w / merge_size) + gw_major) * merge_size + mh) * merge_size + mw); + float* dst = result.pixel_values.data() + patch_index * patch_dim; + size_t dst_i = 0; + for (size_t c = 0; c < channel; ++c) { + for (size_t t = 0; t < static_cast(temporal_patch_size); ++t) { + (void)t; + for (int y = 0; y < patch_size; ++y) { + const size_t img_y = patch_y * patch_size + static_cast(y); + for (int x = 0; x < patch_size; ++x) { + const size_t img_x = patch_x * patch_size + static_cast(x); + const size_t src_off = (img_y * static_cast(target_w) + img_x) * channel + c; + dst[dst_i++] = resized[src_off]; + } + } + } + } + } + } + } + } + + result.grid_t = grid_t; + result.grid_h = grid_h; + result.grid_w = grid_w; + result.patch_dim = patch_dim; + return result; +} + Lfm2VlImagePreprocessed preprocess_lfm2_vl_image(const std::string& image_path, const Config& config) { Siglip2Preprocessor::Config sp_cfg; sp_cfg.patch_size = config.vision_patch_size ? static_cast(config.vision_patch_size) : 16; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index fec6f94f5..41b94cb5a 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -19,10 +20,70 @@ #include #include #include +#include namespace cactus { namespace engine { +float read_scalar_value(Precision precision, const uint8_t* data, size_t index) { + const uint8_t* ptr = data + PrecisionTraits::byte_offset_of(precision, index); + switch (precision) { + case Precision::FP32: + return *reinterpret_cast(ptr); + case Precision::FP16: + return static_cast(*reinterpret_cast(ptr)); + case Precision::INT8: + return static_cast(*reinterpret_cast(ptr)); + default: + return 0.0f; + } +} + +void write_scalar_value(Precision precision, uint8_t* data, size_t index, float value) { + uint8_t* ptr = data + PrecisionTraits::byte_offset_of(precision, index); + switch (precision) { + case Precision::FP32: + *reinterpret_cast(ptr) = value; + break; + case Precision::FP16: + *reinterpret_cast<__fp16*>(ptr) = static_cast<__fp16>(value); + break; + case Precision::INT8: + *reinterpret_cast(ptr) = static_cast(value); + break; + default: + break; + } +} + +bool copy_component_tensor(CactusGraph& source_graph, + const BufferDesc& src_desc, + size_t src_node, + const BufferDesc& dst_desc, + std::vector& dst_buffer, + size_t dst_element_offset, + size_t element_count, + const std::string& name) { + const auto* src_ptr = static_cast(source_graph.get_output(src_node)); + if (src_desc.precision == dst_desc.precision) { + size_t dst_offset = PrecisionTraits::byte_offset_of(dst_desc.precision, dst_element_offset); + std::memcpy( + dst_buffer.data() + dst_offset, + src_ptr, + PrecisionTraits::packed_size_of(src_desc.precision, element_count)); + return true; + } + if (name != "position_ids" && name != "attention_mask") return false; + for (size_t i = 0; i < element_count; ++i) { + write_scalar_value( + dst_desc.precision, + dst_buffer.data(), + dst_element_offset + i, + read_scalar_value(src_desc.precision, src_ptr, i)); + } + return true; +} + void ConvCache::init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision) { num_layers = layers; hidden_size = hidden_dim; @@ -144,42 +205,107 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_ERROR("model", "Tokenizer init failed for bundle: " << bundle_dir); return false; } - if (!load_components()) return false; - - encoder_ = components_.count("lm_encoder_step") ? &components_.at("lm_encoder_step") : nullptr; - decoder_ = components_.count("decoder_step") ? &components_.at("decoder_step") : nullptr; - const bool is_lm = (decoder_ != nullptr); + std::string encoder_name; + std::string decoder_name; + std::unordered_set required_components; const bool is_whisper_transcription = config_.model_type == Config::ModelType::WHISPER && components_.count("audio_encoder") && components_.count("decoder_cross_kv") && components_.count("decoder_step"); + bool has_chunked_prefill = components_.count("lm_encoder_step") + && components_.count("decoder_media_step") + && components_.count("lm_encoder_text_chunk") + && components_.count("decoder_prefill_chunk"); if (is_whisper_transcription) { - decoder_ = &components_.at("decoder_step"); - } - const bool is_transcription = components_.count("audio_encoder") && - (components_.count("decoder") || components_.count("decoder_joint") || is_whisper_transcription); - if (!is_lm && !is_transcription) { + decoder_name = "decoder_step"; + decode_route_ = DecodeRoute::DIRECT_DECODER_STEP; + required_components = {"audio_encoder", "decoder_cross_kv", decoder_name}; + } else if (has_chunked_prefill) { + encoder_name = "lm_encoder_step"; + decoder_name = "decoder_media_step"; + decode_route_ = DecodeRoute::CACHED_STEP; + required_components = { + encoder_name, + decoder_name, + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + }; + } else if (components_.count("decoder_step") + && input_index(components_.at("decoder_step"), "input_ids") >= 0 + && input_index(components_.at("decoder_step"), "position_ids") >= 0) { + decoder_name = "decoder_step"; + decode_route_ = DecodeRoute::DIRECT_DECODER_STEP; + required_components = {decoder_name}; + } else if (components_.count("lm_encoder_step") && components_.count("decoder_step")) { + encoder_name = "lm_encoder_step"; + decoder_name = "decoder_step"; + decode_route_ = DecodeRoute::CACHED_STEP; + required_components = {encoder_name, decoder_name}; + if (components_.count("decoder_prefill_chunk")) { + required_components.insert("decoder_prefill_chunk"); + } + if (components_.count("lm_encoder_text_chunk")) { + required_components.insert("lm_encoder_text_chunk"); + } + } else if (components_.count("text_lm_encoder") && components_.count("decoder")) { + encoder_name = "text_lm_encoder"; + decoder_name = "decoder"; + decode_route_ = DecodeRoute::FULL_CONTEXT_TEXT; + required_components = {encoder_name, decoder_name}; + } else if (components_.count("audio_encoder") && + (components_.count("decoder") || components_.count("decoder_joint"))) { + decoder_name = components_.count("decoder") ? "decoder" : "decoder_joint"; + decode_route_ = DecodeRoute::DIRECT_DECODER_STEP; + required_components = {"audio_encoder", decoder_name}; + } else { CACTUS_LOG_ERROR("model", "Bundle missing required components: need lm_encoder_step+decoder_step (LM), audio_encoder+decoder (transcription), or Whisper audio_encoder+decoder_cross_kv+decoder_step"); return false; } - if (encoder_ && !bind_runtime_buffers(*encoder_)) return false; - if (decoder_ && !bind_runtime_buffers(*decoder_)) return false; - + for (const auto& optional : { + "vision_encoder", + "audio_encoder", + "lm_encoder_media_step", + "decoder_prefill_chunk", + "lm_encoder", + }) { + if (components_.count(optional)) { + required_components.insert(optional); + } + } + if (!load_components(required_components)) return false; + if (!encoder_name.empty()) encoder_ = &components_.at(encoder_name); + if (!decoder_name.empty()) decoder_ = &components_.at(decoder_name); + if (components_.count("decoder_prefill_chunk") && components_.at("decoder_prefill_chunk").graph) { + decoder_prefill_ = &components_.at("decoder_prefill_chunk"); + decoder_prefill_chunk_ = decoder_prefill_; + } + if (components_.count("lm_encoder_text_chunk") && components_.at("lm_encoder_text_chunk").graph) { + prefill_encoder_ = &components_.at("lm_encoder_text_chunk"); + } vision_encoder_ = components_.count("vision_encoder") ? &components_.at("vision_encoder") : nullptr; audio_encoder_ = components_.count("audio_encoder") ? &components_.at("audio_encoder") : nullptr; lm_encoder_media_step_ = components_.count("lm_encoder_media_step") ? &components_.at("lm_encoder_media_step") : nullptr; - decoder_prefill_chunk_ = components_.count("decoder_prefill_chunk") ? &components_.at("decoder_prefill_chunk") : nullptr; lm_encoder_ = components_.count("lm_encoder") ? &components_.at("lm_encoder") : nullptr; - lm_encoder_text_chunk_ = components_.count("lm_encoder_text_chunk") ? &components_.at("lm_encoder_text_chunk") : nullptr; - lm_encoder_media_chunk_ = components_.count("lm_encoder_media_chunk") ? &components_.at("lm_encoder_media_chunk") : nullptr; - if (vision_encoder_ && !bind_runtime_buffers(*vision_encoder_)) return false; - if (audio_encoder_ && !bind_runtime_buffers(*audio_encoder_)) return false; - if (lm_encoder_media_step_ && !bind_runtime_buffers(*lm_encoder_media_step_)) return false; - if (decoder_prefill_chunk_ && !bind_runtime_buffers(*decoder_prefill_chunk_)) return false; - if (lm_encoder_ && !bind_runtime_buffers(*lm_encoder_)) return false; - if (lm_encoder_text_chunk_ && !bind_runtime_buffers(*lm_encoder_text_chunk_)) return false; - if (lm_encoder_media_chunk_ && !bind_runtime_buffers(*lm_encoder_media_chunk_)) return false; + std::vector to_bind = { + encoder_, + prefill_encoder_, + decoder_, + decoder_prefill_, + vision_encoder_, + audio_encoder_, + lm_encoder_media_step_, + lm_encoder_, + }; + if (components_.count("decoder_cross_kv")) { + to_bind.push_back(&components_.at("decoder_cross_kv")); + } + std::unordered_set bound; + for (Component* comp : to_bind) { + if (!comp || !comp->graph || bound.count(comp)) continue; + if (!bind_runtime_buffers(*comp)) return false; + bound.insert(comp); + } if (vision_encoder_ && tokenizer_ && !vision_encoder_->output_node_ids.empty()) { size_t out_node = static_cast(vision_encoder_->output_node_ids[0]); @@ -240,15 +366,15 @@ bool Model::load_manifest() { } } if (c.count("cache_state_node_ids")) { - for (const auto& ev : c.at("cache_state_node_ids").get()) { - if (!ev.is()) continue; - const auto& e = ev.get(); - CacheStateEntry cs; - if (e.count("layer_key")) cs.layer_key = e.at("layer_key").get(); - if (e.count("key") && e.at("key").is()) - cs.key_node_id = static_cast(e.at("key").get()); - if (e.count("value") && e.at("value").is()) - cs.value_node_id = static_cast(e.at("value").get()); + for (const auto& sv : c.at("cache_state_node_ids").get()) { + if (!sv.is()) continue; + const auto& s = sv.get(); + CacheStateBinding cs; + if (s.count("layer_key")) cs.layer_key = s.at("layer_key").get(); + if (s.count("key") && s.at("key").is()) + cs.key_node_id = static_cast(s.at("key").get()); + if (s.count("value") && s.at("value").is()) + cs.value_node_id = static_cast(s.at("value").get()); if (cs.key_node_id >= 0 && cs.value_node_id >= 0) { comp.cache_states.push_back(std::move(cs)); } @@ -273,28 +399,51 @@ bool Model::setup_tokenizer() { return tokenizer_->load_vocabulary_with_config(vocab, merges, cfg); } -bool Model::load_components() { +bool Model::load_components(const std::unordered_set& required_components) { for (auto& [name, comp] : components_) { - if (comp.graph_path.empty()) continue; - fs::path full = fs::path(bundle_dir_) / comp.graph_path; + if (!required_components.empty() && !required_components.count(name)) continue; + if (!load_component_graph(comp)) return false; + } + return true; +} + +bool Model::load_component_graph(Component& comp) { + if (comp.graph) return true; + if (comp.graph_path.empty()) return true; + fs::path full = fs::path(bundle_dir_) / comp.graph_path; + try { + comp.graph = std::make_unique(CactusGraph::load(full.string())); + comp.graph->retain_outputs(comp.output_node_ids); + } catch (const std::exception& e) { + CACTUS_LOG_ERROR("model", "load " << comp.graph_path << ": " << e.what()); + return false; + } + for (const auto& b : comp.bindings) { + if (b.node_id < 0) continue; try { - comp.graph = std::make_unique(CactusGraph::load(full.string())); + fs::path weight_path(b.path); + if (weight_path.is_absolute()) { + fs::path local = fs::path(bundle_dir_) / weight_path.filename(); + if (fs::exists(local)) weight_path = local; + } else { + weight_path = fs::path(bundle_dir_) / weight_path; + } + comp.graph->bind_mmap_weights(static_cast(b.node_id), weight_path.string()); } catch (const std::exception& e) { - CACTUS_LOG_ERROR("model", "load " << comp.graph_path << ": " << e.what()); + CACTUS_LOG_ERROR("model", "bind " << b.path << ": " << e.what()); return false; } - for (const auto& b : comp.bindings) { - if (b.node_id < 0) continue; - try { - comp.graph->bind_mmap_weights(static_cast(b.node_id), - (fs::path(bundle_dir_) / b.path).string()); - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("model", "bind " << b.path << ": " << e.what()); - return false; - } - } } - return true; + return bind_runtime_buffers(comp); +} + +void Model::unload_component_graph(Component& comp) { + if (comp.graph) { + comp.graph->release_runtime_buffers(); + comp.graph->release_all_weight_pages(); + } + comp.input_buffers.clear(); + comp.graph.reset(); } bool Model::bind_runtime_buffers(Component& comp) { @@ -316,23 +465,30 @@ int Model::input_index(const Component& comp, const std::string& name) const { } void Model::write_int_input(Component& comp, const std::string& name, int64_t value) { + write_int_input_at(comp, name, 0, value); +} + +void Model::write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value) { int idx = input_index(comp, name); if (idx < 0) return; size_t node_id = static_cast(comp.runtime_input_node_ids[idx]); const auto& desc = comp.graph->get_output_buffer(node_id); auto& buf = comp.input_buffers[idx]; + if (index >= desc.total_size) return; + size_t offset = PrecisionTraits::byte_offset_of(desc.precision, index); + auto* dst = buf.data() + offset; switch (desc.precision) { case Precision::FP32: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; case Precision::FP16: - *reinterpret_cast<__fp16*>(buf.data()) = static_cast<__fp16>(value); + *reinterpret_cast<__fp16*>(dst) = static_cast<__fp16>(value); break; case Precision::INT8: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; default: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; } } @@ -370,15 +526,398 @@ void Model::copy_encoder_outputs_to_decoder(const Component& enc) { } void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { - if (encoder_) { - write_int_input(*encoder_, "input_ids", static_cast(token_id)); - write_int_input(*encoder_, "position_ids", static_cast(position)); - encoder_->graph->execute(); - copy_encoder_outputs_to_decoder(*encoder_); - } else { + if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { write_int_input(*decoder_, "input_ids", static_cast(token_id)); write_int_input(*decoder_, "position_ids", static_cast(position)); + decoder_->graph->execute(); + return; } + run_encoder_step(token_id, position); + copy_component_outputs_to_inputs(*encoder_, *decoder_); + decoder_->graph->execute(); +} + +void Model::run_encoder_step(uint32_t token_id, size_t position) { + write_int_input(*encoder_, "input_ids", static_cast(token_id)); + write_int_input(*encoder_, "position_ids", static_cast(position)); + encoder_->graph->execute(); +} + +void Model::copy_component_outputs_to_inputs(const Component& source, Component& target) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + std::fill(target.input_buffers[dst_idx].begin(), target.input_buffers[dst_idx].end(), 0); + size_t elements = std::min(src_desc.total_size, dst_desc.total_size); + if (!copy_component_tensor(*source.graph, src_desc, src_node, dst_desc, target.input_buffers[dst_idx], 0, elements, out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +void Model::copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + size_t chunk_tokens = component_chunk_tokens(target, out_name); + if (chunk_tokens <= token_index || chunk_tokens == 0) { + throw std::runtime_error("chunk prefill token index exceeds input capacity for " + out_name); + } + if (dst_desc.total_size % chunk_tokens != 0) { + throw std::runtime_error("chunk prefill input shape is not token-aligned for " + out_name); + } + size_t elements_per_token = dst_desc.total_size / chunk_tokens; + if (src_desc.total_size != elements_per_token) { + throw std::runtime_error("component output/input token shape mismatch for " + out_name); + } + if (!copy_component_tensor( + *source.graph, + src_desc, + src_node, + dst_desc, + target.input_buffers[dst_idx], + token_index * elements_per_token, + src_desc.total_size, + out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +void Model::copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + size_t src_tokens = component_output_tokens(source, out_name); + size_t dst_tokens = component_chunk_tokens(target, out_name); + if (src_tokens == 0 || dst_tokens == 0 || token_offset + src_tokens > dst_tokens) { + throw std::runtime_error("chunk prefill output range exceeds input capacity for " + out_name); + } + if (src_desc.total_size % src_tokens != 0 || dst_desc.total_size % dst_tokens != 0) { + throw std::runtime_error("chunk prefill output/input shape is not token-aligned for " + out_name); + } + size_t src_elements_per_token = src_desc.total_size / src_tokens; + size_t dst_elements_per_token = dst_desc.total_size / dst_tokens; + if (src_elements_per_token != dst_elements_per_token) { + throw std::runtime_error("component output/input token shape mismatch for " + out_name); + } + if (!copy_component_tensor( + *source.graph, + src_desc, + src_node, + dst_desc, + target.input_buffers[dst_idx], + token_offset * dst_elements_per_token, + src_desc.total_size, + out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +bool Model::cache_states_compatible(const Component& source, const Component& target) const { + if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) return false; + for (size_t i = 0; i < source.cache_states.size(); ++i) { + const auto& src = source.cache_states[i]; + const auto& dst = target.cache_states[i]; + if (src.layer_key != dst.layer_key) return false; + if (src.key_node_id < 0 || src.value_node_id < 0 || dst.key_node_id < 0 || dst.value_node_id < 0) return false; + } + return true; +} + +void Model::copy_cache_states(const Component& source, Component& target, size_t logical_current) { + if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) { + throw std::runtime_error("prefill and step cache states are not compatible"); + } + for (size_t i = 0; i < source.cache_states.size(); ++i) { + const auto& src = source.cache_states[i]; + const auto& dst = target.cache_states[i]; + if (src.layer_key != dst.layer_key) { + throw std::runtime_error("prefill and step cache layer mismatch: " + src.layer_key + " != " + dst.layer_key); + } + for (auto [src_node, dst_node] : {std::pair{src.key_node_id, dst.key_node_id}, std::pair{src.value_node_id, dst.value_node_id}}) { + const auto& src_desc = source.graph->get_output_buffer(static_cast(src_node)); + const auto& dst_desc = target.graph->get_output_buffer(static_cast(dst_node)); + if (src_desc.precision != dst_desc.precision) { + std::ostringstream oss; + oss << "prefill and step cache precision mismatch at layer " << src.layer_key + << ": " << static_cast(src_desc.precision) + << " vs " << static_cast(dst_desc.precision); + throw std::runtime_error(oss.str()); + } + void* src_ptr = source.graph->get_output(static_cast(src_node)); + void* dst_ptr = target.graph->get_output(static_cast(dst_node)); + if (src_desc.byte_size == dst_desc.byte_size && logical_current == std::numeric_limits::max()) { + std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); + continue; + } + if (src_desc.precision == Precision::INT8) { + auto* src_meta = static_cast(src_ptr); + auto* dst_meta = static_cast(dst_ptr); + const size_t src_current = static_cast(src_meta[0]); + const size_t effective_current = std::min(src_current, logical_current); + const size_t src_max = static_cast(src_meta[1]); + const size_t kv_heads = static_cast(src_meta[2]); + const size_t head_dim = static_cast(src_meta[3]); + const size_t sink = static_cast(src_meta[4]); + if (src_max == 0 || kv_heads == 0 || head_dim == 0) { + throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); + } + const size_t groups = (head_dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + const size_t int8_stride = kv_heads * head_dim; + const size_t scale_stride = kv_heads * groups; + const size_t row_bytes = int8_stride + scale_stride * sizeof(float); + const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; + if (dst_max == 0) { + throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); + } + const size_t dst_current = std::min(effective_current, dst_max); + dst_meta[0] = dst_current; + dst_meta[1] = dst_max; + dst_meta[2] = kv_heads; + dst_meta[3] = head_dim; + dst_meta[4] = std::min(sink, dst_current); + std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); + + const auto* src_i8 = static_cast(src_ptr) + 64; + const auto* src_scales = reinterpret_cast( + static_cast(src_ptr) + 64 + src_max * int8_stride); + auto* dst_i8 = static_cast(dst_ptr) + 64; + auto* dst_scales = reinterpret_cast( + static_cast(dst_ptr) + 64 + dst_max * int8_stride); + auto copy_rows = [&](size_t dst_row, size_t src_row, size_t rows) { + if (rows == 0) return; + std::memcpy( + dst_i8 + dst_row * int8_stride, + src_i8 + src_row * int8_stride, + rows * int8_stride); + std::memcpy( + dst_scales + dst_row * scale_stride, + src_scales + src_row * scale_stride, + rows * scale_stride * sizeof(float)); + }; + if (effective_current <= dst_max) { + copy_rows(0, 0, effective_current); + } else { + const size_t copied_sink = std::min(sink, dst_max); + const size_t tail_rows = dst_max - copied_sink; + copy_rows(0, 0, copied_sink); + if (tail_rows > 0) copy_rows(copied_sink, effective_current - tail_rows, tail_rows); + } + continue; + } + if (PrecisionTraits::is_cq(src_desc.precision) || src_desc.byte_size < 64 || dst_desc.byte_size < 64) { + std::ostringstream oss; + oss << "prefill and step cache buffer mismatch at layer " << src.layer_key + << ": " << src_desc.byte_size << " bytes vs " << dst_desc.byte_size << " bytes"; + throw std::runtime_error(oss.str()); + } + + auto* src_meta = static_cast(src_ptr); + auto* dst_meta = static_cast(dst_ptr); + const size_t src_current = static_cast(src_meta[0]); + const size_t effective_current = std::min(src_current, logical_current); + const size_t src_max = static_cast(src_meta[1]); + const size_t kv_heads = static_cast(src_meta[2]); + const size_t head_dim = static_cast(src_meta[3]); + const size_t sink = static_cast(src_meta[4]); + if (src_max == 0 || kv_heads == 0 || head_dim == 0) { + throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); + } + const size_t row_bytes = kv_heads * head_dim * PrecisionTraits::size_of(src_desc.precision); + const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; + if (dst_max == 0) { + throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); + } + const size_t dst_current = std::min(effective_current, dst_max); + dst_meta[0] = dst_current; + dst_meta[1] = dst_max; + dst_meta[2] = kv_heads; + dst_meta[3] = head_dim; + dst_meta[4] = std::min(sink, dst_current); + std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); + const auto* src_rows = static_cast(src_ptr) + 64; + auto* dst_rows = static_cast(dst_ptr) + 64; + if (effective_current <= dst_max) { + std::memcpy(dst_rows, src_rows, effective_current * row_bytes); + } else { + const size_t copied_sink = std::min(sink, dst_max); + const size_t tail_rows = dst_max - copied_sink; + if (copied_sink > 0) { + std::memcpy(dst_rows, src_rows, copied_sink * row_bytes); + } + if (tail_rows > 0) { + std::memcpy( + dst_rows + copied_sink * row_bytes, + src_rows + (effective_current - tail_rows) * row_bytes, + tail_rows * row_bytes); + } + } + } + } +} + +void Model::reset_component_cache_states(Component& comp) { + for (const auto& state : comp.cache_states) { + for (int node_id : {state.key_node_id, state.value_node_id}) { + if (node_id < 0) continue; + const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); + if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue; + void* ptr = comp.graph->get_output(static_cast(node_id)); + if (!ptr) continue; + auto* metadata = static_cast(ptr); + metadata[0] = 0; + } + } +} + +size_t Model::component_chunk_tokens(const Component& comp, const std::string& input_name) const { + int idx = input_index(comp, input_name); + if (idx < 0) return 0; + const auto& desc = comp.graph->get_output_buffer(static_cast(comp.runtime_input_node_ids[idx])); + if (desc.shape.size() >= 2 && desc.shape[0] == 1) return desc.shape[1]; + return desc.shape.empty() ? 0 : desc.shape[0]; +} + +size_t Model::component_output_tokens(const Component& comp, const std::string& output_name) const { + for (size_t i = 0; i < comp.logical_outputs.size() && i < comp.output_node_ids.size(); ++i) { + if (comp.logical_outputs[i] != output_name) continue; + const auto& desc = comp.graph->get_output_buffer(static_cast(comp.output_node_ids[i])); + if (desc.shape.size() >= 2 && desc.shape[0] == 1) return desc.shape[1]; + return desc.shape.empty() ? 0 : desc.shape[0]; + } + return 0; +} + +Model::ChunkedPrefillResult Model::run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, bool prepare_decode) { + ChunkedPrefillResult result; + last_prefill_cache_copy_ms_ = 0.0; + last_prefill_padding_tokens_ = 0; + last_prefill_scalar_tail_tokens_ = 0; + if (decode_route_ != DecodeRoute::CACHED_STEP || !encoder_ || !decoder_ || !decoder_prefill_) return result; + if (start_position != 0) return result; + if (!load_component_graph(*decoder_prefill_)) return result; + if (prefill_encoder_ && !load_component_graph(*prefill_encoder_)) return result; + if (!cache_states_compatible(*decoder_prefill_, *decoder_)) return result; + size_t component_tokens = component_chunk_tokens(*decoder_prefill_, "inputs_embeds"); + if (component_tokens <= 1) return result; + size_t effective_chunk = chunk_size > 0 ? std::min(chunk_size, component_tokens) : component_tokens; + if (effective_chunk != component_tokens) effective_chunk = component_tokens; + const size_t whole_chunks_end = (tokens.size() / effective_chunk) * effective_chunk; + const size_t tail_tokens = tokens.size() - whole_chunks_end; + const size_t padding_cutoff = std::max(1, effective_chunk / 16); + const bool pad_tail = family_ != "lfm2_vl" && tail_tokens >= padding_cutoff; + const size_t executable_tokens = whole_chunks_end + (pad_tail ? effective_chunk : 0); + if (executable_tokens == 0) { + result.scalar_tail_tokens = tail_tokens; + last_prefill_scalar_tail_tokens_ = tail_tokens; + return result; + } + result.padding_tokens = executable_tokens > tokens.size() ? executable_tokens - tokens.size() : 0; + result.scalar_tail_tokens = tokens.size() - std::min(tokens.size(), executable_tokens); + last_prefill_padding_tokens_ = result.padding_tokens; + last_prefill_scalar_tail_tokens_ = result.scalar_tail_tokens; + + size_t encoder_chunk = 0; + if (prefill_encoder_ && input_index(*prefill_encoder_, "input_ids") >= 0 && input_index(*prefill_encoder_, "position_ids") >= 0) { + encoder_chunk = component_chunk_tokens(*prefill_encoder_, "input_ids"); + if (encoder_chunk == 0 || effective_chunk % encoder_chunk != 0) { + encoder_chunk = 0; + } + } + + size_t processed = 0; + while (processed + effective_chunk <= executable_tokens) { + for (size_t i = 0; i < decoder_prefill_->input_buffers.size(); ++i) { + std::fill(decoder_prefill_->input_buffers[i].begin(), decoder_prefill_->input_buffers[i].end(), 0); + } + if (encoder_chunk > 0) { + for (size_t chunk_offset = 0; chunk_offset < effective_chunk; chunk_offset += encoder_chunk) { + for (size_t i = 0; i < prefill_encoder_->input_buffers.size(); ++i) { + std::fill(prefill_encoder_->input_buffers[i].begin(), prefill_encoder_->input_buffers[i].end(), 0); + } + for (size_t i = 0; i < encoder_chunk; ++i) { + size_t index = processed + chunk_offset + i; + uint32_t token = index < tokens.size() ? tokens[index] : static_cast(config_.pad_token_id); + write_int_input_at(*prefill_encoder_, "input_ids", i, static_cast(token)); + write_int_input_at(*prefill_encoder_, "position_ids", i, static_cast(start_position + processed + chunk_offset + i)); + } + prefill_encoder_->graph->execute(); + copy_component_outputs_to_chunk_inputs_range(*prefill_encoder_, *decoder_prefill_, chunk_offset); + } + } else { + for (size_t i = 0; i < effective_chunk; ++i) { + size_t index = processed + i; + uint32_t token = index < tokens.size() ? tokens[index] : static_cast(config_.pad_token_id); + run_encoder_step(token, start_position + processed + i); + copy_component_outputs_to_chunk_inputs(*encoder_, *decoder_prefill_, i); + } + } + decoder_prefill_->graph->execute(); + processed += effective_chunk; + } + result.executed_tokens = processed; + result.logical_tokens = std::min(tokens.size(), processed); + if (result.logical_tokens > 0) { + result.last_logit_row = (result.logical_tokens - 1) % effective_chunk; + } + if (processed > 0 && prepare_decode) { + for (size_t i = 0; i < decoder_->input_buffers.size(); ++i) { + std::fill(decoder_->input_buffers[i].begin(), decoder_->input_buffers[i].end(), 0); + } + auto copy_start = std::chrono::high_resolution_clock::now(); + copy_cache_states(*decoder_prefill_, *decoder_, start_position + result.logical_tokens); + auto copy_end = std::chrono::high_resolution_clock::now(); + last_prefill_cache_copy_ms_ = std::chrono::duration_cast(copy_end - copy_start).count() / 1000.0; + } + return result; +} + +void Model::run_full_context_text() { + if (!encoder_ || !decoder_ || context_tokens_.empty()) return; + int input_ids_idx = input_index(*encoder_, "input_ids"); + int attention_mask_idx = input_index(*encoder_, "attention_mask"); + if (input_ids_idx < 0 || attention_mask_idx < 0) { + throw std::runtime_error("text_lm_encoder requires input_ids and attention_mask inputs"); + } + size_t input_node = static_cast(encoder_->runtime_input_node_ids[input_ids_idx]); + const auto& input_desc = encoder_->graph->get_output_buffer(input_node); + if (context_tokens_.size() > input_desc.total_size) { + throw std::runtime_error("context exceeds transpiled text_lm_encoder capacity"); + } + std::fill(encoder_->input_buffers[input_ids_idx].begin(), encoder_->input_buffers[input_ids_idx].end(), 0); + std::fill(encoder_->input_buffers[attention_mask_idx].begin(), encoder_->input_buffers[attention_mask_idx].end(), 0); + for (size_t i = 0; i < context_tokens_.size(); ++i) { + write_int_input_at(*encoder_, "input_ids", i, static_cast(context_tokens_[i])); + write_int_input_at(*encoder_, "attention_mask", i, 1); + } + encoder_->graph->execute(); + for (size_t i = 0; i < encoder_->output_node_ids.size() && i < encoder_->logical_outputs.size(); ++i) { + const std::string& out_name = encoder_->logical_outputs[i]; + int dst_idx = input_index(*decoder_, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(encoder_->output_node_ids[i]); + const auto& src_desc = encoder_->graph->get_output_buffer(src_node); + void* src_ptr = encoder_->graph->get_output(src_node); + std::memcpy(decoder_->input_buffers[dst_idx].data(), src_ptr, src_desc.byte_size); + } + last_logit_position_ = context_tokens_.empty() ? 0 : context_tokens_.size() - 1; decoder_->graph->execute(); } @@ -432,6 +971,9 @@ void Model::run_media_step(size_t position, const uint8_t* feature_row, size_t f void Model::run_vision_encoder(const std::string& image_path) { if (!vision_encoder_) return; + if (!load_component_graph(*vision_encoder_)) { + throw std::runtime_error("failed to load vision_encoder"); + } Gemma4ImagePreprocessed prep = preprocess_gemma4_image(image_path, config_); write_bytes_input(*vision_encoder_, "pixel_values", prep.pixel_values.data(), prep.pixel_values.size() * sizeof(float)); @@ -449,84 +991,141 @@ void Model::run_vision_encoder(const std::string& image_path) { media_feature_shapes_[name] = desc.shape; media_feature_precisions_[name] = desc.precision; } + vision_encoder_->graph->release_runtime_buffers(); + vision_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*vision_encoder_); +} + +void Model::run_audio_encoder_messages(const std::vector>& audio_features_per_message) { + if (!audio_encoder_) return; + if (audio_features_per_message.empty()) return; + if (!load_component_graph(*audio_encoder_)) { + throw std::runtime_error("failed to load audio_encoder"); + } + for (const std::string& logical : audio_encoder_->logical_outputs) { + media_features_.erase(logical); + media_feature_shapes_.erase(logical); + media_feature_precisions_.erase(logical); + } + for (const auto& mel : audio_features_per_message) { + if (mel.empty()) continue; + run_audio_encoder(mel); + } + audio_encoder_->graph->release_runtime_buffers(); + audio_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*audio_encoder_); } void Model::run_audio_encoder(const std::vector& audio_features) { if (!audio_encoder_) return; - std::vector candidate_input_names = {"input_features", "audio_features"}; - bool wrote = false; + const std::vector candidate_input_names = {"input_features", "audio_features"}; + int feature_idx = -1; for (const auto& name : candidate_input_names) { int idx = input_index(*audio_encoder_, name); - if (idx < 0) continue; - size_t node_id = static_cast(audio_encoder_->runtime_input_node_ids[idx]); - const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); - auto& buf = audio_encoder_->input_buffers[idx]; - if (desc.precision == Precision::FP32) { - size_t n = std::min(audio_features.size() * sizeof(float), buf.size()); - std::memcpy(buf.data(), audio_features.data(), n); - if (n < buf.size()) std::memset(buf.data() + n, 0, buf.size() - n); - } else if (desc.precision == Precision::FP16) { - size_t n_elems = std::min(audio_features.size(), buf.size() / sizeof(__fp16)); + if (idx >= 0) { feature_idx = idx; break; } + } + if (feature_idx < 0) { + CACTUS_LOG_WARN("model", "audio_encoder has no input named input_features/audio_features; skipping"); + return; + } + const size_t feature_node = static_cast(audio_encoder_->runtime_input_node_ids[feature_idx]); + const auto& feature_desc = audio_encoder_->graph->get_output_buffer(feature_node); + const size_t mel_bins = feature_desc.shape.size() >= 3 + ? feature_desc.shape[2] + : static_cast(config_.audio_input_feat_size); + const size_t max_frames_per_chunk = feature_desc.shape.size() >= 2 ? feature_desc.shape[1] : 0; + if (max_frames_per_chunk == 0 || mel_bins == 0) { + CACTUS_LOG_WARN("model", "audio_encoder feature input has unexpected shape; skipping"); + return; + } + const size_t total_frames = audio_features.size() / mel_bins; + const size_t num_chunks = total_frames == 0 + ? 1 + : (total_frames + max_frames_per_chunk - 1) / max_frames_per_chunk; + + const int mask_idx = input_index(*audio_encoder_, "input_features_mask"); + + for (size_t chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) { + const size_t frame_start = chunk_idx * max_frames_per_chunk; + const size_t frames_in_chunk = frame_start >= total_frames + ? 0 + : std::min(max_frames_per_chunk, total_frames - frame_start); + const size_t chunk_feature_elems = frames_in_chunk * mel_bins; + const float* chunk_src = audio_features.data() + frame_start * mel_bins; + + auto& buf = audio_encoder_->input_buffers[feature_idx]; + if (feature_desc.precision == Precision::FP32) { + const size_t n_bytes = std::min(chunk_feature_elems * sizeof(float), buf.size()); + std::memcpy(buf.data(), chunk_src, n_bytes); + if (n_bytes < buf.size()) std::memset(buf.data() + n_bytes, 0, buf.size() - n_bytes); + } else if (feature_desc.precision == Precision::FP16) { + const size_t cap_elems = buf.size() / sizeof(__fp16); + const size_t n_elems = std::min(chunk_feature_elems, cap_elems); __fp16* dst = reinterpret_cast<__fp16*>(buf.data()); - for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast<__fp16>(audio_features[i]); - if (n_elems * sizeof(__fp16) < buf.size()) { - std::memset(buf.data() + n_elems * sizeof(__fp16), 0, buf.size() - n_elems * sizeof(__fp16)); + for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast<__fp16>(chunk_src[i]); + if (n_elems < cap_elems) { + std::memset(buf.data() + n_elems * sizeof(__fp16), 0, (cap_elems - n_elems) * sizeof(__fp16)); } } else { - size_t n_elems = std::min(audio_features.size(), buf.size()); + const size_t n_elems = std::min(chunk_feature_elems, buf.size()); int8_t* dst = reinterpret_cast(buf.data()); - for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast(audio_features[i]); + for (size_t i = 0; i < n_elems; ++i) dst[i] = static_cast(chunk_src[i]); if (n_elems < buf.size()) std::memset(buf.data() + n_elems, 0, buf.size() - n_elems); } - wrote = true; - break; - } - if (!wrote) { - CACTUS_LOG_WARN("model", "audio_encoder has no input named input_features/audio_features; skipping"); - return; - } - int mask_idx = input_index(*audio_encoder_, "input_features_mask"); - if (mask_idx >= 0) { - auto& mb = audio_encoder_->input_buffers[mask_idx]; - size_t mask_node = static_cast(audio_encoder_->runtime_input_node_ids[mask_idx]); - const auto& mask_desc = audio_encoder_->graph->get_output_buffer(mask_node); - size_t elem = PrecisionTraits::size_of(mask_desc.precision); - size_t total = elem ? mb.size() / elem : mb.size(); - std::memset(mb.data(), 0, mb.size()); - for (size_t i = 0; i < total; ++i) { - switch (mask_desc.precision) { - case Precision::FP32: reinterpret_cast(mb.data())[i] = 1.0f; break; - case Precision::FP16: reinterpret_cast<__fp16*>(mb.data())[i] = static_cast<__fp16>(1.0f); break; - case Precision::INT8: reinterpret_cast(mb.data())[i] = 1; break; - default: - if (elem == 8) reinterpret_cast(mb.data())[i] = 1; - else if (elem == 4) reinterpret_cast(mb.data())[i] = 1; - else mb[i] = 1; - break; + + if (mask_idx >= 0) { + auto& mb = audio_encoder_->input_buffers[mask_idx]; + const size_t mask_node = static_cast(audio_encoder_->runtime_input_node_ids[mask_idx]); + const auto& mask_desc = audio_encoder_->graph->get_output_buffer(mask_node); + const size_t elem = PrecisionTraits::size_of(mask_desc.precision); + const size_t cap = elem ? mb.size() / elem : 0; + const size_t n = std::min(cap, frames_in_chunk); + for (size_t i = 0; i < n; ++i) { + switch (mask_desc.precision) { + case Precision::FP32: reinterpret_cast(mb.data())[i] = 1.0f; break; + case Precision::FP16: reinterpret_cast<__fp16*>(mb.data())[i] = static_cast<__fp16>(1.0f); break; + case Precision::INT8: reinterpret_cast(mb.data())[i] = 1; break; + default: reinterpret_cast(mb.data())[i] = 1; break; + } } + if (n < cap) std::memset(mb.data() + n * elem, 0, (cap - n) * elem); + } + + audio_encoder_->graph->execute(); + + for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { + const std::string& name = audio_encoder_->logical_outputs[i]; + const size_t node_id = static_cast(audio_encoder_->output_node_ids[i]); + const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); + void* ptr = audio_encoder_->graph->get_output(node_id); + auto& slot = media_features_[name]; + const size_t prev_bytes = slot.size(); + slot.resize(prev_bytes + desc.byte_size); + std::memcpy(slot.data() + prev_bytes, ptr, desc.byte_size); + auto shape_it = media_feature_shapes_.find(name); + if (shape_it == media_feature_shapes_.end() || shape_it->second.empty()) { + media_feature_shapes_[name] = desc.shape; + } else if (desc.shape.size() >= 2 && shape_it->second.size() == desc.shape.size()) { + shape_it->second[shape_it->second.size() - 2] += desc.shape[desc.shape.size() - 2]; + } + media_feature_precisions_[name] = desc.precision; } - } - audio_encoder_->graph->execute(); - for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { - const std::string& name = audio_encoder_->logical_outputs[i]; - size_t node_id = static_cast(audio_encoder_->output_node_ids[i]); - const auto& desc = audio_encoder_->graph->get_output_buffer(node_id); - void* ptr = audio_encoder_->graph->get_output(node_id); - auto& slot = media_features_[name]; - slot.assign(desc.byte_size, 0); - std::memcpy(slot.data(), ptr, desc.byte_size); - media_feature_shapes_[name] = desc.shape; - media_feature_precisions_[name] = desc.precision; } } -uint32_t Model::argmax_last_logits() { - size_t out_node = static_cast(decoder_->output_node_ids.empty() ? 0 : decoder_->output_node_ids[0]); - const auto& desc = decoder_->graph->get_output_buffer(out_node); - void* ptr = decoder_->graph->get_output(out_node); +uint32_t Model::argmax_component_logits(Component& comp, size_t logit_row) { + size_t out_node = static_cast(comp.output_node_ids.empty() ? 0 : comp.output_node_ids[0]); + const auto& desc = comp.graph->get_output_buffer(out_node); + void* ptr = comp.graph->get_output(out_node); size_t vocab = desc.shape.empty() ? 0 : desc.shape.back(); size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; - size_t row_off = (seq > 0 ? (seq - 1) * vocab : 0); + size_t row = seq > 0 ? seq - 1 : 0; + if (logit_row != std::numeric_limits::max()) { + row = std::min(logit_row, seq > 0 ? seq - 1 : 0); + } else if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + row = std::min(last_logit_position_, seq > 0 ? seq - 1 : 0); + } + size_t row_off = row * vocab; uint32_t best = 0; float best_v = -std::numeric_limits::infinity(); if (desc.precision == Precision::FP32) { @@ -545,11 +1144,83 @@ uint32_t Model::argmax_last_logits() { return best; } -void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& /*profile_file*/) { - for (size_t i = 0; i < tokens.size(); ++i) { - run_step(tokens[i], cache_total_seq_len_ + i, /*read_logits=*/false); +uint32_t Model::argmax_last_logits() { + return argmax_component_logits(*decoder_); +} + +bool Model::prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token) { + last_prefill_cache_copy_ms_ = 0.0; + last_prefill_padding_tokens_ = 0; + last_prefill_scalar_tail_tokens_ = 0; + if (tokens.empty() || !decoder_ || cache_total_seq_len_ != 0) { + return false; } - cache_total_seq_len_ += tokens.size(); + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + out_token = argmax_last_logits(); + record_sampled_token(out_token); + return true; + } + if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { + for (size_t i = 0; i < tokens.size(); ++i) { + run_step(tokens[i], i, i + 1 == tokens.size()); + } + cache_total_seq_len_ = tokens.size(); + out_token = argmax_last_logits(); + record_sampled_token(out_token); + return true; + } + if (!encoder_) { + return false; + } + ChunkedPrefillResult chunked; + if (decoder_prefill_) { + chunked = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), true); + if (chunked.logical_tokens == tokens.size() && chunked.padding_tokens > 0 && tokens.size() > 0) { + copy_cache_states(*decoder_prefill_, *decoder_, tokens.size() - 1); + cache_total_seq_len_ = tokens.size() - 1; + run_step(tokens.back(), cache_total_seq_len_, true); + ++cache_total_seq_len_; + out_token = argmax_last_logits(); + record_sampled_token(out_token); + last_prefill_scalar_tail_tokens_ = 1; + return true; + } + cache_total_seq_len_ += chunked.logical_tokens; + } + for (size_t i = chunked.logical_tokens; i < tokens.size(); ++i) { + run_step(tokens[i], cache_total_seq_len_, i + 1 == tokens.size()); + ++cache_total_seq_len_; + } + last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; + if (chunked.logical_tokens == tokens.size() && chunked.logical_tokens > 0 && decoder_prefill_) { + out_token = argmax_component_logits(*decoder_prefill_, chunked.last_logit_row); + } else { + out_token = argmax_last_logits(); + } + record_sampled_token(out_token); + return true; +} + +void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& /*profile_file*/, bool prepare_decode) { + last_prefill_cache_copy_ms_ = 0.0; + last_prefill_padding_tokens_ = 0; + last_prefill_scalar_tail_tokens_ = 0; + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + if (!context_tokens_.empty()) run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + return; + } + ChunkedPrefillResult chunked = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), prepare_decode); + cache_total_seq_len_ += chunked.logical_tokens; + for (size_t i = chunked.logical_tokens; i < tokens.size(); ++i) { + run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false); + ++cache_total_seq_len_; + } + last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; } void Model::prefill_with_images(const std::vector& tokens, @@ -559,9 +1230,9 @@ void Model::prefill_with_images(const std::vector& tokens, } void Model::prefill_with_audio(const std::vector& tokens, - const std::vector& audio_features, + const std::vector>& audio_features_per_message, const std::string& profile_file) { - prefill_with_media(tokens, {}, audio_features, profile_file); + prefill_with_media(tokens, {}, audio_features_per_message, profile_file); } namespace { @@ -632,80 +1303,123 @@ void write_tokens_buffer(std::vector& buf, Precision prec, } } -} // namespace - -void Model::copy_cache_state(const Component& src, Component& dst) { - if (src.cache_states.empty() || dst.cache_states.empty()) return; - if (src.cache_states.size() != dst.cache_states.size()) { - throw std::runtime_error("cache state count mismatch between " + src.name + " and " + dst.name); - } - for (size_t i = 0; i < src.cache_states.size(); ++i) { - const auto& s = src.cache_states[i]; - const auto& d = dst.cache_states[i]; - if (s.layer_key != d.layer_key) { - throw std::runtime_error("cache layer mismatch: " + s.layer_key + " vs " + d.layer_key); - } - for (auto pair : {std::pair{s.key_node_id, d.key_node_id}, - std::pair{s.value_node_id, d.value_node_id}}) { - const auto& sd = src.graph->get_output_buffer(static_cast(pair.first)); - const auto& dd = dst.graph->get_output_buffer(static_cast(pair.second)); - if (sd.byte_size != dd.byte_size) { - throw std::runtime_error("cache byte size mismatch for layer " + s.layer_key); - } - std::memcpy(dst.graph->get_output(static_cast(pair.second)), - src.graph->get_output(static_cast(pair.first)), - sd.byte_size); +void write_int_vector_buffer(std::vector& buf, Precision prec, const std::vector& values) { + const size_t elem = PrecisionTraits::size_of(prec); + const size_t cap = elem ? buf.size() / elem : 0; + const size_t n = std::min(cap, values.size()); + for (size_t i = 0; i < n; ++i) { + int64_t v = values[i]; + switch (prec) { + case Precision::FP32: reinterpret_cast(buf.data())[i] = static_cast(v); break; + case Precision::FP16: reinterpret_cast<__fp16*>(buf.data())[i] = static_cast<__fp16>(v); break; + case Precision::INT8: reinterpret_cast(buf.data())[i] = static_cast(v); break; + default: + if (elem == 8) reinterpret_cast(buf.data())[i] = v; + else if (elem == 4) reinterpret_cast(buf.data())[i] = static_cast(v); + break; } } + if (n < cap) { + std::memset(buf.data() + n * elem, 0, (cap - n) * elem); + } } -void Model::reset_component_cache_states(Component& comp) { - for (const auto& state : comp.cache_states) { - for (int node_id : {state.key_node_id, state.value_node_id}) { - if (node_id < 0) continue; - const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); - if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue; - void* ptr = comp.graph->get_output(static_cast(node_id)); - if (!ptr) continue; - auto* metadata = static_cast(ptr); - metadata[0] = 0; +std::vector qwen3_vl_position_ids(const std::vector& tokens, + size_t capacity, + const std::vector& images, + uint32_t image_token_id) { + std::vector positions(3 * capacity, 0); + size_t token_index = 0; + size_t image_index = 0; + int64_t current_pos = 0; + while (token_index < tokens.size() && token_index < capacity) { + if (image_token_id != 0 && tokens[token_index] == image_token_id) { + if (image_index >= images.size()) { + throw std::runtime_error("Qwen3-VL prompt contains more image token groups than image inputs"); + } + const auto& image = images[image_index++]; + const size_t merge_size = 2; + const size_t grid_t = image.grid_t; + const size_t llm_grid_h = image.grid_h / merge_size; + const size_t llm_grid_w = image.grid_w / merge_size; + const size_t image_seq = grid_t * llm_grid_h * llm_grid_w; + size_t count = 0; + while (token_index + count < tokens.size() + && token_index + count < capacity + && tokens[token_index + count] == image_token_id) { + ++count; + } + if (count != image_seq) { + throw std::runtime_error("Qwen3-VL image token count does not match vision feature grid"); + } + size_t local = 0; + for (size_t t = 0; t < grid_t; ++t) { + (void)t; + for (size_t h = 0; h < llm_grid_h; ++h) { + for (size_t w = 0; w < llm_grid_w; ++w) { + size_t pos = token_index + local++; + positions[pos] = current_pos; + positions[capacity + pos] = current_pos + static_cast(h); + positions[2 * capacity + pos] = current_pos + static_cast(w); + } + } + } + current_pos += static_cast(std::max(image.grid_h, image.grid_w) / merge_size); + token_index += count; + continue; + } + + size_t text_count = 0; + while (token_index + text_count < tokens.size() + && token_index + text_count < capacity + && (image_token_id == 0 || tokens[token_index + text_count] != image_token_id)) { + size_t pos = token_index + text_count; + int64_t value = current_pos + static_cast(text_count); + positions[pos] = value; + positions[capacity + pos] = value; + positions[2 * capacity + pos] = value; + ++text_count; } + current_pos += static_cast(text_count); + token_index += text_count; } + return positions; } +} // namespace + bool Model::build_lm_encoder_outputs_dynamic_gemma4( const std::vector& tokens, std::map>& store_bytes, std::map& store_prec, std::map>& store_shape) { - if (!encoder_ || !lm_encoder_media_step_) return false; - if (tokens.empty()) return false; + if (!encoder_ || !lm_encoder_media_step_ || tokens.empty()) return false; const uint32_t image_tok = config_.image_token_id; const uint32_t audio_tok = config_.audio_token_id; - auto af_it = media_features_.find("audio_features"); - const bool have_af = (af_it != media_features_.end() && !af_it->second.empty()); - size_t af_count = 0; - size_t af_row_bytes = 0; - Precision af_prec = Precision::FP16; - if (have_af) { - const auto& sh = media_feature_shapes_["audio_features"]; - af_count = sh.size() >= 2 ? sh[sh.size() - 2] : 0; - af_prec = media_feature_precisions_["audio_features"]; - af_row_bytes = af_count > 0 ? af_it->second.size() / af_count : af_it->second.size(); - } - - auto im_it = media_features_.find("image_features"); - const bool have_imf = (im_it != media_features_.end() && !im_it->second.empty()); - size_t imf_count = 0; - size_t imf_row_bytes = 0; - Precision imf_prec = Precision::FP16; - if (have_imf) { - const auto& sh = media_feature_shapes_["image_features"]; - imf_count = sh.size() >= 2 ? sh[sh.size() - 2] : 0; - imf_prec = media_feature_precisions_["image_features"]; - imf_row_bytes = imf_count > 0 ? im_it->second.size() / imf_count : im_it->second.size(); + auto audio_it = media_features_.find("audio_features"); + const bool have_audio_features = audio_it != media_features_.end() && !audio_it->second.empty(); + size_t audio_rows = 0; + size_t audio_row_bytes = 0; + Precision audio_prec = Precision::FP16; + if (have_audio_features) { + const auto& shape = media_feature_shapes_["audio_features"]; + audio_rows = shape.size() >= 2 ? shape[shape.size() - 2] : 0; + audio_prec = media_feature_precisions_["audio_features"]; + audio_row_bytes = audio_rows > 0 ? audio_it->second.size() / audio_rows : audio_it->second.size(); + } + + auto image_it = media_features_.find("image_features"); + const bool have_image_features = image_it != media_features_.end() && !image_it->second.empty(); + size_t image_rows = 0; + size_t image_row_bytes = 0; + Precision image_prec = Precision::FP16; + if (have_image_features) { + const auto& shape = media_feature_shapes_["image_features"]; + image_rows = shape.size() >= 2 ? shape[shape.size() - 2] : 0; + image_prec = media_feature_precisions_["image_features"]; + image_row_bytes = image_rows > 0 ? image_it->second.size() / image_rows : image_it->second.size(); } struct OutputInfo { @@ -716,97 +1430,121 @@ bool Model::build_lm_encoder_outputs_dynamic_gemma4( Precision precision = Precision::FP16; std::vector shape_template; }; - std::vector outs; + + std::vector outputs; for (size_t i = 0; i < encoder_->logical_outputs.size() && i < encoder_->output_node_ids.size(); ++i) { - OutputInfo oi; - oi.name = encoder_->logical_outputs[i]; - oi.text_idx = static_cast(i); - size_t node = static_cast(encoder_->output_node_ids[i]); - const auto& desc = encoder_->graph->get_output_buffer(node); - oi.per_token_bytes = desc.byte_size; - oi.precision = desc.precision; - oi.shape_template = desc.shape; - oi.media_idx = output_index(*lm_encoder_media_step_, oi.name); - if (oi.media_idx < 0) { - CACTUS_LOG_WARN("model", "dynamic walker: lm_encoder_media_step missing output '" << oi.name << "'"); - return false; + OutputInfo info; + info.name = encoder_->logical_outputs[i]; + info.text_idx = static_cast(i); + info.media_idx = output_index(*lm_encoder_media_step_, info.name); + if (info.media_idx < 0) { + throw std::runtime_error("lm_encoder_media_step missing output " + info.name); } - outs.push_back(std::move(oi)); - } - - const size_t N = tokens.size(); - for (auto& oi : outs) { - store_bytes[oi.name].assign(N * oi.per_token_bytes, 0); - store_prec[oi.name] = oi.precision; - std::vector sh = oi.shape_template; - if (sh.size() >= 2 && sh[sh.size() - 2] == 1) { - sh[sh.size() - 2] = N; - } else if (sh.size() == 1) { - sh[0] = N; + size_t node_id = static_cast(encoder_->output_node_ids[i]); + const auto& desc = encoder_->graph->get_output_buffer(node_id); + info.per_token_bytes = desc.byte_size; + info.precision = desc.precision; + info.shape_template = desc.shape; + outputs.push_back(std::move(info)); + } + if (outputs.empty()) return false; + + const size_t token_count = tokens.size(); + for (const auto& info : outputs) { + store_bytes[info.name].assign(token_count * info.per_token_bytes, 0); + store_prec[info.name] = info.precision; + std::vector shape = info.shape_template; + if (shape.size() >= 2 && shape[shape.size() - 2] == 1) { + shape[shape.size() - 2] = token_count; + } else if (shape.size() == 1) { + shape[0] = token_count; } - store_shape[oi.name] = sh; + store_shape[info.name] = std::move(shape); } size_t audio_idx = 0; size_t image_idx = 0; - for (size_t pos = 0; pos < N; ++pos) { - const uint32_t tk = tokens[pos]; - Component* enc; + for (size_t pos = 0; pos < token_count; ++pos) { + const uint32_t token = tokens[pos]; + Component* component = encoder_; const uint8_t* media_row = nullptr; size_t media_row_bytes = 0; Precision media_prec = Precision::FP16; - if (have_af && audio_tok != 0 && tk == audio_tok && audio_idx < af_count) { - enc = lm_encoder_media_step_; - media_row = af_it->second.data() + audio_idx * af_row_bytes; - media_row_bytes = af_row_bytes; - media_prec = af_prec; + + if (audio_tok != 0 && token == audio_tok && have_audio_features) { + if (audio_idx >= audio_rows) { + throw std::runtime_error("Gemma4 prompt contains more audio tokens than audio feature rows"); + } + component = lm_encoder_media_step_; + media_row = audio_it->second.data() + audio_idx * audio_row_bytes; + media_row_bytes = audio_row_bytes; + media_prec = audio_prec; ++audio_idx; - } else if (have_imf && image_tok != 0 && tk == image_tok && image_idx < imf_count) { - enc = lm_encoder_media_step_; - media_row = im_it->second.data() + image_idx * imf_row_bytes; - media_row_bytes = imf_row_bytes; - media_prec = imf_prec; + } else if (image_tok != 0 && token == image_tok && have_image_features) { + if (image_idx >= image_rows) { + throw std::runtime_error("Gemma4 prompt contains more image tokens than image feature rows"); + } + component = lm_encoder_media_step_; + media_row = image_it->second.data() + image_idx * image_row_bytes; + media_row_bytes = image_row_bytes; + media_prec = image_prec; ++image_idx; - } else { - enc = encoder_; } - if (enc == lm_encoder_media_step_) { - int e_idx = input_index(*enc, "inputs_embeds"); - if (e_idx >= 0) { - auto& buf = enc->input_buffers[e_idx]; - size_t node_id = static_cast(enc->runtime_input_node_ids[e_idx]); - const auto& desc = enc->graph->get_output_buffer(node_id); - write_typed_buffer(buf, desc.precision, media_row, media_row_bytes, media_prec); + if (component == lm_encoder_media_step_) { + int embeds_idx = input_index(*component, "inputs_embeds"); + if (embeds_idx < 0) { + throw std::runtime_error("lm_encoder_media_step missing inputs_embeds input"); } - write_int_input(*enc, "input_ids", 0); - write_int_input(*enc, "position_ids", static_cast(pos)); + auto& buf = component->input_buffers[embeds_idx]; + size_t node_id = static_cast(component->runtime_input_node_ids[embeds_idx]); + const auto& desc = component->graph->get_output_buffer(node_id); + write_typed_buffer(buf, desc.precision, media_row, media_row_bytes, media_prec); + write_int_input(*component, "input_ids", 0); + write_int_input(*component, "position_ids", static_cast(pos)); } else { - write_int_input(*enc, "input_ids", static_cast(tk)); - write_int_input(*enc, "position_ids", static_cast(pos)); + write_int_input(*component, "input_ids", static_cast(token)); + write_int_input(*component, "position_ids", static_cast(pos)); } - enc->graph->execute(); - - for (auto& oi : outs) { - int out_i = (enc == encoder_) ? oi.text_idx : oi.media_idx; - if (out_i < 0) continue; - size_t node = static_cast(enc->output_node_ids[out_i]); - const auto& desc = enc->graph->get_output_buffer(node); - if (desc.byte_size != oi.per_token_bytes) continue; - const void* ptr = enc->graph->get_output(node); - std::memcpy(store_bytes[oi.name].data() + pos * oi.per_token_bytes, ptr, oi.per_token_bytes); + component->graph->execute(); + + for (const auto& info : outputs) { + int out_idx = component == encoder_ ? info.text_idx : info.media_idx; + size_t node_id = static_cast(component->output_node_ids[out_idx]); + const auto& desc = component->graph->get_output_buffer(node_id); + if (desc.byte_size != info.per_token_bytes || desc.precision != info.precision) { + throw std::runtime_error("Gemma4 dynamic output shape mismatch for " + info.name); + } + const void* ptr = component->graph->get_output(node_id); + std::memcpy(store_bytes[info.name].data() + pos * info.per_token_bytes, ptr, info.per_token_bytes); } + component->graph->release_runtime_buffers(); } + encoder_->graph->release_all_weight_pages(); + lm_encoder_media_step_->graph->release_all_weight_pages(); return true; } + bool Model::run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features) { + const std::vector>& audio_features_per_message) { + if (cache_total_seq_len_ > 0) return false; const bool have_images = !image_paths.empty() && vision_encoder_ != nullptr; - const bool have_audio = !audio_features.empty() && audio_encoder_ != nullptr; + bool any_audio = false; + for (const auto& mel : audio_features_per_message) { if (!mel.empty()) { any_audio = true; break; } } + const bool have_audio = any_audio && audio_encoder_ != nullptr; + std::vector qwen_images; if (have_images) { + if (!load_component_graph(*vision_encoder_)) { + throw std::runtime_error("failed to load vision_encoder"); + } + for (const std::string& logical : vision_encoder_->logical_outputs) { + media_features_.erase(logical); + media_feature_shapes_.erase(logical); + media_feature_precisions_.erase(logical); + } for (const auto& path : image_paths) { if (family_ == "lfm2_vl") { Lfm2VlImagePreprocessed prep = preprocess_lfm2_vl_image(path, config_); @@ -839,6 +1577,20 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, } if (n < cap) std::memset(pm_buf.data() + n * elem, 0, (cap - n) * elem); } + } else if (family_ == "qwen3_5" || family_ == "qwen3_vl" || config_.model_type == Config::ModelType::QWEN) { + Qwen3VlImagePreprocessed prep = preprocess_qwen3_vl_image(path, config_); + int pv_idx = input_index(*vision_encoder_, "pixel_values"); + if (pv_idx < 0) { + throw std::runtime_error("Qwen3-VL vision_encoder missing pixel_values input"); + } + auto& pv_buf = vision_encoder_->input_buffers[pv_idx]; + size_t pv_node = static_cast(vision_encoder_->runtime_input_node_ids[pv_idx]); + const auto& pv_desc = vision_encoder_->graph->get_output_buffer(pv_node); + write_typed_buffer(pv_buf, pv_desc.precision, + prep.pixel_values.data(), + prep.pixel_values.size() * sizeof(float), + Precision::FP32); + qwen_images.push_back(std::move(prep)); } else { Gemma4ImagePreprocessed prep = preprocess_gemma4_image(path, config_); int pv_idx = input_index(*vision_encoder_, "pixel_values"); @@ -882,30 +1634,40 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, const auto& desc = vision_encoder_->graph->get_output_buffer(node_id); void* ptr = vision_encoder_->graph->get_output(node_id); auto& slot = media_features_[name]; - slot.assign(desc.byte_size, 0); - std::memcpy(slot.data(), ptr, desc.byte_size); - media_feature_shapes_[name] = desc.shape; + const size_t prev_bytes = slot.size(); + slot.resize(prev_bytes + desc.byte_size); + std::memcpy(slot.data() + prev_bytes, ptr, desc.byte_size); + auto shape_it = media_feature_shapes_.find(name); + if (shape_it == media_feature_shapes_.end() || shape_it->second.empty()) { + media_feature_shapes_[name] = desc.shape; + } else if (desc.shape.size() >= 2 && shape_it->second.size() == desc.shape.size()) { + shape_it->second[shape_it->second.size() - 2] += desc.shape[desc.shape.size() - 2]; + } media_feature_precisions_[name] = desc.precision; } + vision_encoder_->graph->release_runtime_buffers(); + vision_encoder_->graph->release_all_weight_pages(); } + unload_component_graph(*vision_encoder_); } if (have_audio) { - run_audio_encoder(audio_features); + run_audio_encoder_messages(audio_features_per_message); } std::map> store_bytes; std::map store_prec; std::map> store_shape; - - const bool needs_dynamic_walk = (family_ == "gemma4") && have_audio - && encoder_ != nullptr && lm_encoder_media_step_ != nullptr; + const bool needs_dynamic_walk = family_ == "gemma4" && (have_images || have_audio) && lm_encoder_media_step_ != nullptr; if (needs_dynamic_walk) { if (!build_lm_encoder_outputs_dynamic_gemma4(tokens, store_bytes, store_prec, store_shape)) { return false; } } else { + if (!load_component_graph(*lm_encoder_)) { + throw std::runtime_error("failed to load lm_encoder"); + } int ids_idx = input_index(*lm_encoder_, "input_ids"); if (ids_idx >= 0) { auto& ids_buf = lm_encoder_->input_buffers[ids_idx]; @@ -922,6 +1684,18 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, fill_int_buffer(mb, mdesc.precision, 1, tokens.size()); } + int pos_idx = input_index(*lm_encoder_, "position_ids"); + if (pos_idx >= 0) { + auto& pos_buf = lm_encoder_->input_buffers[pos_idx]; + size_t pos_node = static_cast(lm_encoder_->runtime_input_node_ids[pos_idx]); + const auto& pos_desc = lm_encoder_->graph->get_output_buffer(pos_node); + if (pos_desc.shape.size() >= 3 && pos_desc.shape[0] == 3 && !qwen_images.empty()) { + size_t capacity = pos_desc.shape[pos_desc.shape.size() - 1]; + auto positions = qwen3_vl_position_ids(tokens, capacity, qwen_images, config_.image_token_id); + write_int_vector_buffer(pos_buf, pos_desc.precision, positions); + } + } + for (const auto& kv : media_features_) { const std::string& name = kv.first; int idx = input_index(*lm_encoder_, name); @@ -947,6 +1721,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, store_prec[name] = desc.precision; store_shape[name] = desc.shape; } + lm_encoder_->graph->release_runtime_buffers(); + lm_encoder_->graph->release_all_weight_pages(); + unload_component_graph(*lm_encoder_); } auto embeds_shape_it = store_shape.find("inputs_embeds"); @@ -963,6 +1740,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, size_t chunk_seq = 0; { + if (!load_component_graph(*decoder_prefill_chunk_)) { + throw std::runtime_error("failed to load decoder_prefill_chunk"); + } int idx = input_index(*decoder_prefill_chunk_, "inputs_embeds"); if (idx < 0) return false; size_t node_id = static_cast(decoder_prefill_chunk_->runtime_input_node_ids[idx]); @@ -1024,7 +1804,9 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, decoder_prefill_chunk_->graph->execute(); } if (whole_chunks_end > 0 && decoder_ != nullptr) { - copy_cache_state(*decoder_prefill_chunk_, *decoder_); + copy_cache_states(*decoder_prefill_chunk_, *decoder_); + decoder_prefill_chunk_->graph->release_runtime_buffers(); + unload_component_graph(*decoder_prefill_chunk_); } for (size_t pos = whole_chunks_end; pos < valid_seq; ++pos) { for (const auto& kv : store_bytes) { @@ -1047,11 +1829,19 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, void Model::prefill_with_media(const std::vector& tokens, const std::vector& image_paths, - const std::vector& audio_features, + const std::vector>& audio_features_per_message, const std::string& profile_file) { if (tokens.empty()) return; - const bool have_images = !image_paths.empty() && vision_encoder_ != nullptr; - const bool have_audio = !audio_features.empty() && audio_encoder_ != nullptr; + if (!image_paths.empty() && vision_encoder_ == nullptr) { + throw std::runtime_error("Model bundle does not include a vision_encoder for image input"); + } + bool any_audio = false; + for (const auto& mel : audio_features_per_message) { if (!mel.empty()) { any_audio = true; break; } } + if (any_audio && audio_encoder_ == nullptr) { + throw std::runtime_error("Model bundle does not include an audio_encoder for audio input"); + } + const bool have_images = !image_paths.empty(); + const bool have_audio = any_audio; if (!have_images && !have_audio) { prefill(tokens, get_prefill_chunk_size(), profile_file); return; @@ -1061,7 +1851,7 @@ void Model::prefill_with_media(const std::vector& tokens, lm_encoder_ != nullptr && decoder_prefill_chunk_ != nullptr && (vision_encoder_ != nullptr || audio_encoder_ != nullptr); if (can_chunk_prefill) { - if (run_chunk_prefill_path(tokens, image_paths, audio_features)) { + if (run_chunk_prefill_path(tokens, image_paths, audio_features_per_message)) { (void)profile_file; return; } @@ -1078,7 +1868,7 @@ void Model::prefill_with_media(const std::vector& tokens, } } if (have_audio) { - run_audio_encoder(audio_features); + run_audio_encoder_messages(audio_features_per_message); } std::string image_feature_name; @@ -1167,6 +1957,15 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* size_t /*top_k*/, const std::string& /*profile_file*/, float* out_entropy, float /*min_p*/, float /*repetition_penalty*/) { if (tokens.empty()) return 0; + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + if (out_entropy) *out_entropy = 0.0f; + uint32_t result = argmax_last_logits(); + record_sampled_token(result); + return result; + } for (size_t i = 0; i + 1 < tokens.size(); ++i) { run_step(tokens[i], cache_total_seq_len_ + i, /*read_logits=*/false); } @@ -1178,7 +1977,8 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* return result; } -uint32_t Model::decode_with_audio(const std::vector& tokens, const std::vector& /*mel_bins*/, +uint32_t Model::decode_with_audio(const std::vector& tokens, + const std::vector>& /*audio_features_per_message*/, float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, float min_p, float repetition_penalty, float* /*out_token_time_start*/, float* /*out_token_time_end*/) { @@ -1502,6 +2302,8 @@ std::vector Model::get_audio_embeddings(const std::vector& /*mel_b void Model::reset_cache() { cache_total_seq_len_ = 0; + last_logit_position_ = 0; + context_tokens_.clear(); token_history_.clear(); media_features_.clear(); media_feature_shapes_.clear(); @@ -1509,16 +2311,7 @@ void Model::reset_cache() { for (auto& kv : components_) { Component& comp = kv.second; if (!comp.graph) continue; - for (const auto& cs : comp.cache_states) { - for (int node_id : {cs.key_node_id, cs.value_node_id}) { - if (node_id < 0) continue; - const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); - if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue; - void* data = comp.graph->get_output(static_cast(node_id)); - if (!data) continue; - *static_cast(data) = 0; - } - } + reset_component_cache_states(comp); } } diff --git a/cactus-engine/src/transcribe.cpp b/cactus-engine/src/transcribe.cpp index 1cb781940..ae91ad236 100644 --- a/cactus-engine/src/transcribe.cpp +++ b/cactus-engine/src/transcribe.cpp @@ -315,7 +315,7 @@ int cactus_transcribe( float token_entropy = 0.0f; uint32_t next_token = handle->model->decode_with_audio( - tokens, audio_features, + tokens, std::vector>{audio_features}, options.temperature, options.top_p, options.top_k, "", &token_entropy, options.min_p, options.repetition_penalty, diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 1e18fa899..2648274c2 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -415,16 +415,6 @@ inline AudioPreprocessResult preprocess_audio_for_gemma4( AudioPreprocessResult result; if (audio_samples.empty()) return result; - size_t max_soft = model_config.audio_soft_tokens; - if (max_soft > 0) { - size_t max_frames = (max_soft * 2 - 1) * 2 - 1; - size_t hop_length = 160; - size_t max_samples = max_frames * hop_length; - if (audio_samples.size() > max_samples) { - audio_samples.resize(max_samples); - } - } - size_t pad_amt = 320 - (audio_samples.size() % 320); if (pad_amt < 320) audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); @@ -441,8 +431,15 @@ inline AudioPreprocessResult preprocess_audio_for_gemma4( result.num_frames = mel.size() / mel_bins; result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); - size_t after_stage1 = (result.num_frames + 1) / 2; - result.num_soft_tokens = (after_stage1 + 1) / 2; + size_t max_soft = model_config.audio_soft_tokens; + size_t max_frames_per_chunk = max_soft > 0 ? (max_soft * 2 - 1) * 2 - 1 : 0; + if (max_soft > 0 && result.num_frames > max_frames_per_chunk) { + size_t num_chunks = (result.num_frames + max_frames_per_chunk - 1) / max_frames_per_chunk; + result.num_soft_tokens = num_chunks * max_soft; + } else { + size_t after_stage1 = (result.num_frames + 1) / 2; + result.num_soft_tokens = (after_stage1 + 1) / 2; + } return result; } diff --git a/cactus-engine/tests/asr.cpp b/cactus-engine/tests/asr.cpp index 494a62e1d..7cc10455a 100644 --- a/cactus-engine/tests/asr.cpp +++ b/cactus-engine/tests/asr.cpp @@ -52,7 +52,7 @@ int main(int argc, char** argv) { std::cout << "Model loaded.\n"; std::ostringstream opts; - opts << "{\"language\":\"" << language << "\",\"telemetry_enabled\":false}"; + opts << "{\"language\":\"" << language << "\",\"telemetry_enabled\":false,\"auto_handoff\":false,\"max_tokens\":512}"; constexpr size_t kBufSize = 1 << 16; std::string response(kBufSize, '\0'); diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index 9a5eeb1b1..594c75de2 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -283,6 +283,32 @@ bool file_exists(const std::string& path) { return f.good(); } +std::vector parse_token_ids(const std::string& text) { + std::vector tokens; + std::stringstream ss(text); + std::string part; + while (std::getline(ss, part, ',')) { + size_t start = part.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) continue; + size_t end = part.find_last_not_of(" \t\r\n"); + std::string value = part.substr(start, end - start + 1); + char* parsed_end = nullptr; + unsigned long parsed = std::strtoul(value.c_str(), &parsed_end, 10); + if (!parsed_end || *parsed_end != '\0') { + throw std::runtime_error("Invalid token id: " + value); + } + tokens.push_back(static_cast(parsed)); + } + return tokens; +} + +bool write_text_file(const std::string& path, const std::string& text) { + std::ofstream out(path); + if (!out) return false; + out << text; + return out.good(); +} + std::string json_string_value(const std::string& json, const std::string& key) { std::string needle = "\"" + key + "\":\""; size_t start = json.find(needle); @@ -338,7 +364,8 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 << " [--system ] [--image ] [--audio ]" - << " [--prompt ] [--thinking]\n"; + << " [--prompt ] [--input-ids ] [--max-new-tokens ]" + << " [--result-json ] [--thinking]\n"; } } // namespace @@ -354,6 +381,9 @@ int main(int argc, char** argv) { std::string current_image; std::string current_audio; std::string initial_prompt; + std::string input_ids; + std::string result_json; + int max_new_tokens = kMaxTokens; bool thinking = false; for (int i = 2; i < argc; ++i) { @@ -366,6 +396,12 @@ int main(int argc, char** argv) { current_audio = expand_tilde(argv[++i]); } else if (arg == "--prompt" && i + 1 < argc) { initial_prompt = argv[++i]; + } else if (arg == "--input-ids" && i + 1 < argc) { + input_ids = argv[++i]; + } else if (arg == "--max-new-tokens" && i + 1 < argc) { + max_new_tokens = std::max(0, std::atoi(argv[++i])); + } else if (arg == "--result-json" && i + 1 < argc) { + result_json = argv[++i]; } else if (arg == "--thinking") { thinking = true; } @@ -387,6 +423,39 @@ int main(int argc, char** argv) { return 1; } + if (!input_ids.empty()) { + std::vector tokens; + try { + tokens = parse_token_ids(input_ids); + } catch (const std::exception& e) { + std::cerr << e.what() << "\n"; + cactus_destroy(model); + return 1; + } + if (tokens.empty()) { + std::cerr << "--input-ids did not contain any token ids\n"; + cactus_destroy(model); + return 1; + } + std::vector response(kResponseBufferSize, 0); + int rc = cactus_benchmark_tokens( + model, + tokens.data(), + tokens.size(), + static_cast(max_new_tokens), + response.data(), + response.size()); + std::string response_json(response.data()); + if (!result_json.empty() && !write_text_file(result_json, response_json)) { + std::cerr << "Failed to write result JSON: " << result_json << "\n"; + cactus_destroy(model); + return 1; + } + std::cout << response_json << "\n"; + cactus_destroy(model); + return rc < 0 ? 1 : 0; + } + std::cout << "Model loaded.\n"; std::cout << "Commands: /image [prompt], /audio [prompt], "; #ifdef HAVE_SDL2 @@ -485,7 +554,7 @@ int main(int argc, char** argv) { history.push_back({"user", input}); std::string messages = build_messages(system_prompt, history, current_image, current_audio, attach_media); std::string options = "{\"temperature\":0.7,\"top_p\":0.95,\"top_k\":40,\"max_tokens\":" - + std::to_string(kMaxTokens) + + std::to_string(max_new_tokens) + ",\"enable_thinking_if_supported\":" + (thinking ? "true" : "false") + ",\"auto_handoff\":false,\"confidence_threshold\":0.0" + ",\"stop_sequences\":[\"<|im_end|>\",\"\"]}"; @@ -512,6 +581,9 @@ int main(int argc, char** argv) { current_pcm.size()); std::string response_json(response.data()); + if (!result_json.empty() && !write_text_file(result_json, response_json)) { + std::cerr << "Failed to write result JSON: " << result_json << "\n"; + } double ram_mb = json_number_value(response_json, "ram_usage_mb"); printer.print_stats(ram_mb); std::cout << "\n"; diff --git a/cactus-engine/tests/test_stt.cpp b/cactus-engine/tests/test_stt.cpp index d22396a6e..55f9b2ca0 100644 --- a/cactus-engine/tests/test_stt.cpp +++ b/cactus-engine/tests/test_stt.cpp @@ -35,7 +35,7 @@ bool test_transcription() { Timer timer; int rc = cactus_transcribe(model, audio_path.c_str(), nullptr, response, sizeof(response), - R"({"max_tokens": 200, "telemetry_enabled": false})", + R"({"max_tokens": 200, "telemetry_enabled": false, "auto_handoff": false})", nullptr, nullptr, nullptr, 0); double elapsed = timer.elapsed_ms(); @@ -105,7 +105,7 @@ bool test_transcription_pcm() { Timer timer; int rc = cactus_transcribe(model, nullptr, nullptr, response, sizeof(response), - R"({"max_tokens": 200, "telemetry_enabled": false})", + R"({"max_tokens": 200, "telemetry_enabled": false, "auto_handoff": false})", nullptr, nullptr, pcm_data.data(), pcm_data.size()); double elapsed = timer.elapsed_ms(); diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index c5373bcbe..02a6eadda 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -299,6 +299,7 @@ struct BufferDesc { void allocate(); void allocate_from_pool(BufferPool& pool); void release_to_pool(BufferPool& pool); + void release_memory(BufferPool& pool); void set_external(void* ptr); }; @@ -676,6 +677,9 @@ class CactusGraph { void release_weight_pages(size_t node_id); void prefetch_weight_pages(size_t node_id); void release_all_weight_pages(); + void release_runtime_buffers(); + void clear_buffer_pool(); + void retain_outputs(const std::vector& node_ids); size_t persistent(size_t source_node); bool is_populated(size_t persistent_node_id) const; @@ -715,6 +719,7 @@ class CactusGraph { std::unordered_set persistent_node_ids_; std::unordered_set populated_node_ids_; std::unordered_set embedded_input_node_ids_; + std::unordered_set retained_output_node_ids_; }; namespace GraphFile { diff --git a/cactus-graph/src/core.cpp b/cactus-graph/src/core.cpp index 691f57b41..78a0a6523 100644 --- a/cactus-graph/src/core.cpp +++ b/cactus-graph/src/core.cpp @@ -160,9 +160,18 @@ void BufferDesc::release_to_pool(BufferPool& pool) { } } +void BufferDesc::release_memory(BufferPool& pool) { + release_to_pool(pool); + data.reset(); + external_data = nullptr; +} + void BufferDesc::set_external(void* ptr) { external_data = ptr; data.reset(); + if (pooled_data) { + delete[] pooled_data; + } pooled_data = nullptr; } @@ -249,3 +258,26 @@ void CactusGraph::allocate_buffers() { } } } + +void CactusGraph::release_runtime_buffers() { + for (auto& node : nodes_) { + if (node->op_type == OpType::INPUT) continue; + if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) continue; + if (persistent_node_ids_.count(node->id)) continue; + node->output_buffer.release_memory(buffer_pool_); + } + buffer_pool_.clear(); + shrink_thread_local_buffers(); +} + +void CactusGraph::clear_buffer_pool() { + buffer_pool_.clear(); +} + +void CactusGraph::retain_outputs(const std::vector& node_ids) { + for (int node_id : node_ids) { + if (node_id >= 0) { + retained_output_node_ids_.insert(static_cast(node_id)); + } + } +} diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 71fdb3449..a7dc86e1f 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -359,6 +359,57 @@ void CactusGraph::execute(const std::string& profile_file) { } }; + auto can_release_node = [&](size_t node_idx) { + const auto& node = nodes_[node_idx]; + if (node->op_type == OpType::INPUT) return false; + if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) return false; + if (persistent_node_ids_.count(node->id)) return false; + if (retained_output_node_ids_.count(node->id)) return false; + return true; + }; + + auto may_alias_input = [](const GraphNode& node) { + return (node.op_type == OpType::SLICE && node.params.axis == 0) || + (node.op_type == OpType::INDEX && node.params.axis == 0); + }; + + auto preallocates_output = [](const GraphNode& node) { + return !((node.op_type == OpType::SLICE && node.params.axis == 0) || + (node.op_type == OpType::INDEX && node.params.axis == 0)); + }; + + std::vector last_use(n, 0); + std::vector use_count(n, 0); + for (size_t i = 0; i < n; ++i) { + for (size_t input_id : nodes_[i]->input_ids) { + auto it = node_index_map_.find(input_id); + if (it != node_index_map_.end()) { + last_use[it->second] = std::max(last_use[it->second], i); + ++use_count[it->second]; + } + } + } + + std::vector keep_until_graph_cleanup(n, false); + for (size_t i = 0; i < n; ++i) { + const auto& node = *nodes_[i]; + if (!may_alias_input(node) || node.input_ids.empty()) continue; + auto it = node_index_map_.find(node.input_ids[0]); + if (it == node_index_map_.end()) continue; + size_t base_idx = it->second; + if (use_count[i] == 0) { + keep_until_graph_cleanup[base_idx] = true; + } else { + last_use[base_idx] = std::max(last_use[base_idx], last_use[i]); + } + } + + std::vector> release_after(n); + for (size_t i = 0; i < n; ++i) { + if (!can_release_node(i) || use_count[i] == 0 || keep_until_graph_cleanup[i]) continue; + release_after[last_use[i]].push_back(i); + } + if (!need_debug) { for (size_t i = 0; i < n; ++i) { auto& node = nodes_[i]; @@ -366,26 +417,24 @@ void CactusGraph::execute(const std::string& profile_file) { if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) { dispatch_node(*node, nodes_, node_index_map_); populated_node_ids_.insert(node->id); + for (size_t release_idx : release_after[i]) { + nodes_[release_idx]->output_buffer.release_memory(pool); + } continue; } - node->output_buffer.allocate_from_pool(pool); + if (preallocates_output(*node)) { + node->output_buffer.allocate_from_pool(pool); + } dispatch_node(*node, nodes_, node_index_map_); trace_nonfinite(i, *node); if (node->op_type == OpType::PERSISTENT) { populated_node_ids_.insert(node->id); } - } - return; - } - - std::vector last_use(n, 0); - for (size_t i = 0; i < n; ++i) { - for (size_t input_id : nodes_[i]->input_ids) { - auto it = node_index_map_.find(input_id); - if (it != node_index_map_.end()) { - last_use[it->second] = std::max(last_use[it->second], i); + for (size_t release_idx : release_after[i]) { + nodes_[release_idx]->output_buffer.release_memory(pool); } } + return; } auto get_env_str = [](const char* name) -> std::string { diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 04fa56207..ad722f7ca 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -146,24 +146,33 @@ void compute_kv_cache_append_node( size_t new_total = current_len + new_seq_len; bool needs_eviction = new_total > window; if (needs_eviction) { - size_t sink_eff = std::min(sink, window); - if (new_seq_len >= window) { - size_t src_skip = new_seq_len - window; - std::memcpy(fp16_base, source + src_skip * stride, window * stride * sizeof(__fp16)); - } else { - size_t remaining = (new_seq_len + sink_eff < window) - ? window - sink_eff - new_seq_len : 0; - size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; - if (remaining > 0 && shift_src > sink_eff) { - std::memmove( - fp16_base + sink_eff * stride, - fp16_base + shift_src * stride, - remaining * stride * sizeof(__fp16)); + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) { + if (tail_capacity > 0) { + size_t source_offset = new_seq_len - tail_capacity; + std::memcpy( + fp16_base + keep_sink * stride, + source + source_offset * stride, + tail_capacity * stride * sizeof(__fp16)); } - size_t append_offset = window - new_seq_len; - std::memcpy(fp16_base + append_offset * stride, source, new_seq_len * stride * sizeof(__fp16)); + meta->current_seq_len = keep_sink + tail_capacity; + *node.output_buffer.data_as() = static_cast(meta->current_seq_len); + return; } - meta->current_seq_len = window; + + size_t remaining = tail_capacity - new_seq_len; + remaining = std::min(remaining, current_len - keep_sink); + size_t shift_src = current_len - remaining; + if (remaining > 0 && shift_src > keep_sink) { + std::memmove( + fp16_base + keep_sink * stride, + fp16_base + shift_src * stride, + remaining * stride * sizeof(__fp16)); + } + size_t append_offset = keep_sink + remaining; + std::memcpy(fp16_base + append_offset * stride, source, new_seq_len * stride * sizeof(__fp16)); + meta->current_seq_len = append_offset + new_seq_len; } else { std::memcpy(fp16_base + current_len * stride, source, new_seq_len * stride * sizeof(__fp16)); meta->current_seq_len = new_total; @@ -183,37 +192,43 @@ void compute_kv_cache_append_node( bool needs_eviction = new_total > window; if (needs_eviction) { - size_t sink_eff = std::min(sink, window); - if (new_seq_len >= window) { - size_t src_skip = new_seq_len - window; - cactus_quantize_kv_fp16_to_int8( - new_kv.data_as<__fp16>() + src_skip * (kv_heads * hdim), - int8_base, - scale_base, - window, kv_heads, hdim); - } else { - size_t remaining = (new_seq_len + sink_eff < window) - ? window - sink_eff - new_seq_len : 0; - size_t shift_src = (current_len > remaining) ? current_len - remaining : 0; - - if (remaining > 0 && shift_src > sink_eff) { - std::memmove(int8_base + sink_eff * int8_stride, - int8_base + shift_src * int8_stride, - remaining * int8_stride); - std::memmove(scale_base + sink_eff * scale_stride, - scale_base + shift_src * scale_stride, - remaining * scale_stride * sizeof(float)); + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) { + if (tail_capacity > 0) { + size_t source_offset = new_seq_len - tail_capacity; + cactus_quantize_kv_fp16_to_int8( + new_kv.data_as<__fp16>() + source_offset * int8_stride, + int8_base + keep_sink * int8_stride, + scale_base + keep_sink * scale_stride, + tail_capacity, kv_heads, hdim); } + meta->current_seq_len = keep_sink + tail_capacity; + *node.output_buffer.data_as() = static_cast(meta->current_seq_len); + return; + } - size_t append_offset = window - new_seq_len; - cactus_quantize_kv_fp16_to_int8( - new_kv.data_as<__fp16>(), - int8_base + append_offset * int8_stride, - scale_base + append_offset * scale_stride, - new_seq_len, kv_heads, hdim); + size_t remaining = tail_capacity - new_seq_len; + remaining = std::min(remaining, current_len - keep_sink); + size_t shift_src = current_len - remaining; + + if (remaining > 0 && shift_src > keep_sink) { + std::memmove(int8_base + keep_sink * int8_stride, + int8_base + shift_src * int8_stride, + remaining * int8_stride); + std::memmove(scale_base + keep_sink * scale_stride, + scale_base + shift_src * scale_stride, + remaining * scale_stride * sizeof(float)); } - meta->current_seq_len = window; + size_t append_offset = keep_sink + remaining; + cactus_quantize_kv_fp16_to_int8( + new_kv.data_as<__fp16>(), + int8_base + append_offset * int8_stride, + scale_base + append_offset * scale_stride, + new_seq_len, kv_heads, hdim); + + meta->current_seq_len = append_offset + new_seq_len; } else { cactus_quantize_kv_fp16_to_int8( new_kv.data_as<__fp16>(), @@ -260,9 +275,15 @@ void compute_attention_cached_node( size_t new_seq_len = key_new_buf.total_size / (kv_heads * hdim); size_t history_len = (cache_len >= new_seq_len) ? cache_len - new_seq_len : 0; + bool cache_only_attention = false; size_t position_offset = node.params.position_offset; if (position_offset == std::numeric_limits::max()) { position_offset = history_len; + } else if (position_offset == std::numeric_limits::max() - 1) { + position_offset = (cache_len >= seq_len) ? cache_len - seq_len : 0; + history_len = cache_len; + new_seq_len = 0; + cache_only_attention = true; } if (k_cache_buf.precision == Precision::FP16 || v_cache_buf.precision == Precision::FP16) { @@ -284,19 +305,19 @@ void compute_attention_cached_node( return; } - cactus_attention_hybrid_int8_fp16( - query_buf.data_as<__fp16>(), - cached_keys, - cached_values, - k_scales, - v_scales, - key_new_buf.data_as<__fp16>(), - val_new_buf.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, history_len, seq_len, - num_q_heads, kv_heads, hdim, - node.params.scale, - position_offset, + cactus_attention_hybrid_int8_fp16( + query_buf.data_as<__fp16>(), + cached_keys, + cached_values, + k_scales, + v_scales, + key_new_buf.data_as<__fp16>(), + val_new_buf.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, history_len, cache_only_attention ? 0 : seq_len, + num_q_heads, kv_heads, hdim, + node.params.scale, + position_offset, true, node.params.window_size, KV_QUANT_GROUP_SIZE, diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index 909328e5e..e87fc110b 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -104,6 +104,33 @@ void dispatch_binary_op_f16(OpType op, const __fp16* lhs, const __fp16* rhs, __f } } +static float apply_binary_op_f32(OpType op, float lhs, float rhs) { + switch (op) { + case OpType::ADD: + case OpType::ADD_CLIPPED: + return lhs + rhs; + case OpType::SUBTRACT: + return lhs - rhs; + case OpType::MULTIPLY: + return lhs * rhs; + case OpType::DIVIDE: + return lhs / rhs; + case OpType::NOT_EQUAL: + return lhs != rhs ? 1.0f : 0.0f; + default: + return lhs; + } +} + +void dispatch_binary_op_f32(OpType op, const float* lhs, const float* rhs, float* output, size_t count) { + CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + for (size_t i = start_idx; i < end_idx; ++i) { + output[i] = apply_binary_op_f32(op, lhs[i], rhs[i]); + } + }); +} + void dispatch_unary_op_f16(OpType op, const __fp16* input, __fp16* output, size_t count, float param) { ScalarOpType scalar_op; switch (op) { @@ -154,13 +181,42 @@ void compute_binary_op_node(GraphNode& node, const std::vector(lhs.precision)) + ")"); + if (lhs.precision != rhs.precision) { + throw std::runtime_error("Binary operations require matching precision"); + } + if (lhs.precision != Precision::FP16 && lhs.precision != Precision::FP32) { + throw std::runtime_error("Binary operations only support FP16/FP32 precision"); } if (node.params.broadcast_info.needs_broadcasting) { std::vector lhs_strides = compute_strides(lhs.shape, node.params.broadcast_info.output_shape); std::vector rhs_strides = compute_strides(rhs.shape, node.params.broadcast_info.output_shape); + const auto& output_shape = node.params.broadcast_info.output_shape; + size_t total_elements = 1; + for (size_t dim : output_shape) { + total_elements *= dim; + } + if (lhs.precision == Precision::FP32) { + const float* lhs_data = lhs.data_as(); + const float* rhs_data = rhs.data_as(); + float* out_data = node.output_buffer.data_as(); + CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + std::vector coords(output_shape.size()); + size_t tmp = start_idx; + for (int axis = static_cast(output_shape.size()) - 1; axis >= 0; --axis) { + coords[static_cast(axis)] = tmp % output_shape[static_cast(axis)]; + tmp /= output_shape[static_cast(axis)]; + } + for (size_t linear_idx = start_idx; linear_idx < end_idx; ++linear_idx) { + size_t lhs_idx = broadcast_linear_index(coords, lhs_strides); + size_t rhs_idx = broadcast_linear_index(coords, rhs_strides); + out_data[linear_idx] = apply_binary_op_f32(node.op_type, lhs_data[lhs_idx], rhs_data[rhs_idx]); + increment_broadcast_coords(coords, output_shape); + } + }); + return; + } switch (node.op_type) { case OpType::ADD: @@ -196,11 +252,6 @@ void compute_binary_op_node(GraphNode& node, const std::vector(); const __fp16* rhs_data = rhs.data_as<__fp16>(); __fp16* out_data = node.output_buffer.data_as<__fp16>(); - const auto& output_shape = node.params.broadcast_info.output_shape; - size_t total_elements = 1; - for (size_t dim : output_shape) { - total_elements *= dim; - } CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ELEMENT_WISE, [&](size_t start_idx, size_t end_idx) { std::vector coords(output_shape.size()); @@ -222,9 +273,15 @@ void compute_binary_op_node(GraphNode& node, const std::vector(), - rhs.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); + if (lhs.precision == Precision::FP16) { + dispatch_binary_op_f16(node.op_type, lhs.data_as<__fp16>(), + rhs.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + } else { + dispatch_binary_op_f32(node.op_type, lhs.data_as(), + rhs.data_as(), node.output_buffer.data_as(), + node.output_buffer.total_size); + } } } @@ -329,8 +386,8 @@ void compute_reduce_node(GraphNode& node, const std::vector(axis)); + if (input_buffer.precision == Precision::FP32) { + const float* input = input_buffer.data_as(); + float* output = node.output_buffer.data_as(); + for (size_t outer = 0; outer < dims.outer; ++outer) { + for (size_t inner = 0; inner < dims.inner; ++inner) { + float running = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + size_t flat_index = ((outer * dims.axis_size) + axis_index) * dims.inner + inner; + running += input[flat_index]; + output[flat_index] = running; + } + } + } + return; + } + const __fp16* input = input_buffer.data_as<__fp16>(); __fp16* output = node.output_buffer.data_as<__fp16>(); @@ -355,6 +428,103 @@ void compute_reduce_node(GraphNode& node, const std::vector(); + float* output = node.output_buffer.data_as(); + if (axis == -1) { + float result = 0.0f; + switch (node.op_type) { + case OpType::SUM: + case OpType::MEAN: + for (size_t i = 0; i < input_buffer.total_size; ++i) result += input[i]; + if (node.op_type == OpType::MEAN && input_buffer.total_size > 0) { + result /= static_cast(input_buffer.total_size); + } + output[0] = result; + break; + case OpType::VARIANCE: { + float mean = 0.0f; + for (size_t i = 0; i < input_buffer.total_size; ++i) mean += input[i]; + mean /= static_cast(std::max(input_buffer.total_size, 1)); + float variance = 0.0f; + for (size_t i = 0; i < input_buffer.total_size; ++i) { + float diff = input[i] - mean; + variance += diff * diff; + } + output[0] = variance / static_cast(std::max(input_buffer.total_size, 1)); + break; + } + case OpType::MIN: + result = input_buffer.total_size == 0 ? 0.0f : input[0]; + for (size_t i = 1; i < input_buffer.total_size; ++i) result = std::min(result, input[i]); + output[0] = result; + break; + case OpType::MAX: + result = input_buffer.total_size == 0 ? 0.0f : input[0]; + for (size_t i = 1; i < input_buffer.total_size; ++i) result = std::max(result, input[i]); + output[0] = result; + break; + default: + break; + } + return; + } + + auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); + CactusThreading::parallel_for(dims.outer * dims.inner, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + for (size_t out_idx = start_idx; out_idx < end_idx; ++out_idx) { + size_t outer = out_idx / dims.inner; + size_t inner = out_idx % dims.inner; + size_t base = outer * dims.axis_size * dims.inner + inner; + float result = 0.0f; + switch (node.op_type) { + case OpType::SUM: + case OpType::MEAN: + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + result += input[base + axis_index * dims.inner]; + } + if (node.op_type == OpType::MEAN && dims.axis_size > 0) { + result /= static_cast(dims.axis_size); + } + output[out_idx] = result; + break; + case OpType::VARIANCE: { + float mean = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + mean += input[base + axis_index * dims.inner]; + } + mean /= static_cast(std::max(dims.axis_size, 1)); + float variance = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + float diff = input[base + axis_index * dims.inner] - mean; + variance += diff * diff; + } + output[out_idx] = variance / static_cast(std::max(dims.axis_size, 1)); + break; + } + case OpType::MIN: + result = dims.axis_size == 0 ? 0.0f : input[base]; + for (size_t axis_index = 1; axis_index < dims.axis_size; ++axis_index) { + result = std::min(result, input[base + axis_index * dims.inner]); + } + output[out_idx] = result; + break; + case OpType::MAX: + result = dims.axis_size == 0 ? 0.0f : input[base]; + for (size_t axis_index = 1; axis_index < dims.axis_size; ++axis_index) { + result = std::max(result, input[base + axis_index * dims.inner]); + } + output[out_idx] = result; + break; + default: + break; + } + } + }); + return; + } + if (axis == -1) { switch (node.op_type) { case OpType::SUM: { diff --git a/cactus-graph/src/ops_tensor.cpp b/cactus-graph/src/ops_tensor.cpp index a0dae609f..42f967ce5 100644 --- a/cactus-graph/src/ops_tensor.cpp +++ b/cactus-graph/src/ops_tensor.cpp @@ -136,6 +136,13 @@ void compute_embedding_node(GraphNode& node, const std::vector(); + } else if (indices_buffer.precision == Precision::FP16) { + indices_float.resize(num_indices); + const __fp16* half_indices = indices_buffer.data_as<__fp16>(); + for (size_t i = 0; i < num_indices; i++) { + indices_float[i] = static_cast(half_indices[i]); + } + indices_ptr = indices_float.data(); } else { indices_float.resize(num_indices); const int8_t* int_indices = indices_buffer.data_as(); diff --git a/cactus-graph/tests/test_cache.cpp b/cactus-graph/tests/test_cache.cpp index 38947597f..21cbdca7d 100644 --- a/cactus-graph/tests/test_cache.cpp +++ b/cactus-graph/tests/test_cache.cpp @@ -163,6 +163,32 @@ bool test_kv_cache_append_eviction() { return true; } +bool test_kv_cache_append_full_window_eviction() { + CactusGraph g; + + const size_t kv_heads = 1, head_dim = 16; + const size_t window = 8, sink = 2; + + size_t cache_node = g.kv_cache_state(window, kv_heads, head_dim, window, sink); + + for (int pass = 0; pass < 2; pass++) { + if (pass > 0) g.soft_reset(); + const size_t tokens = window; + const size_t elements = tokens * kv_heads * head_dim; + size_t kv_input = g.input({elements}, Precision::FP16); + std::vector<__fp16> data(elements, static_cast<__fp16>(static_cast(pass + 1))); + g.set_input(kv_input, data.data(), Precision::FP16); + g.kv_cache_append(kv_input, cache_node, window, sink); + g.execute(); + + auto* raw = static_cast(g.get_output(cache_node)); + uint64_t seq = *reinterpret_cast(raw); + if (seq != window) return false; + } + + return true; +} + bool test_attention_cached_basic() { const size_t b = 1, s = 1, h = 2, kv = 2, d = 16; const size_t max_seq = 64; @@ -493,6 +519,7 @@ int main() { runner.run_test("KV Cache Append Basic", test_kv_cache_append_basic()); runner.run_test("KV Cache Append Multiple", test_kv_cache_append_multiple()); runner.run_test("KV Cache Append Eviction", test_kv_cache_append_eviction()); + runner.run_test("KV Cache Append Full Window Eviction", test_kv_cache_append_full_window_eviction()); runner.run_test("Attention Cached Basic", test_attention_cached_basic()); runner.run_test("Attention Cached Multistep", test_attention_cached_multistep()); runner.run_test("KV Cache Invalidate", test_kv_cache_invalidate()); diff --git a/cactus-graph/tests/test_io.cpp b/cactus-graph/tests/test_io.cpp index 826bf54dc..298e6c440 100644 --- a/cactus-graph/tests/test_io.cpp +++ b/cactus-graph/tests/test_io.cpp @@ -75,7 +75,7 @@ bool test_graph_save_load() { size_t input_a = graph.input({2, 3}, Precision::FP16); size_t input_b = graph.input({2, 3}, Precision::FP16); size_t sum_id = graph.add(input_a, input_b); - graph.pow(sum_id, 2.0f); + size_t pow_id = graph.pow(sum_id, 2.0f); graph.save(filename); @@ -88,8 +88,8 @@ bool test_graph_save_load() { return false; } - if (sg.graph_inputs.size() != 2 || sg.graph_inputs[0] != 0 || - sg.graph_inputs[1] != 1) { + if (sg.graph_inputs.size() != 2 || sg.graph_inputs[0] != input_a || + sg.graph_inputs[1] != input_b) { std::cout << "[graph_save_load] unexpected graph_inputs:"; for (uint32_t idx : sg.graph_inputs) { std::cout << " " << idx; @@ -99,7 +99,7 @@ bool test_graph_save_load() { return false; } - if (sg.graph_outputs.size() != 1 || sg.graph_outputs[0] != 3) { + if (sg.graph_outputs.size() != 1 || sg.graph_outputs[0] != pow_id) { std::cout << "[graph_save_load] unexpected graph_outputs:"; for (uint32_t idx : sg.graph_outputs) { std::cout << " " << idx; diff --git a/cactus-graph/tests/test_utils.h b/cactus-graph/tests/test_utils.h index 31bd14945..511fd446e 100644 --- a/cactus-graph/tests/test_utils.h +++ b/cactus-graph/tests/test_utils.h @@ -135,7 +135,7 @@ inline void fill_random_fp16(std::vector<__fp16>& data) { } inline size_t runtime_id_from_serialized_index(uint32_t serialized_index) { - return static_cast(serialized_index) + 1; + return static_cast(serialized_index); } inline bool test_basic_operation(const std::string& op_name, diff --git a/cactus-kernels/src/threading.h b/cactus-kernels/src/threading.h index 665fecc69..8d5c7cdcc 100644 --- a/cactus-kernels/src/threading.h +++ b/cactus-kernels/src/threading.h @@ -240,11 +240,18 @@ namespace CactusThreading { inline bool pin_current_thread_to_cores(const std::vector& cores) { if (cores.empty()) return false; + cpu_set_t current_mask; + const bool has_current_mask = sched_getaffinity(0, sizeof(current_mask), ¤t_mask) == 0; cpu_set_t mask; CPU_ZERO(&mask); + bool selected = false; for (int core : cores) { - CPU_SET(core, &mask); + if (!has_current_mask || CPU_ISSET(core, ¤t_mask)) { + CPU_SET(core, &mask); + selected = true; + } } + if (!selected && has_current_mask) return true; return sched_setaffinity(0, sizeof(mask), &mask) == 0; } #endif diff --git a/cactus/build.sh b/cactus/build.sh index 051de97c2..cd913a9d4 100755 --- a/cactus/build.sh +++ b/cactus/build.sh @@ -21,7 +21,17 @@ cd "$(dirname "$0")" echo "Building Cactus library..." -rm -rf build +LOCK_DIR="${TMPDIR:-/tmp}/cactus-build.lock" +while ! mkdir "$LOCK_DIR" 2>/dev/null; do + if [ -f "$LOCK_DIR/pid" ] && ! kill -0 "$(cat "$LOCK_DIR/pid" 2>/dev/null)" 2>/dev/null; then + rm -rf "$LOCK_DIR" + continue + fi + sleep 2 +done +echo "$$" > "$LOCK_DIR/pid" +trap 'rm -rf "$LOCK_DIR"' EXIT + mkdir -p build cd build diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index a6db1ec8b..3edc83616 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -170,10 +170,20 @@ def create_parser(): help="Path to image file for VLM inference (attached to first message)") run_parser.add_argument("--audio", help="Path to audio file (WAV) for audio chat (attached to first message)") + run_parser.add_argument("--file", dest="audio_file", default=None, + help="Audio file for transpiled bundles") + run_parser.add_argument("--image-file", action="append", default=[], + help="Repeatable image input for transpiled bundles") run_parser.add_argument("--system", help="System prompt to prepend to all messages") run_parser.add_argument("--prompt", help="Initial prompt to send immediately") + run_parser.add_argument("--input-ids", default=None, + help="Comma-separated token ids for transpiled causal-LM bundles") + run_parser.add_argument("--max-new-tokens", type=int, default=None, + help="Maximum tokens to generate for transpiled causal-LM bundles") + run_parser.add_argument("--result-json", default=None, + help="Optional path to save transpiled bundle results as JSON") run_parser.add_argument("--thinking", action="store_true", help="Enable thinking/reasoning for models that support it") diff --git a/python/cactus/cli/compile.py b/python/cactus/cli/compile.py index ed3207ec6..af29a639b 100644 --- a/python/cactus/cli/compile.py +++ b/python/cactus/cli/compile.py @@ -183,6 +183,14 @@ def cmd_build(args): if result.returncode != 0: print_color(RED, "Failed to build cactus library") return 1 + for _ in range(50): + if lib_path.exists(): + break + import time + time.sleep(0.1) + if not lib_path.exists(): + print_color(RED, f"Build did not produce {lib_path}") + return 1 sdl2 = _detect_sdl2() if sdl2[0]: diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 2a71e4390..92a10c5c9 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -41,12 +41,14 @@ def cmd_run(args): print_color(RED, "Chat binary not found. Run `cactus build` first.") return 1 + image_file = next((image for image in getattr(args, 'image_file', []) if image), None) + cmd = [str(chat), str(bundle_dir)] for flag, value in ( - ("--system", args.system), - ("--prompt", args.prompt), - ("--image", args.image), - ("--audio", args.audio), + ("--system", getattr(args, 'system', None)), + ("--prompt", getattr(args, 'prompt', None)), + ("--image", getattr(args, 'image', None) or image_file), + ("--audio", getattr(args, 'audio', None) or getattr(args, 'audio_file', None)), ): if value: cmd.extend([ @@ -55,7 +57,14 @@ def cmd_run(args): if flag in ("--image", "--audio") else str(value), ]) - if args.thinking: + for flag, value in ( + ("--input-ids", getattr(args, 'input_ids', None)), + ("--max-new-tokens", getattr(args, 'max_new_tokens', None)), + ("--result-json", getattr(args, 'result_json', None)), + ): + if value is not None: + cmd.extend([flag, str(value)]) + if getattr(args, 'thinking', False): cmd.append("--thinking") print_color(GREEN, f"Starting Cactus Chat with model: {bundle_dir}") diff --git a/python/cactus/convert/cactus_adapters/tensor_io.py b/python/cactus/convert/cactus_adapters/tensor_io.py index ad36c8c38..3ce614cc7 100644 --- a/python/cactus/convert/cactus_adapters/tensor_io.py +++ b/python/cactus/convert/cactus_adapters/tensor_io.py @@ -281,8 +281,7 @@ def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=Fal data = data * GEMMA4_WEIGHT_SCALE elif 'router_scale' in filename: data = data / GEMMA4_WEIGHT_SCALE - elif filename in ('token_embeddings.weights', 'output_weight.weights', - 'embed_vision_proj.weights', 'embed_vision_embedding.weights'): + elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights'): data = data / GEMMA4_WEIGHT_SCALE elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE diff --git a/python/cactus/convert/model_adapters/adapters.py b/python/cactus/convert/model_adapters/adapters.py index 0b41f4259..6e6da78f9 100644 --- a/python/cactus/convert/model_adapters/adapters.py +++ b/python/cactus/convert/model_adapters/adapters.py @@ -162,7 +162,7 @@ def policy(self, match: NameMatch, shape: tuple[int, ...], requested_bits: int) policy = super().policy(match, shape, requested_bits) name = match.source_name if name == "model.embed_vision.embedding_projection.weight": - return replace(policy, use_gptq=False, fallback_reason=policy.fallback_reason or "vision embedding projection has no stable Hessian target") + return policy if ".self_attn." in name and (name.endswith(".k_proj.weight") or name.endswith(".v_proj.weight")): parts = name.split(".") try: diff --git a/python/cactus/convert/model_adapters/naming.py b/python/cactus/convert/model_adapters/naming.py index 0cd3b2726..cebbd9014 100644 --- a/python/cactus/convert/model_adapters/naming.py +++ b/python/cactus/convert/model_adapters/naming.py @@ -23,7 +23,6 @@ class NameMatch: GEMMA4_DIV_BASENAMES = { "token_embeddings", "output_weight", - "embed_vision_proj", "embed_vision_embedding", } diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index d74794030..a56d7fe31 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -24,6 +24,8 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("ignored", "none", None, component, False, "none", "no output filename") if "norm" in out.lower(): return TensorPolicy("fallback", "FP16", None, component, False, "none", "norm tensor") + if family == "gemma4" and name == "model.embed_vision.embedding_projection.weight": + return TensorPolicy("fallback", "FP16", None, component, False, "none", "vision embedding projection scale-sensitive") if family in {"parakeet", "parakeet_tdt"} and out.endswith(".bias") and ( "conv_" in out or out.startswith("subsampling_") or out.startswith("ctc_head_") ): @@ -44,6 +46,8 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("fallback", "FP16", None, component, False, "none", "conformer conv tensor") if family == "whisper" and out.startswith("encoder.layer_"): return TensorPolicy("fallback", "FP16", None, component, False, "none", "whisper encoder tensor") + if family == "gemma4" and component == "audio" and (name.endswith(".bias") or out.endswith(".bias") or ".bias." in out): + return TensorPolicy("fallback", "FP16", None, component, False, "none", "audio bias tensor") if name.endswith(".bias") or out.endswith(".bias") or ".bias." in out: return TensorPolicy("fallback", "FP16", None, component, False, "none", "bias tensor") if len(shape) != 2: diff --git a/python/cactus/convert/tests/test_policy.py b/python/cactus/convert/tests/test_policy.py index 441b94d2c..3a39f9fe8 100644 --- a/python/cactus/convert/tests/test_policy.py +++ b/python/cactus/convert/tests/test_policy.py @@ -38,7 +38,7 @@ def test_gemma4_adapter_disables_gptq_for_unhookable_tensors(): vision_proj = cactus_name_for_tensor("model.embed_vision.embedding_projection.weight", "gemma4", 35) vision_policy = adapter.policy(vision_proj, (1536, 1152), 4) - assert vision_policy.precision == "CQ4" + assert vision_policy.precision == "FP16" assert not vision_policy.use_gptq vision_tower = cactus_name_for_tensor("model.vision_tower.encoder.layers.0.mlp.down_proj.linear.weight", "gemma4", 35) @@ -56,11 +56,11 @@ def test_policy_audio_no_gptq(): assert not p.use_gptq -def test_policy_bias_int8(): +def test_policy_gemma4_audio_bias_fp16(): match = cactus_name_for_tensor("model.audio_tower.output_proj.bias", "gemma4", 1) p = policy_for_tensor(match, (1536,), 4, "gemma4") - assert p.precision == "INT8" - assert p.bits == 8 + assert p.precision == "FP16" + assert p.bits is None def test_policy_position_embedding_fp16(): diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index 16ba1cb8a..b0e4ee0ad 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -311,6 +311,7 @@ def _write_component_bundle( "audio_encoder", "vision_encoder", "lm_encoder", + "lm_encoder_text_chunk", "decoder_prefill_chunk", "decoder", "lm_encoder_step", @@ -2479,6 +2480,14 @@ def _load_transformers_bundle( ): config = _load_config_json(model_id) config_model_type = str(config.get("model_type", "") or "").lower() + if not config_model_type: + normalized_model_id = model_id.lower().replace("_", "-") + if "qwen3-vl" in normalized_model_id: + config_model_type = "qwen3_vl" + elif "lfm2.5-vl" in normalized_model_id or "lfm2-vl" in normalized_model_id: + config_model_type = "lfm2_vl" + if config_model_type: + config = {"model_type": config_model_type} external_transformers_site_packages = _ensure_transformers_supports_model_type(config_model_type) patch_note = _patch_transformers_torchvision_probe() if patch_note: @@ -2532,7 +2541,7 @@ def _load_transformers_bundle( f"Could not load tokenizer for {model_id}:\n" + "\n".join(tokenizer_errors) ) - if config_model_type in {"lfm2_vl", "qwen3_5"}: + if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}: model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -2543,7 +2552,7 @@ def _load_transformers_bundle( tie_note = _tie_lfm2_vl_lm_head_if_needed(model) if tie_note: print(f"note={tie_note}") - elif config_model_type == "qwen3_5" and isinstance(config.get("text_config"), dict): + elif config_model_type in {"qwen3_5", "qwen3_vl"} and isinstance(config.get("text_config"), dict): model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -2592,7 +2601,7 @@ def _load_transformers_bundle( + processor_config_hint ) - if config_model_type in {"lfm2_vl", "qwen3_5"}: + if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}: model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -3025,7 +3034,7 @@ def main() -> int: system_prompt=args.system_prompt, enable_thinking_if_supported=args.enable_thinking, ) - elif config_model_type == "qwen3_5": + elif config_model_type in {"qwen3_5", "qwen3_vl"}: prepared = _prepare_qwen3_5_multimodal_inputs( processor_or_tokenizer, prompt=args.prompt, diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index 4925e8e47..5ec2e48c0 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -23,6 +23,16 @@ _DYNAMIC_KV_POSITION_OFFSET = (1 << 64) - 1 +_DYNAMIC_KV_QUERY_POSITION_OFFSET = (1 << 64) - 2 +_KV_CACHE_TRANSPARENT_OPS = { + "expand", + "precision_cast", + "reshape", + "slice", + "transpose", + "unsqueeze", + "view", +} @dataclass @@ -254,6 +264,13 @@ def _lower_attention_with_internal_kv_cache( sink_size = int(ir.meta.get("cache_sink_size", 4) or 4) g.kv_cache_append(key, k_cache, window_size=window_size, sink_size=sink_size) g.kv_cache_append(value, v_cache, window_size=window_size, sink_size=sink_size) + kv_input_cache_states: dict[tuple[str, str], tuple[Tensor, Tensor, int, int]] = env.setdefault( + "__internal_kv_cache_by_kv_inputs", + {}, + ) # type: ignore[assignment] + cache_entry = (k_cache, v_cache, window_size, int(value_shape[3])) + kv_input_cache_states[(node.inputs[1], node.inputs[2])] = cache_entry + kv_input_cache_states[_canonical_kv_cache_input_key(ir, node)] = cache_entry out = g.attention_cached( query, key, @@ -270,6 +287,66 @@ def _lower_attention_with_internal_kv_cache( return [g.permute(out, (0, 2, 1, 3))] +def _lower_attention_with_shared_internal_kv_cache( + g: Graph, + ir: IRGraph, + env: dict[str, Any], + node: IRNode, + *, + query: Tensor, + key: Tensor, + value: Tensor, + output_layout: str, +) -> list[Tensor] | None: + kv_input_cache_states: dict[tuple[str, str], tuple[Tensor, Tensor, int, int]] = env.get( + "__internal_kv_cache_by_kv_inputs", + {}, + ) + cache_state = kv_input_cache_states.get((node.inputs[1], node.inputs[2])) + if cache_state is None: + cache_state = kv_input_cache_states.get(_canonical_kv_cache_input_key(ir, node)) + if cache_state is None: + return None + k_cache, v_cache, window_size, v_head_dim = cache_state + out = g.attention_cached( + query, + key, + value, + k_cache, + v_cache, + scale=_resolve_attention_scale(node, query), + position_offset=_DYNAMIC_KV_QUERY_POSITION_OFFSET, + window_size=window_size, + v_head_dim=v_head_dim, + ) + if output_layout == "bthd": + return [out] + return [g.permute(out, (0, 2, 1, 3))] + + +def _canonical_kv_cache_input_key(ir: IRGraph, node: IRNode) -> tuple[str, str]: + return ( + _canonical_kv_cache_value_id(ir, node.inputs[1]), + _canonical_kv_cache_value_id(ir, node.inputs[2]), + ) + + +def _canonical_kv_cache_value_id(ir: IRGraph, value_id: str) -> str: + current = value_id + seen: set[str] = set() + while current not in seen: + seen.add(current) + value = ir.values.get(current) + producer_id = None if value is None else value.producer + if producer_id is None: + break + producer = ir.nodes.get(producer_id) + if producer is None or producer.op not in _KV_CACHE_TRANSPARENT_OPS or len(producer.inputs) != 1: + break + current = producer.inputs[0] + return current + + def _should_lower_conv1d_with_internal_conv_cache(ir: IRGraph, node: IRNode, x: Tensor, weight: Tensor) -> bool: if not bool(ir.meta.get("use_internal_conv_cache", False)): return False @@ -1090,6 +1167,18 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> if mask_tensor is not None: mask_tensor = _ensure_fp16_tensor(g, mask_tensor) mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + shared_cached = _lower_attention_with_shared_internal_kv_cache( + g, + ir, + env, + node, + query=query, + key=key, + value=value, + output_layout=output_layout, + ) + if shared_cached is not None: + return shared_cached if _should_lower_attention_with_internal_kv_cache(ir, node): return _lower_attention_with_internal_kv_cache( g, @@ -2672,9 +2761,7 @@ def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: def _ensure_scalar_math_tensor(g: Graph, tensor: Tensor) -> Tensor: - if tensor.dtype == Graph.FP16: - return tensor - return g.precision_cast(tensor, Graph.FP16) + return tensor def _ensure_tensor_dtype(g: Graph, tensor: Tensor, dtype: int) -> Tensor: diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 42e91222c..a31973b0e 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from dataclasses import dataclass import inspect +import math import struct import os from pathlib import Path @@ -18,6 +19,7 @@ _GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE = 1.0 / 64.0 +_UNSET = object() try: from transformers.models.gemma4.modeling_gemma4 import create_bidirectional_mask as _GEMMA4_CREATE_BIDIRECTIONAL_MASK # type: ignore except Exception: @@ -330,6 +332,10 @@ def _torch_is_compiling() -> bool: def _gemma4_cpu_safe_text_mlp_enabled(layer: torch.nn.Module, hidden_states: torch.Tensor) -> bool: if hidden_states.device.type != "cpu" or hidden_states.dtype != torch.float16: return False + return _gemma4_text_mlp_manual_available(layer) + + +def _gemma4_text_mlp_manual_available(layer: torch.nn.Module) -> bool: if bool(getattr(layer, "enable_moe_block", False)): return False mlp = getattr(layer, "mlp", None) @@ -417,8 +423,12 @@ def _gemma4_text_attention_forward( value_states = attn.v_norm(value_states) value_states = value_states.transpose(1, 2) - if bool(getattr(attn, "store_full_length_kv", False)): - shared_kv_states[int(attn.layer_idx)] = (key_states, value_states) + layer_idx = getattr(attn, "layer_idx", None) + if layer_idx is not None and ( + bool(getattr(attn, "store_full_length_kv", False)) + or not bool(getattr(attn, "is_kv_shared_layer", False)) + ): + shared_kv_states[int(layer_idx)] = (key_states, value_states) attention_interface = eager_attention_forward if getattr(attn.config, "_attn_implementation", "eager") != "eager": @@ -450,7 +460,8 @@ def _gemma4_text_decoder_layer_forward( use_cache: bool = False, shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, ) -> torch.Tensor: - if not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): + manual_attention_required = shared_kv_states is not None + if not manual_attention_required and not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): return layer( hidden_states, position_embeddings=position_embeddings, @@ -460,6 +471,8 @@ def _gemma4_text_decoder_layer_forward( past_key_values=past_key_values, use_cache=use_cache, ) + if not _gemma4_text_mlp_manual_available(layer): + raise RuntimeError("Gemma4 KV sharing requires a supported text MLP structure") residual = hidden_states @@ -506,6 +519,10 @@ def _gemma4_text_backbone_forward( per_layer_inputs: torch.Tensor | None, causal_mask_mapping: dict[str, torch.Tensor], position_ids: torch.LongTensor, + layer_start: int = 0, + layer_end: int | None = None, + apply_norm: bool = True, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None | object = _UNSET, ) -> torch.Tensor: hidden_states = inputs_embeds layer_types = tuple(dict.fromkeys(getattr(backbone.config, "layer_types", ()))) @@ -513,10 +530,14 @@ def _gemma4_text_backbone_forward( layer_type: backbone.rotary_emb(hidden_states, position_ids, layer_type) for layer_type in layer_types } - shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + if shared_kv_states is _UNSET: + num_shared_layers = int(getattr(backbone.config, "num_kv_shared_layers", 0) or 0) + shared_kv_states = {} if num_shared_layers > 0 else None config_layer_types = tuple(getattr(backbone.config, "layer_types", ())) - for layer_index, decoder_layer in enumerate(backbone.layers[: backbone.config.num_hidden_layers]): + end = backbone.config.num_hidden_layers if layer_end is None else min(int(layer_end), int(backbone.config.num_hidden_layers)) + for layer_index in range(max(0, int(layer_start)), end): + decoder_layer = backbone.layers[layer_index] layer_per_input = None if per_layer_inputs is not None: layer_per_input = per_layer_inputs[:, :, decoder_layer.layer_idx, :] @@ -537,7 +558,7 @@ def _gemma4_text_backbone_forward( shared_kv_states=shared_kv_states, ) - return backbone.norm(hidden_states) + return backbone.norm(hidden_states) if apply_norm else hidden_states def _gemma4_strip_audio_padding(audio_output: object) -> torch.Tensor: @@ -712,7 +733,15 @@ def _gemma4_pool_vision_hidden_native_like( else: valid_bins = torch.logical_not((weights == 0).all(dim=0)) pooled_batches.append(pooled_full[valid_bins]) - return torch.cat(pooled_batches, dim=0) + pooled_hidden = pooled_batches[0] if len(pooled_batches) == 1 else torch.cat(pooled_batches, dim=0) + hidden_size = int(_module_or_config_attr(vision_tower, "hidden_size", pooled_hidden.shape[-1]) or pooled_hidden.shape[-1]) + pooled_hidden = pooled_hidden * math.sqrt(float(hidden_size)) + if bool(_module_or_config_attr(vision_tower, "standardize", False)): + std_bias = getattr(vision_tower, "std_bias", None) + std_scale = getattr(vision_tower, "std_scale", None) + if std_bias is not None and std_scale is not None: + pooled_hidden = (pooled_hidden - std_bias) * std_scale + return pooled_hidden def _gemma4_vision_encoder_hidden_states( @@ -837,10 +866,14 @@ def _gemma4_compute_native_like_image_features( projection = getattr(embed_vision, "embedding_projection", None) if not isinstance(projection, torch.nn.Linear): raise TypeError("Gemma4 vision embedder is missing embedding_projection") + pre_projection_norm = getattr(embed_vision, "embedding_pre_projection_norm", None) + if isinstance(pre_projection_norm, torch.nn.Module): + pooled_hidden = pre_projection_norm(pooled_hidden) + pooled_hidden = pooled_hidden.to(dtype=projection.weight.dtype) projected = F.linear( pooled_hidden, - projection.weight.float(), - None if projection.bias is None else projection.bias.float(), + projection.weight, + projection.bias, ) if post_proj_norm_weight is not None and post_proj_norm_weight.numel() > 0: return _gemma4_rms_norm( @@ -848,7 +881,7 @@ def _gemma4_compute_native_like_image_features( post_proj_norm_weight, eps=_gemma4_layer_norm_eps(multimodal_backbone), ) - return _gemma4_rms_norm_no_scale(projected, eps=float(getattr(embed_vision, "eps", 1e-6))) + return projected def _gemma4_compute_native_like_audio_features( @@ -1688,6 +1721,10 @@ def get_transpile_metadata(self): } +class Lfm2VlLMEncoderTextChunkAdapter(Lfm2VlLMEncoderStepAdapter): + pass + + class Lfm2VlDecoderAdapter(torch.nn.Module): def __init__( self, @@ -3040,6 +3077,75 @@ class Gemma4DecoderStepAdapter(Gemma4DecoderAdapter): pass +class Gemma4DecoderPrefillChunkAdapter(Gemma4DecoderAdapter): + def forward( + self, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor, + position_ids: torch.LongTensor, + ) -> torch.Tensor: + normalized_per_layer_inputs = per_layer_inputs + if normalized_per_layer_inputs.numel() == 0: + normalized_per_layer_inputs = None + attention_mask = torch.ones( + position_ids.shape, + dtype=torch.long, + device=position_ids.device, + ) + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + num_layers = int(getattr(self.backbone.config, "num_hidden_layers", len(self.backbone.layers))) + num_shared_layers = int(getattr(self.backbone.config, "num_kv_shared_layers", 0) or 0) + first_shared_layer = max(0, num_layers - num_shared_layers) + if first_shared_layer <= 0 or first_shared_layer >= num_layers or int(inputs_embeds.shape[1]) <= 1: + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + ) + else: + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + layer_end=first_shared_layer, + apply_norm=False, + shared_kv_states=shared_kv_states, + ) + tail_masks = { + key: value[:, :, -1:, :] if value.ndim == 4 else value + for key, value in causal_mask_mapping.items() + } + tail_per_layer_inputs = ( + None + if normalized_per_layer_inputs is None + else normalized_per_layer_inputs[:, -1:, :, :] + ) + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=hidden_states[:, -1:, :], + per_layer_inputs=tail_per_layer_inputs, + causal_mask_mapping=tail_masks, + position_ids=position_ids[:, -1:], + layer_start=first_shared_layer, + apply_norm=True, + shared_kv_states=shared_kv_states, + ) + logits = self.model.lm_head(hidden_states[:, -1:, :]) + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def _build_gemma4_multimodal_component_specs( model: torch.nn.Module, *, @@ -3076,32 +3182,27 @@ def _require(component: str) -> None: _require("lm_encoder_step") _require("lm_encoder_media_step") _require("decoder_step") - if "decoder_prefill_chunk" in requested_set: + if "lm_encoder" in requested_set: _require("vision_encoder") _require("audio_encoder") _require("lm_encoder") + if "decoder_prefill_chunk" in requested_set: _require("decoder_prefill_chunk") - elif "lm_encoder" in requested_set: + if "vision_encoder" in requested_set: _require("vision_encoder") + if "audio_encoder" in requested_set: _require("audio_encoder") - _require("lm_encoder") - else: - if "vision_encoder" in requested_set: - _require("vision_encoder") - if "audio_encoder" in requested_set: - _require("audio_encoder") - if "lm_encoder_step" in requested_set: - _require("lm_encoder_step") - if "lm_encoder_text_chunk" in requested_set: - _require("lm_encoder_text_chunk") - if "lm_encoder_media_step" in requested_set: - _require("lm_encoder_media_step") - if "lm_encoder_media_chunk" in requested_set: - _require("lm_encoder_media_chunk") - if "decoder_step" in requested_set: - _require("lm_encoder_step") - _require("lm_encoder_media_step") - _require("decoder_step") + if "lm_encoder_step" in requested_set: + _require("lm_encoder_step") + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "lm_encoder_media_step" in requested_set: + _require("lm_encoder_media_step") + if "lm_encoder_media_chunk" in requested_set: + _require("lm_encoder_media_chunk") + if "decoder_step" in requested_set: + _require("lm_encoder_step") + _require("decoder_step") vision_tower = getattr(getattr(model, "model", model), "vision_tower", None) pooling_kernel_size = int(_module_or_config_attr(vision_tower, "pooling_kernel_size", 3) or 3) @@ -3125,6 +3226,7 @@ def _require(component: str) -> None: image_features: torch.Tensor | None = None audio_features: torch.Tensor | None = None decoder_inputs: tuple[torch.Tensor, ...] | None = None + prefill_decoder_inputs: tuple[torch.Tensor, ...] | None = None decoder_step_inputs: tuple[torch.Tensor, ...] | None = None step_input_ids: torch.Tensor | None = None step_position_ids: torch.Tensor | None = None @@ -3135,16 +3237,14 @@ def _require(component: str) -> None: if "vision_encoder" in expanded_components and ( "lm_encoder" in expanded_components or "decoder" in expanded_components - or "decoder_prefill_chunk" in expanded_components ): image_features = vision_encoder(pixel_values, pixel_position_ids) if "audio_encoder" in expanded_components and ( "lm_encoder" in expanded_components or "decoder" in expanded_components - or "decoder_prefill_chunk" in expanded_components ): audio_features = audio_encoder(input_features, input_features_mask) - if "lm_encoder" in expanded_components or "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + if "lm_encoder" in expanded_components or "decoder" in expanded_components: if image_features is None: image_features = vision_encoder(pixel_values, pixel_position_ids) if audio_features is None: @@ -3160,17 +3260,18 @@ def _require(component: str) -> None: weights_dir=weights_dir, native_merge_plan=native_merge_plan, ).eval() - encoder_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_ENCODER_CHUNK", "32") or "32")) + encoder_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_ENCODER_CHUNK", "128") or "128")) encoder_chunk_size = min(encoder_chunk_size, int(input_ids.shape[1])) lm_encoder_step = Gemma4LMEncoderStepAdapter(model, weights_dir=weights_dir).eval() lm_encoder_text_chunk = Gemma4LMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() lm_encoder_media_step = Gemma4LMEncoderMediaStepAdapter(model, weights_dir=weights_dir).eval() lm_encoder_media_chunk = Gemma4LMEncoderMediaChunkAdapter(model, weights_dir=weights_dir).eval() decoder = Gemma4DecoderAdapter(model, weights_dir=weights_dir).eval() + decoder_prefill = Gemma4DecoderPrefillChunkAdapter(model, weights_dir=weights_dir).eval() decoder_step = Gemma4DecoderStepAdapter(model, weights_dir=weights_dir).eval() with torch.no_grad(): - if "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + if "decoder" in expanded_components: decoder_inputs = lm_encoder( input_ids, attention_mask, @@ -3209,6 +3310,8 @@ def _require(component: str) -> None: dtype=torch.long, device=input_ids.device, ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + if "decoder_prefill_chunk" in expanded_components and decoder_inputs is None: + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) if media_step_embeds is not None: hidden_size = int(media_step_embeds.shape[-1]) media_chunk_embeds = torch.zeros( @@ -3268,7 +3371,7 @@ def _require(component: str) -> None: metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) cache_seq_len = max(1024, int(input_ids.shape[1]) + 256) - prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "64") or "64")) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "128") or "128")) prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) if "decoder" in expanded_components: if decoder_inputs is None: @@ -3283,17 +3386,21 @@ def _require(component: str) -> None: metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) if "decoder_prefill_chunk" in expanded_components: - if decoder_inputs is None: + if decoder_inputs is not None: + chunk_inputs = tuple( + value[:, :prefill_chunk_size, ...].contiguous() + if value.ndim >= 2 + else value + for value in decoder_inputs + ) + elif prefill_decoder_inputs is not None: + prefill_chunk_size = int(prefill_decoder_inputs[0].shape[1]) + chunk_inputs = tuple(prefill_decoder_inputs) + else: raise RuntimeError("Gemma4 decoder_prefill_chunk spec requires precomputed decoder inputs") - chunk_inputs = tuple( - value[:, :prefill_chunk_size, ...].contiguous() - if value.ndim >= 2 - else value - for value in decoder_inputs - ) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", - module=decoder, + module=decoder_prefill, example_inputs=chunk_inputs, input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, output_keys=("logits",), @@ -3422,11 +3529,15 @@ def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[tor position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=linear_attn_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3443,7 +3554,7 @@ def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[tor return self.model.lm_head(hidden_states), checkpoints def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3488,11 +3599,15 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch. hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=linear_attn_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3502,7 +3617,7 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch. return self.model.lm_head(hidden_states[:, -1:, :]) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3534,6 +3649,26 @@ def _resolve_qwen35_multimodal_backbone(model: torch.nn.Module) -> torch.nn.Modu raise AttributeError(f"{type(model).__name__} does not expose a Qwen3.5 multimodal backbone") +def _qwen35_layer_types(config) -> tuple[str, ...]: + layer_types = tuple(str(value) for value in (getattr(config, "layer_types", ()) or ())) + if layer_types: + return layer_types + num_layers = int(getattr(config, "num_hidden_layers", 0) or 0) + return tuple("full_attention" for _ in range(num_layers)) + + +def _qwen35_attention_mask_for_layer( + config, + layer_index: int, + *, + causal_mask, + linear_attention_mask, +): + layer_types = _qwen35_layer_types(config) + layer_type = layer_types[layer_index] if layer_index < len(layer_types) else "full_attention" + return linear_attention_mask if layer_type == "linear_attention" else causal_mask + + class Qwen35VisionEncoderAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module, *, image_grid_thw: torch.Tensor): super().__init__() @@ -3677,11 +3812,15 @@ def forward( hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = attention_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=attention_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3690,7 +3829,7 @@ def forward( return self.model.lm_head(hidden_states) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3732,6 +3871,10 @@ def get_transpile_metadata(self): } +class Qwen35LMEncoderTextChunkAdapter(Qwen35LMEncoderStepAdapter): + pass + + class Qwen35EmbedsCausalLMStepAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module): super().__init__() @@ -3762,11 +3905,15 @@ def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> to hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = None if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=None, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3775,7 +3922,7 @@ def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> to return self.model.lm_head(hidden_states[:, -1:, :]) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3790,6 +3937,10 @@ def get_transpile_metadata(self): } +class Qwen35EmbedsCausalLMPrefillChunkAdapter(Qwen35EmbedsCausalLMStepAdapter): + pass + + class Qwen3CausalLMLogitsAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): super().__init__() @@ -4011,6 +4162,8 @@ def _build_qwen3_5_multimodal_component_specs( "vision_encoder", "lm_encoder", "decoder", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", "lm_encoder_step", "decoder_media_step", "decoder_step", @@ -4026,9 +4179,16 @@ def _require(component: str) -> None: _require("vision_encoder") _require("lm_encoder") _require("decoder") + _require("lm_encoder_text_chunk") + _require("decoder_prefill_chunk") if "lm_encoder" in requested_set: _require("vision_encoder") _require("lm_encoder") + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "decoder_prefill_chunk" in requested_set: + _require("lm_encoder_text_chunk") + _require("decoder_prefill_chunk") if "vision_encoder" in requested_set: _require("vision_encoder") if "decoder_step" in requested_set: @@ -4042,11 +4202,14 @@ def _require(component: str) -> None: lm_encoder = Qwen35LMEncoderAdapter(model, input_ids=input_ids).eval() decoder = Qwen35EmbedsCausalLMLogitsAdapter(model).eval() lm_encoder_step = Qwen35LMEncoderStepAdapter(model).eval() + lm_encoder_text_chunk = Qwen35LMEncoderTextChunkAdapter(model).eval() decoder_media_step = Qwen35EmbedsCausalLMStepAdapter(model).eval() + decoder_prefill_chunk = Qwen35EmbedsCausalLMPrefillChunkAdapter(model).eval() decoder_step = Qwen35CausalLMStepAdapter(model, pad_token_id=_resolve_model_pad_token_id(model)).eval() image_features: torch.Tensor | None = None decoder_inputs: tuple[torch.Tensor, ...] | None = None + prefill_decoder_inputs: tuple[torch.Tensor, ...] | None = None with torch.no_grad(): if "vision_encoder" in expanded_components and ("lm_encoder" in expanded_components or "decoder" in expanded_components): image_features = vision_encoder(pixel_values) @@ -4054,6 +4217,16 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values) decoder_inputs = lm_encoder(input_ids, attention_mask, position_ids, image_features) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_QWEN_PREFILL_CHUNK", "128") or "128")) + prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) + chunk_input_ids = input_ids[:, :prefill_chunk_size].contiguous() + chunk_position_ids = torch.arange( + prefill_chunk_size, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + if "decoder_prefill_chunk" in expanded_components: + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) common_graph_meta = { "weights_dir": weights_dir, @@ -4107,6 +4280,38 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, )) + if "lm_encoder_text_chunk" in expanded_components: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "position_ids"), + graph_meta={**common_graph_meta, "component": "lm_encoder_text_chunk"}, + metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, + )) + if "decoder_prefill_chunk" in expanded_components: + if prefill_decoder_inputs is None: + raise RuntimeError("Qwen3.5 decoder_prefill_chunk spec requires precomputed text chunk decoder inputs") + max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + specs.append(ComponentModuleSpec( + component="decoder_prefill_chunk", + module=decoder_prefill_chunk, + example_inputs=prefill_decoder_inputs, + input_keys=("inputs_embeds", "position_ids"), + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_prefill_chunk", + "use_internal_kv_cache": True, + "use_internal_conv_cache": True, + "use_internal_gated_deltanet_cache": True, + "max_cache_seq_len": max_cache_seq_len, + "cache_sink_size": 4, + "prefill_chunk_size": prefill_chunk_size, + }, + metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, + )) if "decoder_media_step" in expanded_components: if decoder_inputs is None: raise RuntimeError("Qwen3.5 decoder_media_step spec requires precomputed decoder inputs") @@ -4388,6 +4593,8 @@ def _family_key(model: torch.nn.Module) -> str: return "gemma" if module_name.startswith("transformers.models.qwen3_5."): return "qwen3_5" + if module_name.startswith("transformers.models.qwen3_vl."): + return "qwen3_5" if module_name.startswith("transformers.models.qwen3."): return "qwen3" if module_name.startswith("transformers.models.lfm2_vl."): @@ -4567,7 +4774,7 @@ def _build_lfm2_vl_multimodal_component_specs( dtype=attention_mask.dtype, device=attention_mask.device, ) - prefill_chunk = max(2, int(os.environ.get("CACTUS_LFM2_PREFILL_CHUNK", "64") or "64")) + prefill_chunk = max(2, int(os.environ.get("CACTUS_LFM2_PREFILL_CHUNK", "128") or "128")) prefill_chunk = min(prefill_chunk, int(input_ids.shape[1])) requested_components = tuple(components or ("vision_encoder", "lm_encoder", "text_lm_encoder", "decoder")) @@ -4588,28 +4795,28 @@ def _require(component: str) -> None: _require("decoder") _require("text_decoder") _require("lm_encoder_step") + _require("lm_encoder_text_chunk") _require("decoder_prefill_chunk") _require("decoder_step") - elif "lm_encoder" in requested_set: + if "lm_encoder" in requested_set: _require("vision_encoder") _require("lm_encoder") - elif "lm_encoder_step" in requested_set: + if "lm_encoder_step" in requested_set: _require("lm_encoder_step") - if "decoder_step" in requested_set: - _require("decoder_step") - elif "decoder_step" in requested_set: + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "decoder_step" in requested_set: _require("lm_encoder_step") _require("decoder_step") - elif "decoder_prefill_chunk" in requested_set: + if "decoder_prefill_chunk" in requested_set: + _require("lm_encoder_text_chunk") _require("decoder_prefill_chunk") - elif "text_lm_encoder" in requested_set: + if "text_lm_encoder" in requested_set: _require("text_lm_encoder") - if "text_decoder" in requested_set: - _require("text_decoder") - elif "text_decoder" in requested_set: + if "text_decoder" in requested_set: _require("text_lm_encoder") _require("text_decoder") - elif "vision_encoder" in requested_set: + if "vision_encoder" in requested_set: _require("vision_encoder") vision_encoder = Lfm2VlVisionEncoderAdapter( @@ -4621,6 +4828,7 @@ def _require(component: str) -> None: lm_encoder = Lfm2VlLMEncoderAdapter(model, input_ids=input_ids, weights_dir=weights_dir).eval() text_lm_encoder = Lfm2VlTextLMEncoderAdapter(model, weights_dir=weights_dir).eval() lm_encoder_step = Lfm2VlLMEncoderStepAdapter(model, weights_dir=weights_dir).eval() + lm_encoder_text_chunk = Lfm2VlLMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() decoder = Lfm2VlDecoderAdapter(model, weights_dir=weights_dir, last_token_only=False).eval() decoder_last_token = Lfm2VlDecoderAdapter(model, weights_dir=weights_dir, last_token_only=True).eval() @@ -4637,6 +4845,13 @@ def _require(component: str) -> None: if "text_lm_encoder" in expanded_components: text_decoder_inputs = text_lm_encoder(text_input_ids, text_attention_mask) + chunk_input_ids = input_ids[:, :prefill_chunk].contiguous() + chunk_position_ids = torch.arange( + prefill_chunk, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + common_graph_meta = { "weights_dir": weights_dir, "task": "multimodal_causal_lm_logits", @@ -4715,6 +4930,20 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, )) + if "lm_encoder_text_chunk" in expanded_components: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "attention_mask", "position_ids"), + graph_meta={ + **common_graph_meta, + "component": "lm_encoder_text_chunk", + "encoder_chunk_size": prefill_chunk, + }, + metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, + )) if "decoder_prefill_chunk" in expanded_components: if decoder_inputs is None: if image_features is None: diff --git a/python/cactus/transpile/model_profiles.py b/python/cactus/transpile/model_profiles.py index 7e84e4c69..6a07d9c7c 100644 --- a/python/cactus/transpile/model_profiles.py +++ b/python/cactus/transpile/model_profiles.py @@ -166,7 +166,7 @@ class ModelProfile: QWEN_PROFILE = ModelProfile( family="qwen", - model_types=("qwen", "qwen2", "qwen3", "qwen3_5", "qwen3.5"), + model_types=("qwen", "qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"), multimodal_context_tokens=512, input_combinations=((), ("image",)), model_id_aliases=( @@ -177,16 +177,25 @@ class ModelProfile: ("qwen3", "Qwen/Qwen3-1.7B"), ), model_id_markers=("qwen",), - family_aliases=("qwen2", "qwen3", "qwen3_5", "qwen3.5"), + family_aliases=("qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"), stop_tokens=("<|im_end|>",), avoid_native_loader=True, - cached_step_components=("decoder_step",), + cached_step_components=("decoder_media_step",), cached_step_skip_components=("decoder",), - fp16_kv_cache_components=("decoder_step",), + fp16_kv_cache_components=("decoder_prefill_chunk", "decoder_media_step", "decoder_step"), prompt_style="qwen_chat", multimodal_preprocessor="qwen3_5", default_task="multimodal_causal_lm_logits", - default_components=("vision_encoder", "lm_encoder", "decoder", "lm_encoder_step", "decoder_media_step", "decoder_step"), + default_components=( + "vision_encoder", + "lm_encoder", + "decoder", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + "lm_encoder_step", + "decoder_media_step", + "decoder_step", + ), needs_image=True, force_component_pipeline=True, ) diff --git a/python/cactus/transpile/runtime_compat.py b/python/cactus/transpile/runtime_compat.py index 677b15cb3..9c17c722e 100644 --- a/python/cactus/transpile/runtime_compat.py +++ b/python/cactus/transpile/runtime_compat.py @@ -233,31 +233,31 @@ def permute(self, x, permutation, backend=Graph.CPU): return orig_permute(self, _ensure_scalar_tensor(self, x), permutation, backend=backend) def scalar_add(self, x, value): - return orig_scalar_add(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_add(self, self._ensure_tensor(x), value) def scalar_subtract(self, x, value): - return orig_scalar_subtract(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_subtract(self, self._ensure_tensor(x), value) def scalar_multiply(self, x, value): - return orig_scalar_multiply(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_multiply(self, self._ensure_tensor(x), value) def scalar_divide(self, x, value): - return orig_scalar_divide(self, _ensure_scalar_tensor(self, x), value) + return orig_scalar_divide(self, self._ensure_tensor(x), value) def scalar_exp(self, x): - return orig_scalar_exp(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_exp(self, self._ensure_tensor(x)) def scalar_sqrt(self, x): - return orig_scalar_sqrt(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_sqrt(self, self._ensure_tensor(x)) def scalar_cos(self, x): - return orig_scalar_cos(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_cos(self, self._ensure_tensor(x)) def scalar_sin(self, x): - return orig_scalar_sin(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_sin(self, self._ensure_tensor(x)) def scalar_log(self, x): - return orig_scalar_log(self, _ensure_scalar_tensor(self, x)) + return orig_scalar_log(self, self._ensure_tensor(x)) def clamp(self, x, lo, hi): if orig_clamp is None: @@ -284,10 +284,10 @@ def cat(self, tensors, axis=0): return orig_cat(self, legalized, axis=axis) def abs(self, x): - return orig_abs(self, _ensure_scalar_tensor(self, x)) + return orig_abs(self, self._ensure_tensor(x)) def pow(self, x, exponent): - return orig_pow(self, _ensure_scalar_tensor(self, x), exponent) + return orig_pow(self, self._ensure_tensor(x), exponent) def expand(self, x, shape): x = self._ensure_tensor(x) diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py index 23847fc98..7f4a4170f 100644 --- a/python/tests/test_bindings.py +++ b/python/tests/test_bindings.py @@ -323,6 +323,22 @@ def test_run_command(self): assert args.model_id == "gemma4" assert args.prompt == "hi" + def test_run_command_chunked_bundle_flags(self): + args = self.parser.parse_args([ + "run", "bundle", + "--file", "audio.wav", + "--image-file", "image.png", + "--input-ids", "1,2,3", + "--max-new-tokens", "4", + "--result-json", "result.json", + ]) + assert args.command == "run" + assert args.audio_file == "audio.wav" + assert args.image_file == ["image.png"] + assert args.input_ids == "1,2,3" + assert args.max_new_tokens == 4 + assert args.result_json == "result.json" + def test_convert_command(self): args = self.parser.parse_args(["convert", "qwen", "--bits", "2"]) assert args.command == "convert" diff --git a/python/tests/test_cli_run.py b/python/tests/test_cli_run.py new file mode 100644 index 000000000..877400e07 --- /dev/null +++ b/python/tests/test_cli_run.py @@ -0,0 +1,60 @@ +from argparse import Namespace +from pathlib import Path +from types import SimpleNamespace + +import cactus.cli.run as run_mod + + +def test_cmd_run_forwards_chunked_bundle_flags(monkeypatch, tmp_path: Path) -> None: + bundle_dir = tmp_path / "bundle" + (bundle_dir / "components").mkdir(parents=True) + (bundle_dir / "components" / "manifest.json").write_text("{}", encoding="utf-8") + + fake_pkg = tmp_path / "pkg" + fake_chat = fake_pkg / "bin" / "chat" + fake_chat.parent.mkdir(parents=True) + fake_chat.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + monkeypatch.setattr(run_mod, "__file__", str(fake_pkg / "cli" / "run.py")) + + image_file = tmp_path / "image.png" + audio_file = tmp_path / "audio.wav" + result_json = tmp_path / "result.json" + image_file.write_bytes(b"image") + audio_file.write_bytes(b"audio") + + calls = [] + + def fake_run(cmd): + calls.append(cmd) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(run_mod.subprocess, "run", fake_run) + + args = Namespace( + no_cloud_tele=False, + model_id=str(bundle_dir), + token=None, + cache_dir=None, + reconvert=False, + system=None, + prompt="hi", + image=None, + image_file=[str(image_file)], + audio=None, + audio_file=str(audio_file), + input_ids="1,2,3", + max_new_tokens=4, + result_json=str(result_json), + thinking=False, + ) + + assert run_mod.cmd_run(args) == 0 + assert len(calls) == 1 + cmd = calls[0] + assert cmd[:2] == [str(fake_chat), str(bundle_dir)] + assert cmd[cmd.index("--prompt") + 1] == "hi" + assert cmd[cmd.index("--image") + 1] == str(image_file.resolve()) + assert cmd[cmd.index("--audio") + 1] == str(audio_file.resolve()) + assert cmd[cmd.index("--input-ids") + 1] == "1,2,3" + assert cmd[cmd.index("--max-new-tokens") + 1] == "4" + assert cmd[cmd.index("--result-json") + 1] == str(result_json) diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py index 5b1141dd0..570db6ec1 100644 --- a/python/tests/test_cli_transpile_defaults.py +++ b/python/tests/test_cli_transpile_defaults.py @@ -200,7 +200,6 @@ def _fake_cmd_transpile(transpile_args): assert extra_args[extra_args.index("--task") + 1] == "seq2seq_transcription" assert "--audio-file" in extra_args - def test_cmd_convert_infers_multimodal_for_qwen_and_lfm(monkeypatch, tmp_path: Path) -> None: """Qwen/LFM models WITH vision_config get multimodal task.""" parser = cli.create_parser() @@ -208,9 +207,21 @@ def test_cmd_convert_infers_multimodal_for_qwen_and_lfm(monkeypatch, tmp_path: P import cactus.convert.cli as cq_cli import cactus.cli.transpile as transpile_mod - for alias, model_type, arch, vision in ( - ("qwen", "qwen3", "Qwen3ForCausalLM", "qwen3_vision"), - ("lfm", "lfm2_vl", "Lfm2VlForConditionalGeneration", "siglip2_vision_model"), + for alias, model_type, arch, vision, expected_components in ( + ( + "qwen", + "qwen3", + "Qwen3ForCausalLM", + "qwen3_vision", + "vision_encoder,lm_encoder,decoder,lm_encoder_text_chunk,decoder_prefill_chunk,lm_encoder_step,decoder_media_step,decoder_step", + ), + ( + "lfm", + "lfm2_vl", + "Lfm2VlForConditionalGeneration", + "siglip2_vision_model", + "vision_encoder,lm_encoder,decoder", + ), ): output_dir = tmp_path / (alias + "_mm") args = parser.parse_args(["convert", alias, str(output_dir), "--reconvert"]) @@ -237,6 +248,10 @@ def _fake_cmd_transpile(transpile_args): assert rc == 0 extra_args = transpile_calls[0].extra_args assert extra_args[extra_args.index("--task") + 1] == "multimodal_causal_lm_logits" + assert extra_args[extra_args.index("--component-pipeline") + 1] == "on" + assert extra_args[extra_args.index("--components") + 1] == expected_components + assert "--image-file" in extra_args + assert "--audio-file" not in extra_args def test_cmd_convert_infers_causal_lm_for_qwen_and_lfm_without_vision(monkeypatch, tmp_path: Path) -> None: