From 77f808ed0a05f9cb49a86d66a52969903a9337c9 Mon Sep 17 00:00:00 2001 From: justinl66 Date: Wed, 20 May 2026 17:12:51 -0700 Subject: [PATCH 1/2] audio fix --- cactus-engine/src/engine.h | 7 ++ cactus-engine/src/model.cpp | 191 ++++++++++++++++++++++++++++++++---- 2 files changed, 180 insertions(+), 18 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 6cce3a484..bb19781c7 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -688,6 +688,11 @@ class Model { bool run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, const std::vector& audio_features); + bool build_lm_encoder_outputs_dynamic_gemma4( + const std::vector& tokens, + std::map>& store_bytes, + std::map& store_prec, + std::map>& store_shape); std::string bundle_dir_; std::map components_; @@ -698,6 +703,8 @@ class Model { Component* lm_encoder_media_step_ = nullptr; Component* decoder_prefill_chunk_ = nullptr; Component* lm_encoder_ = nullptr; + Component* lm_encoder_text_chunk_ = nullptr; + Component* lm_encoder_media_chunk_ = nullptr; std::string family_; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 4efc957f2..016f98fc5 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -163,11 +163,15 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, lm_encoder_media_step_ = components_.count("lm_encoder_media_step") ? &components_.at("lm_encoder_media_step") : nullptr; decoder_prefill_chunk_ = components_.count("decoder_prefill_chunk") ? &components_.at("decoder_prefill_chunk") : nullptr; lm_encoder_ = components_.count("lm_encoder") ? &components_.at("lm_encoder") : nullptr; + lm_encoder_text_chunk_ = components_.count("lm_encoder_text_chunk") ? &components_.at("lm_encoder_text_chunk") : nullptr; + lm_encoder_media_chunk_ = components_.count("lm_encoder_media_chunk") ? &components_.at("lm_encoder_media_chunk") : nullptr; if (vision_encoder_ && !bind_runtime_buffers(*vision_encoder_)) return false; if (audio_encoder_ && !bind_runtime_buffers(*audio_encoder_)) return false; if (lm_encoder_media_step_ && !bind_runtime_buffers(*lm_encoder_media_step_)) return false; if (decoder_prefill_chunk_ && !bind_runtime_buffers(*decoder_prefill_chunk_)) return false; if (lm_encoder_ && !bind_runtime_buffers(*lm_encoder_)) return false; + if (lm_encoder_text_chunk_ && !bind_runtime_buffers(*lm_encoder_text_chunk_)) return false; + if (lm_encoder_media_chunk_ && !bind_runtime_buffers(*lm_encoder_media_chunk_)) return false; if (vision_encoder_ && tokenizer_ && !vision_encoder_->output_node_ids.empty()) { size_t out_node = static_cast(vision_encoder_->output_node_ids[0]); @@ -471,7 +475,23 @@ void Model::run_audio_encoder(const std::vector& audio_features) { int mask_idx = input_index(*audio_encoder_, "input_features_mask"); if (mask_idx >= 0) { auto& mb = audio_encoder_->input_buffers[mask_idx]; - std::fill(mb.begin(), mb.end(), static_cast(1)); + size_t mask_node = static_cast(audio_encoder_->runtime_input_node_ids[mask_idx]); + const auto& mask_desc = audio_encoder_->graph->get_output_buffer(mask_node); + size_t elem = PrecisionTraits::size_of(mask_desc.precision); + size_t total = elem ? mb.size() / elem : mb.size(); + std::memset(mb.data(), 0, mb.size()); + for (size_t i = 0; i < total; ++i) { + switch (mask_desc.precision) { + case Precision::FP32: reinterpret_cast(mb.data())[i] = 1.0f; break; + case Precision::FP16: reinterpret_cast<__fp16*>(mb.data())[i] = static_cast<__fp16>(1.0f); break; + case Precision::INT8: reinterpret_cast(mb.data())[i] = 1; break; + default: + if (elem == 8) reinterpret_cast(mb.data())[i] = 1; + else if (elem == 4) reinterpret_cast(mb.data())[i] = 1; + else mb[i] = 1; + break; + } + } } audio_encoder_->graph->execute(); for (size_t i = 0; i < audio_encoder_->output_node_ids.size() && i < audio_encoder_->logical_outputs.size(); ++i) { @@ -626,6 +646,133 @@ void Model::copy_cache_state(const Component& src, Component& dst) { } } +bool Model::build_lm_encoder_outputs_dynamic_gemma4( + const std::vector& tokens, + std::map>& store_bytes, + std::map& store_prec, + std::map>& store_shape) { + if (!encoder_ || !lm_encoder_media_step_) return false; + if (tokens.empty()) return false; + + const uint32_t image_tok = config_.image_token_id; + const uint32_t audio_tok = config_.audio_token_id; + + auto af_it = media_features_.find("audio_features"); + const bool have_af = (af_it != media_features_.end() && !af_it->second.empty()); + size_t af_count = 0; + size_t af_row_bytes = 0; + Precision af_prec = Precision::FP16; + if (have_af) { + const auto& sh = media_feature_shapes_["audio_features"]; + af_count = sh.size() >= 2 ? sh[sh.size() - 2] : 0; + af_prec = media_feature_precisions_["audio_features"]; + af_row_bytes = af_count > 0 ? af_it->second.size() / af_count : af_it->second.size(); + } + + auto im_it = media_features_.find("image_features"); + const bool have_imf = (im_it != media_features_.end() && !im_it->second.empty()); + size_t imf_count = 0; + size_t imf_row_bytes = 0; + Precision imf_prec = Precision::FP16; + if (have_imf) { + const auto& sh = media_feature_shapes_["image_features"]; + imf_count = sh.size() >= 2 ? sh[sh.size() - 2] : 0; + imf_prec = media_feature_precisions_["image_features"]; + imf_row_bytes = imf_count > 0 ? im_it->second.size() / imf_count : im_it->second.size(); + } + + struct OutputInfo { + std::string name; + int text_idx = -1; + int media_idx = -1; + size_t per_token_bytes = 0; + Precision precision = Precision::FP16; + std::vector shape_template; + }; + std::vector outs; + for (size_t i = 0; i < encoder_->logical_outputs.size() && i < encoder_->output_node_ids.size(); ++i) { + OutputInfo oi; + oi.name = encoder_->logical_outputs[i]; + oi.text_idx = static_cast(i); + size_t node = static_cast(encoder_->output_node_ids[i]); + const auto& desc = encoder_->graph->get_output_buffer(node); + oi.per_token_bytes = desc.byte_size; + oi.precision = desc.precision; + oi.shape_template = desc.shape; + oi.media_idx = output_index(*lm_encoder_media_step_, oi.name); + if (oi.media_idx < 0) { + CACTUS_LOG_WARN("model", "dynamic walker: lm_encoder_media_step missing output '" << oi.name << "'"); + return false; + } + outs.push_back(std::move(oi)); + } + + const size_t N = tokens.size(); + for (auto& oi : outs) { + store_bytes[oi.name].assign(N * oi.per_token_bytes, 0); + store_prec[oi.name] = oi.precision; + std::vector sh = oi.shape_template; + if (sh.size() >= 2 && sh[sh.size() - 2] == 1) { + sh[sh.size() - 2] = N; + } else if (sh.size() == 1) { + sh[0] = N; + } + store_shape[oi.name] = sh; + } + + size_t audio_idx = 0; + size_t image_idx = 0; + for (size_t pos = 0; pos < N; ++pos) { + const uint32_t tk = tokens[pos]; + Component* enc; + const uint8_t* media_row = nullptr; + size_t media_row_bytes = 0; + Precision media_prec = Precision::FP16; + if (have_af && audio_tok != 0 && tk == audio_tok && audio_idx < af_count) { + enc = lm_encoder_media_step_; + media_row = af_it->second.data() + audio_idx * af_row_bytes; + media_row_bytes = af_row_bytes; + media_prec = af_prec; + ++audio_idx; + } else if (have_imf && image_tok != 0 && tk == image_tok && image_idx < imf_count) { + enc = lm_encoder_media_step_; + media_row = im_it->second.data() + image_idx * imf_row_bytes; + media_row_bytes = imf_row_bytes; + media_prec = imf_prec; + ++image_idx; + } else { + enc = encoder_; + } + + if (enc == lm_encoder_media_step_) { + int e_idx = input_index(*enc, "inputs_embeds"); + if (e_idx >= 0) { + auto& buf = enc->input_buffers[e_idx]; + size_t node_id = static_cast(enc->runtime_input_node_ids[e_idx]); + const auto& desc = enc->graph->get_output_buffer(node_id); + write_typed_buffer(buf, desc.precision, media_row, media_row_bytes, media_prec); + } + write_int_input(*enc, "input_ids", 0); + write_int_input(*enc, "position_ids", static_cast(pos)); + } else { + write_int_input(*enc, "input_ids", static_cast(tk)); + write_int_input(*enc, "position_ids", static_cast(pos)); + } + enc->graph->execute(); + + for (auto& oi : outs) { + int out_i = (enc == encoder_) ? oi.text_idx : oi.media_idx; + if (out_i < 0) continue; + size_t node = static_cast(enc->output_node_ids[out_i]); + const auto& desc = enc->graph->get_output_buffer(node); + if (desc.byte_size != oi.per_token_bytes) continue; + const void* ptr = enc->graph->get_output(node); + std::memcpy(store_bytes[oi.name].data() + pos * oi.per_token_bytes, ptr, oi.per_token_bytes); + } + } + return true; +} + bool Model::run_chunk_prefill_path(const std::vector& tokens, const std::vector& image_paths, const std::vector& audio_features) { @@ -720,7 +867,18 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, run_audio_encoder(audio_features); } - { + std::map> store_bytes; + std::map store_prec; + std::map> store_shape; + + const bool needs_dynamic_walk = (family_ == "gemma4") && have_audio + && encoder_ != nullptr && lm_encoder_media_step_ != nullptr; + + if (needs_dynamic_walk) { + if (!build_lm_encoder_outputs_dynamic_gemma4(tokens, store_bytes, store_prec, store_shape)) { + return false; + } + } else { int ids_idx = input_index(*lm_encoder_, "input_ids"); if (ids_idx >= 0) { auto& ids_buf = lm_encoder_->input_buffers[ids_idx]; @@ -748,23 +906,20 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, write_typed_buffer(dst_buf, desc.precision, kv.second.data(), kv.second.size(), src_prec); } - } - lm_encoder_->graph->execute(); + lm_encoder_->graph->execute(); - std::map> store_bytes; - std::map store_prec; - std::map> store_shape; - for (size_t i = 0; i < lm_encoder_->output_node_ids.size() - && i < lm_encoder_->logical_outputs.size(); ++i) { - const std::string& name = lm_encoder_->logical_outputs[i]; - size_t node_id = static_cast(lm_encoder_->output_node_ids[i]); - const auto& desc = lm_encoder_->graph->get_output_buffer(node_id); - void* ptr = lm_encoder_->graph->get_output(node_id); - auto& slot = store_bytes[name]; - slot.assign(desc.byte_size, 0); - std::memcpy(slot.data(), ptr, desc.byte_size); - store_prec[name] = desc.precision; - store_shape[name] = desc.shape; + for (size_t i = 0; i < lm_encoder_->output_node_ids.size() + && i < lm_encoder_->logical_outputs.size(); ++i) { + const std::string& name = lm_encoder_->logical_outputs[i]; + size_t node_id = static_cast(lm_encoder_->output_node_ids[i]); + const auto& desc = lm_encoder_->graph->get_output_buffer(node_id); + void* ptr = lm_encoder_->graph->get_output(node_id); + auto& slot = store_bytes[name]; + slot.assign(desc.byte_size, 0); + std::memcpy(slot.data(), ptr, desc.byte_size); + store_prec[name] = desc.precision; + store_shape[name] = desc.shape; + } } auto embeds_shape_it = store_shape.find("inputs_embeds"); From a3d09d0695166df2b3337fec096d6dc79a257bc4 Mon Sep 17 00:00:00 2001 From: Karen Mosoyan Date: Wed, 20 May 2026 23:38:51 -0700 Subject: [PATCH 2/2] transcription works, awaiting cleanup --- cactus-engine/src/embed.cpp | 1 + cactus-engine/src/engine.h | 7 + cactus-engine/src/model.cpp | 148 +++++++++++++++++- cactus-engine/src/transcribe.cpp | 31 +++- cactus-engine/src/utils.h | 20 +++ python/cactus/convert/cli.py | 10 +- python/cactus/convert/export/files.py | 1 + .../cactus/convert/model_adapters/adapters.py | 37 ++--- .../cactus/convert/model_adapters/naming.py | 4 + python/cactus/convert/model_adapters/nemo.py | 140 +++++++++++++++++ .../cactus/convert/model_adapters/policy.py | 10 +- python/cactus/transpile/hf_model.py | 10 +- python/cactus/transpile/tdt_runtime.py | 76 ++++++--- 13 files changed, 448 insertions(+), 47 deletions(-) create mode 100644 python/cactus/convert/model_adapters/nemo.py diff --git a/cactus-engine/src/embed.cpp b/cactus-engine/src/embed.cpp index 61715785a..3228fa6d7 100644 --- a/cactus-engine/src/embed.cpp +++ b/cactus-engine/src/embed.cpp @@ -19,6 +19,7 @@ static std::vector compute_mel_from_wav(const std::string& wav_path, size int norm_type = is_v3 ? 1 : 1; int scale_type = is_v3 ? 2 : 2; + cactus::audio::pad_or_trim_whisper_waveform(waveform_16k); std::vector mel = cactus::audio::compute_spectrogram_graph( waveform_16k, cfg, num_mel_filters, 0.0f, 8000.0f, 16000, norm_type, scale_type); if (mel.empty()) return mel; diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index bb19781c7..a06791308 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "cactus_graph.h" @@ -605,6 +606,11 @@ class Model { float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); std::vector transcribe_parakeet_tdt(const std::vector& audio_features); + std::vector transcribe_whisper_seq2seq(const std::vector& audio_features, + const std::vector& decoder_prompt_tokens, + size_t max_tokens, + const std::vector>& stop_token_sequences, + const std::atomic* should_stop = nullptr); std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); @@ -669,6 +675,7 @@ class Model { }; void copy_cache_state(const Component& src, Component& dst); + void reset_component_cache_states(Component& comp); bool load_manifest(); bool setup_tokenizer(); diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 016f98fc5..e4d56ebb4 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -149,10 +149,18 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, encoder_ = components_.count("lm_encoder_step") ? &components_.at("lm_encoder_step") : nullptr; decoder_ = components_.count("decoder_step") ? &components_.at("decoder_step") : nullptr; const bool is_lm = (encoder_ != nullptr && decoder_ != nullptr); + const bool is_whisper_transcription = + config_.model_type == Config::ModelType::WHISPER && + components_.count("audio_encoder") && + components_.count("decoder_cross_kv") && + components_.count("decoder_step"); + if (is_whisper_transcription) { + decoder_ = &components_.at("decoder_step"); + } const bool is_transcription = components_.count("audio_encoder") && - (components_.count("decoder") || components_.count("decoder_joint")); + (components_.count("decoder") || components_.count("decoder_joint") || is_whisper_transcription); if (!is_lm && !is_transcription) { - CACTUS_LOG_ERROR("model", "Bundle missing required components: need lm_encoder_step+decoder_step (LM) or audio_encoder+decoder (transcription)"); + CACTUS_LOG_ERROR("model", "Bundle missing required components: need lm_encoder_step+decoder_step (LM), audio_encoder+decoder (transcription), or Whisper audio_encoder+decoder_cross_kv+decoder_step"); return false; } if (encoder_ && !bind_runtime_buffers(*encoder_)) return false; @@ -646,6 +654,20 @@ void Model::copy_cache_state(const Component& src, Component& dst) { } } +void Model::reset_component_cache_states(Component& comp) { + for (const auto& state : comp.cache_states) { + for (int node_id : {state.key_node_id, state.value_node_id}) { + if (node_id < 0) continue; + const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); + if (desc.byte_size < sizeof(uint64_t) || !desc.get_data()) continue; + void* ptr = comp.graph->get_output(static_cast(node_id)); + if (!ptr) continue; + auto* metadata = static_cast(ptr); + metadata[0] = 0; + } + } +} + bool Model::build_lm_encoder_outputs_dynamic_gemma4( const std::vector& tokens, std::map>& store_bytes, @@ -1158,6 +1180,125 @@ uint32_t Model::decode_with_audio(const std::vector& tokens, const std return decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); } +std::vector Model::transcribe_whisper_seq2seq( + const std::vector& audio_features, + const std::vector& decoder_prompt_tokens, + size_t max_tokens, + const std::vector>& stop_token_sequences, + const std::atomic* should_stop) { + std::vector emitted; + if (decoder_prompt_tokens.empty() || max_tokens == 0) return emitted; + + Component* audio_enc = components_.count("audio_encoder") ? &components_.at("audio_encoder") : nullptr; + Component* cross_kv = components_.count("decoder_cross_kv") ? &components_.at("decoder_cross_kv") : nullptr; + Component* step = components_.count("decoder_step") ? &components_.at("decoder_step") : nullptr; + if (!audio_enc || !cross_kv || !step) { + CACTUS_LOG_ERROR("model", "Whisper bundle missing audio_encoder, decoder_cross_kv, or decoder_step component"); + return emitted; + } + if (!bind_runtime_buffers(*audio_enc)) return emitted; + if (!bind_runtime_buffers(*cross_kv)) return emitted; + if (!bind_runtime_buffers(*step)) return emitted; + + reset_component_cache_states(*step); + cache_total_seq_len_ = 0; + token_history_.clear(); + + const int feat_idx = input_index(*audio_enc, "input_features"); + if (feat_idx < 0) { + CACTUS_LOG_ERROR("model", "Whisper audio_encoder has no input_features input"); + return emitted; + } + auto& feat_buf = audio_enc->input_buffers[feat_idx]; + const size_t feat_node = static_cast(audio_enc->runtime_input_node_ids[feat_idx]); + const auto& feat_desc = audio_enc->graph->get_output_buffer(feat_node); + write_typed_buffer( + feat_buf, + feat_desc.precision, + audio_features.data(), + audio_features.size() * sizeof(float), + Precision::FP32); + audio_enc->graph->execute(); + + const int hidden_idx = output_index(*audio_enc, "encoder_hidden_states"); + if (hidden_idx < 0) { + CACTUS_LOG_ERROR("model", "Whisper audio_encoder has no encoder_hidden_states output"); + return emitted; + } + const size_t hidden_node = static_cast(audio_enc->output_node_ids[hidden_idx]); + const auto& hidden_desc = audio_enc->graph->get_output_buffer(hidden_node); + const void* hidden_ptr = audio_enc->graph->get_output(hidden_node); + if (hidden_ptr == nullptr || hidden_desc.byte_size == 0) { + CACTUS_LOG_ERROR("model", "Whisper encoder_hidden_states output is empty"); + return emitted; + } + + const int cross_hidden_idx = input_index(*cross_kv, "encoder_hidden_states"); + if (cross_hidden_idx < 0) { + CACTUS_LOG_ERROR("model", "Whisper decoder_cross_kv missing encoder_hidden_states input"); + return emitted; + } + auto& cross_hidden_buf = cross_kv->input_buffers[cross_hidden_idx]; + const size_t cross_hidden_node = static_cast(cross_kv->runtime_input_node_ids[cross_hidden_idx]); + const auto& cross_hidden_desc = cross_kv->graph->get_output_buffer(cross_hidden_node); + write_typed_buffer( + cross_hidden_buf, + cross_hidden_desc.precision, + hidden_ptr, + hidden_desc.byte_size, + hidden_desc.precision); + cross_kv->graph->execute(); + + for (size_t i = 0; i < cross_kv->output_node_ids.size() && i < cross_kv->logical_outputs.size(); ++i) { + const std::string& name = cross_kv->logical_outputs[i]; + int idx = input_index(*step, name); + if (idx < 0) continue; + const size_t src_node = static_cast(cross_kv->output_node_ids[i]); + const auto& src_desc = cross_kv->graph->get_output_buffer(src_node); + const void* src_ptr = cross_kv->graph->get_output(src_node); + if (src_ptr == nullptr || src_desc.byte_size == 0) continue; + auto& dst_buf = step->input_buffers[idx]; + const size_t dst_node = static_cast(step->runtime_input_node_ids[idx]); + const auto& dst_desc = step->graph->get_output_buffer(dst_node); + write_typed_buffer(dst_buf, dst_desc.precision, src_ptr, src_desc.byte_size, src_desc.precision); + } + + const int ids_idx = input_index(*step, "decoder_input_ids"); + const int pos_idx = input_index(*step, "position_ids"); + if (ids_idx < 0 || pos_idx < 0) { + CACTUS_LOG_ERROR("model", "Whisper decoder_step missing decoder_input_ids or position_ids input"); + return emitted; + } + + std::vector tokens = decoder_prompt_tokens; + auto stopped = [&]() { + for (const auto& stop_seq : stop_token_sequences) { + if (stop_seq.empty() || emitted.size() < stop_seq.size()) continue; + if (std::equal(stop_seq.rbegin(), stop_seq.rend(), emitted.rbegin())) return true; + } + return false; + }; + + for (size_t i = 0; i < max_tokens; ++i) { + if (should_stop && should_stop->load()) break; + const size_t start = cache_total_seq_len_ < tokens.size() ? cache_total_seq_len_ : tokens.size() - 1; + for (size_t pos = start; pos < tokens.size(); ++pos) { + write_int_input(*step, "decoder_input_ids", static_cast(tokens[pos])); + write_int_input(*step, "position_ids", static_cast(pos)); + step->graph->execute(); + } + cache_total_seq_len_ = tokens.size(); + + uint32_t next_token = argmax_last_logits(); + record_sampled_token(next_token); + emitted.push_back(next_token); + if (stopped()) break; + tokens.push_back(next_token); + } + + return emitted; +} + std::vector Model::transcribe_parakeet_tdt(const std::vector& audio_features) { std::vector emitted; @@ -1357,6 +1498,9 @@ std::vector Model::get_audio_embeddings(const std::vector& /*mel_b void Model::reset_cache() { cache_total_seq_len_ = 0; token_history_.clear(); + media_features_.clear(); + media_feature_shapes_.clear(); + media_feature_precisions_.clear(); for (auto& kv : components_) { Component& comp = kv.second; if (!comp.graph) continue; diff --git a/cactus-engine/src/transcribe.cpp b/cactus-engine/src/transcribe.cpp index f64370d5d..1cb781940 100644 --- a/cactus-engine/src/transcribe.cpp +++ b/cactus-engine/src/transcribe.cpp @@ -73,6 +73,7 @@ int cactus_preprocess_audio_features( cactus::audio::trim_mel_frames(features, bins, valid_frames); frames = features.size() / bins; } else { + cactus::audio::pad_or_trim_whisper_waveform(audio_samples); auto cfg = cactus::audio::get_whisper_spectrogram_config(); const bool is_whisper_v3 = bins > 80; if (is_whisper_v3) cactus::audio::apply_whisper_v3_overrides(cfg); @@ -192,6 +193,7 @@ int cactus_transcribe( handle_error_response("No audio input provided", response_buffer, buffer_size); return -1; } + const size_t original_audio_sample_count = audio_samples.size(); const size_t mel_bins = std::max(1, static_cast(handle->model->get_config().num_mel_bins)); std::vector audio_features; @@ -207,6 +209,7 @@ int cactus_transcribe( if (valid_frames == 0) valid_frames = 1; cactus::audio::trim_mel_frames(audio_features, mel_bins, valid_frames); } else { + cactus::audio::pad_or_trim_whisper_waveform(audio_samples); auto cfg = cactus::audio::get_whisper_spectrogram_config(); const bool is_whisper_v3 = mel_bins > 80; if (is_whisper_v3) cactus::audio::apply_whisper_v3_overrides(cfg); @@ -253,7 +256,7 @@ int cactus_transcribe( append_exact_stop_sequence(""); const float audio_length_sec = - static_cast(audio_samples.size()) / static_cast(cactus::audio::WHISPER_SAMPLE_RATE); + static_cast(original_audio_sample_count) / static_cast(cactus::audio::WHISPER_SAMPLE_RATE); if (options.max_tokens == 100 && (!options_json || std::string(options_json).find("\"max_tokens\"") == std::string::npos)) { options.max_tokens = std::max(100, static_cast(audio_length_sec * (is_parakeet ? 30.0f : 20.0f))); } @@ -282,13 +285,35 @@ int cactus_transcribe( callback(piece.c_str(), tok, user_data); } } + } else if (is_whisper) { + generated_tokens = handle->model->transcribe_whisper_seq2seq( + audio_features, + tokens, + options.max_tokens, + stop_token_sequences, + &handle->should_stop); + auto t_first = std::chrono::high_resolution_clock::now(); + time_to_first_token = + std::chrono::duration_cast(t_first - start_time).count() / 1000.0; + for (const auto& stop_seq : stop_token_sequences) { + if (stop_seq.empty() || generated_tokens.size() < stop_seq.size()) continue; + if (std::equal(stop_seq.rbegin(), stop_seq.rend(), generated_tokens.rbegin())) { + generated_tokens.resize(generated_tokens.size() - stop_seq.size()); + break; + } + } + final_text = tokenizer->decode(generated_tokens); + if (callback) { + for (uint32_t tok : generated_tokens) { + std::string piece = tokenizer->decode({tok}); + callback(piece.c_str(), tok, user_data); + } + } } else { for (size_t i = 0; i < options.max_tokens; ++i) { if (handle->should_stop) break; float token_entropy = 0.0f; - float tok_time_start = 0.0f; - float tok_time_end = 0.0f; uint32_t next_token = handle->model->decode_with_audio( tokens, audio_features, options.temperature, options.top_p, options.top_k, diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 42001d8e9..c09b7d688 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -98,6 +98,15 @@ namespace audio { static constexpr size_t WHISPER_TARGET_FRAMES = 3000; static constexpr int WHISPER_SAMPLE_RATE = 16000; +static constexpr size_t WHISPER_TARGET_SAMPLES = WHISPER_SAMPLE_RATE * 30; + +inline void pad_or_trim_whisper_waveform(std::vector& samples) { + if (samples.size() > WHISPER_TARGET_SAMPLES) { + samples.resize(WHISPER_TARGET_SAMPLES); + } else if (samples.size() < WHISPER_TARGET_SAMPLES) { + samples.resize(WHISPER_TARGET_SAMPLES, 0.0f); + } +} inline cactus::engine::SpectrogramConfig get_whisper_spectrogram_config() { cactus::engine::SpectrogramConfig cfg{}; @@ -313,6 +322,17 @@ inline std::vector normalize_whisper_mel(std::vector& mel, size_t if (mel.empty() || n_mels == 0) return mel; size_t n_frames = mel.size() / n_mels; + if (n_frames > WHISPER_TARGET_FRAMES) { + std::vector trimmed(n_mels * WHISPER_TARGET_FRAMES); + for (size_t m = 0; m < n_mels; ++m) { + const float* src = &mel[m * n_frames]; + float* dst = &trimmed[m * WHISPER_TARGET_FRAMES]; + std::copy(src, src + WHISPER_TARGET_FRAMES, dst); + } + mel = std::move(trimmed); + n_frames = WHISPER_TARGET_FRAMES; + } + float max_val = -std::numeric_limits::infinity(); for (float v : mel) if (v > max_val) max_val = v; diff --git a/python/cactus/convert/cli.py b/python/cactus/convert/cli.py index 6b3d60971..4c9ac0349 100644 --- a/python/cactus/convert/cli.py +++ b/python/cactus/convert/cli.py @@ -30,6 +30,7 @@ from .export.validate import validate_qdq from .model_adapters.detection import SUPPORTED_FAMILIES, detect_family from .model_adapters.adapters import adapter_for_family +from .model_adapters.nemo import ensure_parakeet_tdt_nemo_source from .quantization.cq import quantize_hadamard, quantize_orthogonal, write_cq_tensor from .compat import patch_transformers_import_compat @@ -48,6 +49,9 @@ def _load_hf(model_id_or_path: str, device: str): warnings.filterwarnings("ignore", message=".*You are using a model of type.*") for note in patch_transformers_import_compat(): print(f"note={note}") + nemo_export = ensure_parakeet_tdt_nemo_source(model_id_or_path, cache_dir=_hf_cache_dir()) + if nemo_export is not None: + model_id_or_path = nemo_export try: from transformers import AutoConfig, AutoModel except Exception as exc: # pragma: no cover @@ -123,6 +127,9 @@ def _bits_for_component(component: str, args: argparse.Namespace) -> int: def _load_checkpoint_state_dict(model_id_or_path: str) -> dict[str, Any] | None: + nemo_export = ensure_parakeet_tdt_nemo_source(model_id_or_path, cache_dir=_hf_cache_dir()) + if nemo_export is not None: + model_id_or_path = nemo_export root = Path(model_id_or_path) if not root.exists() or not root.is_dir(): try: @@ -370,6 +377,7 @@ def convert(args: argparse.Namespace) -> None: out_dir.mkdir(parents=True, exist_ok=True) cfg, processor, model = _load_hf(args.model, args.device) + runtime_source = ensure_parakeet_tdt_nemo_source(args.model, cache_dir=_hf_cache_dir()) or args.model family = detect_family(cfg, args.model_family) adapter = adapter_for_family(family) checkpoint_state = _load_checkpoint_state_dict(args.model) @@ -382,7 +390,7 @@ def convert(args: argparse.Namespace) -> None: model_config = adapter.runtime_config(cfg) model_config["model_type"] = adapter.runtime_model_type() write_config_txt({**_config_dict(cfg), **model_config}, out_dir) - copy_runtime_files(args.model, out_dir, token=getattr(args, "token", None), cache_dir=_hf_cache_dir()) + copy_runtime_files(runtime_source, out_dir, token=getattr(args, "token", None), cache_dir=_hf_cache_dir()) try: from .cactus_adapters.tokenizer import convert_hf_tokenizer diff --git a/python/cactus/convert/export/files.py b/python/cactus/convert/export/files.py index 20b395def..4529a4db8 100644 --- a/python/cactus/convert/export/files.py +++ b/python/cactus/convert/export/files.py @@ -11,6 +11,7 @@ "merges.txt", "tokenizer.json", "tokenizer.model", + "vocab.txt", "tokenizer_config.json", "special_tokens_map.json", "added_tokens.json", diff --git a/python/cactus/convert/model_adapters/adapters.py b/python/cactus/convert/model_adapters/adapters.py index bf51ec941..0b41f4259 100644 --- a/python/cactus/convert/model_adapters/adapters.py +++ b/python/cactus/convert/model_adapters/adapters.py @@ -389,24 +389,25 @@ def runtime_config(self, cfg: Any) -> dict[str, Any]: def normalize_state_dict(self, state_dict: dict[str, Any]) -> NormalizedState: augmented = dict(state_dict) provenance: dict[str, TensorProvenance] = {} - i = 0 - while True: - ih_key = f"decoder.lstm.bias_ih_l{i}" - hh_key = f"decoder.lstm.bias_hh_l{i}" - out_key = f"decoder.lstm.bias_l{i}" - if ih_key not in state_dict and hh_key not in state_dict: - break - sources = [k for k in (ih_key, hh_key) if k in state_dict] - if ih_key in state_dict and hh_key in state_dict: - augmented[out_key] = state_dict[ih_key] + state_dict[hh_key] - elif ih_key in state_dict: - augmented[out_key] = state_dict[ih_key] - else: - augmented[out_key] = state_dict[hh_key] - augmented.pop(ih_key, None) - augmented.pop(hh_key, None) - provenance[out_key] = TensorProvenance(sources, "parakeet_tdt_lstm_bias_sum", "hf_key") - i += 1 + for prefix in ("decoder.lstm", "decoder.prediction.dec_rnn.lstm"): + i = 0 + while True: + ih_key = f"{prefix}.bias_ih_l{i}" + hh_key = f"{prefix}.bias_hh_l{i}" + out_key = f"{prefix}.bias_l{i}" + if ih_key not in state_dict and hh_key not in state_dict: + break + sources = [k for k in (ih_key, hh_key) if k in state_dict] + if ih_key in state_dict and hh_key in state_dict: + augmented[out_key] = state_dict[ih_key] + state_dict[hh_key] + elif ih_key in state_dict: + augmented[out_key] = state_dict[ih_key] + else: + augmented[out_key] = state_dict[hh_key] + augmented.pop(ih_key, None) + augmented.pop(hh_key, None) + provenance[out_key] = TensorProvenance(sources, "parakeet_tdt_lstm_bias_sum", "hf_key") + i += 1 return NormalizedState(augmented, provenance) diff --git a/python/cactus/convert/model_adapters/naming.py b/python/cactus/convert/model_adapters/naming.py index e74c26236..0cd3b2726 100644 --- a/python/cactus/convert/model_adapters/naming.py +++ b/python/cactus/convert/model_adapters/naming.py @@ -243,6 +243,10 @@ def _parakeet_tdt_predictor_match(name: str) -> str | None: if new: suffix = {"weight_ih": "weight_ih.weights", "weight_hh": "weight_hh.weights", "bias": "bias.weights"}[new.group(1)] return f"tdt_predictor_lstm_{int(new.group(2))}_{suffix}" + nemo = re.fullmatch(r"decoder\.prediction\.dec_rnn\.lstm\.(weight_ih|weight_hh|bias)_l(\d+)", name) + if nemo: + suffix = {"weight_ih": "weight_ih.weights", "weight_hh": "weight_hh.weights", "bias": "bias.weights"}[nemo.group(1)] + return f"tdt_predictor_lstm_{int(nemo.group(2))}_{suffix}" return None diff --git a/python/cactus/convert/model_adapters/nemo.py b/python/cactus/convert/model_adapters/nemo.py new file mode 100644 index 000000000..af828a65c --- /dev/null +++ b/python/cactus/convert/model_adapters/nemo.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import json +import shutil +import tarfile +from pathlib import Path +from typing import Any + + +_NEMO_EXPORT_FILES = { + "model_config.yaml", + "model_weights.ckpt", + "vocab.txt", +} + + +def ensure_parakeet_tdt_nemo_source( + model_id_or_path: str, + *, + token: str | None = None, + cache_dir: str | None = None, +) -> str | None: + nemo_path = _find_single_nemo(model_id_or_path, token=token, cache_dir=cache_dir) + if nemo_path is None: + return None + + out = nemo_path.parent / f"{nemo_path.stem}-cactus-hf" + if (out / "config.json").exists() and (out / "pytorch_model.bin").exists() and (out / "vocab.txt").exists(): + return str(out) + + tmp = out.with_name(f"{out.name}.tmp") + if tmp.exists(): + shutil.rmtree(tmp) + tmp.mkdir(parents=True) + + with tarfile.open(nemo_path, "r:*") as archive: + for member in archive.getmembers(): + name = Path(member.name).name + if not _keep_nemo_member(name): + continue + extracted = archive.extractfile(member) + if extracted is None: + continue + with extracted, (tmp / name).open("wb") as dst: + shutil.copyfileobj(extracted, dst, length=1024 * 1024) + + weights = tmp / "model_weights.ckpt" + config = _read_yaml(tmp / "model_config.yaml") + if not weights.exists() or not config: + raise RuntimeError(f"{nemo_path} is missing model_config.yaml or model_weights.ckpt") + + hf_config = _parakeet_tdt_config_from_nemo(config) + (tmp / "config.json").write_text(json.dumps(hf_config, indent=2, sort_keys=True), encoding="utf-8") + weights.rename(tmp / "pytorch_model.bin") + _write_parakeet_tdt_tokenizer_files(tmp, hf_config) + + if out.exists(): + shutil.rmtree(out) + tmp.rename(out) + return str(out) + + +def _find_single_nemo(model_id_or_path: str, *, token: str | None, cache_dir: str | None) -> Path | None: + path = Path(model_id_or_path) + if path.suffix == ".nemo" and path.is_file(): + return path + if path.is_dir(): + files = sorted(path.glob("*.nemo")) + return files[0] if len(files) == 1 else None + + try: + from huggingface_hub import hf_hub_download, list_repo_files + + files = [name for name in list_repo_files(model_id_or_path, token=token) if name.endswith(".nemo")] + if len(files) != 1: + return None + return Path(hf_hub_download(model_id_or_path, files[0], token=token, cache_dir=cache_dir)) + except Exception: + return None + + +def _keep_nemo_member(name: str) -> bool: + return ( + name in _NEMO_EXPORT_FILES + or name.endswith("_vocab.txt") + or name.endswith("_tokenizer.model") + or name.endswith("_tokenizer.vocab") + ) + + +def _read_yaml(path: Path) -> dict[str, Any]: + try: + import yaml + except Exception as exc: + raise RuntimeError("PyYAML is required to import Parakeet TDT .nemo checkpoints") from exc + if not path.exists(): + return {} + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + return loaded if isinstance(loaded, dict) else {} + + +def _get(mapping: dict[str, Any], key: str, default: Any = None) -> Any: + value = mapping.get(key, default) + return default if value is None else value + + +def _parakeet_tdt_config_from_nemo(root: dict[str, Any]) -> dict[str, Any]: + decoder = _get(root, "decoder", {}) or {} + prednet = _get(decoder, "prednet", _get(decoder, "prediction", {})) or {} + joint = _get(root, "joint", {}) or {} + labels = [str(token) for token in (_get(joint, "vocabulary", []) or [])] + + decoder_vocab = int(_get(decoder, "vocab_size", len(labels))) + + config = dict(root) + config.update(architectures=["ParakeetForTDT"], model_type="parakeet_tdt", labels=labels) + config["decoder"] = {**decoder, "prediction": dict(prednet), "vocab_size": decoder_vocab} + config["joint"] = {**joint, "vocabulary": labels} + return config + + +def _write_parakeet_tdt_tokenizer_files(root: Path, config: dict[str, Any]) -> None: + tokenizer_model = next(iter(sorted(root.glob("*_tokenizer.model"))), None) + if tokenizer_model is not None and not (root / "tokenizer.model").exists(): + tokenizer_model.rename(root / "tokenizer.model") + + labels = config.get("labels") + if isinstance(labels, list) and labels: + with (root / "vocab.txt").open("w", encoding="utf-8") as f: + for idx, token in enumerate(labels): + f.write(f"{idx}\t{token}\n") + else: + nemo_vocab = next(iter(sorted(root.glob("*_vocab.txt"))), None) + if nemo_vocab is not None and not (root / "vocab.txt").exists(): + nemo_vocab.rename(root / "vocab.txt") + + (root / "tokenizer_config.json").write_text( + json.dumps({"model_type": "parakeet_tdt", "tokenizer_class": "SentencePieceProcessor"}, indent=2), + encoding="utf-8", + ) diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index 118683495..d74794030 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -32,6 +32,12 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("fallback", "FP16", None, component, False, "none", "lstm recurrent tensor") if family == "parakeet_tdt" and out.startswith("tdt_"): return TensorPolicy("fallback", "FP16", None, component, False, "none", "tdt decoder tensor") + if family in {"parakeet", "parakeet_tdt"} and "self_attn_bias_" in out: + return TensorPolicy("fallback", "FP16", None, component, False, "none", "relative attention bias tensor") + if family in {"parakeet", "parakeet_tdt"} and "conv_pointwise" in out and len(shape) == 3 and shape[2] == 1: + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "pointwise conv tensor") + if "conv_depthwise.weights" in out and len(shape) == 3 and shape[1] == 1: + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "depthwise conv tensor") if family in {"parakeet", "parakeet_tdt"} and out.startswith("layer_") and ( "conv_pointwise" in out or "conv_depthwise" in out ): @@ -40,10 +46,6 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("fallback", "FP16", None, component, False, "none", "whisper encoder tensor") if name.endswith(".bias") or out.endswith(".bias") or ".bias." in out: return TensorPolicy("fallback", "FP16", None, component, False, "none", "bias tensor") - if family in {"parakeet", "parakeet_tdt"} and "conv_pointwise" in out and len(shape) == 3 and shape[2] == 1: - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "pointwise conv tensor") - if "conv_depthwise.weights" in out and len(shape) == 3 and shape[1] == 1: - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "depthwise conv tensor") if len(shape) != 2: return TensorPolicy("fallback", "FP16", None, component, False, "none", "non-2d tensor") if "position_embedding" in out.lower() or "pos_embed" in out.lower() or "embed_positions" in out.lower(): diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index d3daf70f3..16ba1cb8a 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -24,6 +24,7 @@ sys.path.insert(0, str(PYTHON_ROOT)) from cactus.transpile.runtime_compat import Graph +from cactus.convert.model_adapters.nemo import ensure_parakeet_tdt_nemo_source from cactus.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features from cactus.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform from cactus.transpile.audio_preprocess import prepare_cactus_audio_features @@ -75,6 +76,9 @@ def _ensure_transformers_supports_model_type(model_type: str) -> str | None: def _resolve_local_snapshot(model_id_or_path: str) -> str | None: explicit = Path(model_id_or_path) if explicit.exists(): + nemo_export = ensure_parakeet_tdt_nemo_source(model_id_or_path) + if nemo_export is not None: + return nemo_export return str(explicit) snapshots_dir = ( @@ -90,7 +94,11 @@ def _resolve_local_snapshot(model_id_or_path: str) -> str | None: snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) if not snapshots: return None - return str(snapshots[-1]) + snapshot = str(snapshots[-1]) + nemo_export = ensure_parakeet_tdt_nemo_source(snapshot) + if nemo_export is not None: + return nemo_export + return snapshot def _snapshot_has_model_weights(path: str | Path) -> bool: diff --git a/python/cactus/transpile/tdt_runtime.py b/python/cactus/transpile/tdt_runtime.py index bdb6f59fb..7f6d2d562 100644 --- a/python/cactus/transpile/tdt_runtime.py +++ b/python/cactus/transpile/tdt_runtime.py @@ -51,9 +51,15 @@ def alias(target: str, source: str) -> None: for index in range(4): alias(f"decoder.prediction.dec_rnn.lstm.{index}.Wx", f"decoder.lstm.weight_ih_l{index}") alias(f"decoder.prediction.dec_rnn.lstm.{index}.Wh", f"decoder.lstm.weight_hh_l{index}") + alias(f"decoder.prediction.dec_rnn.lstm.{index}.Wx", f"decoder.prediction.dec_rnn.lstm.weight_ih_l{index}") + alias(f"decoder.prediction.dec_rnn.lstm.{index}.Wh", f"decoder.prediction.dec_rnn.lstm.weight_hh_l{index}") bias_key = f"decoder.prediction.dec_rnn.lstm.{index}.bias" bias_ih = state_dict.get(f"decoder.lstm.bias_ih_l{index}") bias_hh = state_dict.get(f"decoder.lstm.bias_hh_l{index}") + if bias_ih is None: + bias_ih = state_dict.get(f"decoder.prediction.dec_rnn.lstm.bias_ih_l{index}") + if bias_hh is None: + bias_hh = state_dict.get(f"decoder.prediction.dec_rnn.lstm.bias_hh_l{index}") if bias_key not in state_dict and bias_ih is not None and bias_hh is not None: state_dict[bias_key] = bias_ih + bias_hh @@ -412,10 +418,18 @@ def _relative_position_bias(query: torch.Tensor, relative_key: torch.Tensor, *, class ParakeetTDTFeedForward(nn.Module): def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): super().__init__() - self.linear1 = nn.Linear(config.hidden_dim, config.ff_intermediate_dim, bias=False) - self.linear2 = nn.Linear(config.ff_intermediate_dim, config.hidden_dim, bias=False) - _copy_linear_weight(self.linear1, state_dict[f"{prefix}.linear1.weight"]) - _copy_linear_weight(self.linear2, state_dict[f"{prefix}.linear2.weight"]) + self.linear1 = nn.Linear(config.hidden_dim, config.ff_intermediate_dim, bias=f"{prefix}.linear1.bias" in state_dict) + self.linear2 = nn.Linear(config.ff_intermediate_dim, config.hidden_dim, bias=f"{prefix}.linear2.bias" in state_dict) + _copy_linear_weight( + self.linear1, + state_dict[f"{prefix}.linear1.weight"], + bias=state_dict.get(f"{prefix}.linear1.bias"), + ) + _copy_linear_weight( + self.linear2, + state_dict[f"{prefix}.linear2.weight"], + bias=state_dict.get(f"{prefix}.linear2.bias"), + ) def forward(self, x: torch.Tensor, *, activation: str) -> torch.Tensor: return self.linear2(_apply_activation(self.linear1(x), activation)) @@ -424,15 +438,19 @@ def forward(self, x: torch.Tensor, *, activation: str) -> torch.Tensor: class ParakeetTDTSelfAttention(nn.Module): def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): super().__init__() - self.linear_q = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) - self.linear_k = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) - self.linear_v = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) - self.linear_out = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_q = nn.Linear(config.hidden_dim, config.hidden_dim, bias=f"{prefix}.linear_q.bias" in state_dict) + self.linear_k = nn.Linear(config.hidden_dim, config.hidden_dim, bias=f"{prefix}.linear_k.bias" in state_dict) + self.linear_v = nn.Linear(config.hidden_dim, config.hidden_dim, bias=f"{prefix}.linear_v.bias" in state_dict) + self.linear_out = nn.Linear(config.hidden_dim, config.hidden_dim, bias=f"{prefix}.linear_out.bias" in state_dict) self.linear_pos = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) - _copy_linear_weight(self.linear_q, state_dict[f"{prefix}.linear_q.weight"]) - _copy_linear_weight(self.linear_k, state_dict[f"{prefix}.linear_k.weight"]) - _copy_linear_weight(self.linear_v, state_dict[f"{prefix}.linear_v.weight"]) - _copy_linear_weight(self.linear_out, state_dict[f"{prefix}.linear_out.weight"]) + _copy_linear_weight(self.linear_q, state_dict[f"{prefix}.linear_q.weight"], bias=state_dict.get(f"{prefix}.linear_q.bias")) + _copy_linear_weight(self.linear_k, state_dict[f"{prefix}.linear_k.weight"], bias=state_dict.get(f"{prefix}.linear_k.bias")) + _copy_linear_weight(self.linear_v, state_dict[f"{prefix}.linear_v.weight"], bias=state_dict.get(f"{prefix}.linear_v.bias")) + _copy_linear_weight( + self.linear_out, + state_dict[f"{prefix}.linear_out.weight"], + bias=state_dict.get(f"{prefix}.linear_out.bias"), + ) _copy_linear_weight(self.linear_pos, state_dict[f"{prefix}.linear_pos.weight"]) self.pos_bias_u = nn.Parameter(state_dict[f"{prefix}.pos_bias_u"].clone()) self.pos_bias_v = nn.Parameter(state_dict[f"{prefix}.pos_bias_v"].clone()) @@ -486,20 +504,42 @@ def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, super().__init__() hidden_dim = config.hidden_dim kernel = config.conv_kernel_size - self.pointwise_conv1 = nn.Conv1d(hidden_dim, hidden_dim * 2, kernel_size=1, bias=False) + self.pointwise_conv1 = nn.Conv1d( + hidden_dim, + hidden_dim * 2, + kernel_size=1, + bias=f"{prefix}.pointwise_conv1.bias" in state_dict, + ) self.depthwise_conv = nn.Conv1d( hidden_dim, hidden_dim, kernel_size=kernel, padding=kernel // 2, groups=hidden_dim, - bias=False, + bias=f"{prefix}.depthwise_conv.bias" in state_dict, ) self.batch_norm = nn.BatchNorm1d(hidden_dim, eps=1e-5, affine=True, track_running_stats=True) - self.pointwise_conv2 = nn.Conv1d(hidden_dim, hidden_dim, kernel_size=1, bias=False) - _copy_conv1d_weight(self.pointwise_conv1, state_dict[f"{prefix}.pointwise_conv1.weight"]) - _copy_conv1d_weight(self.depthwise_conv, state_dict[f"{prefix}.depthwise_conv.weight"]) - _copy_conv1d_weight(self.pointwise_conv2, state_dict[f"{prefix}.pointwise_conv2.weight"]) + self.pointwise_conv2 = nn.Conv1d( + hidden_dim, + hidden_dim, + kernel_size=1, + bias=f"{prefix}.pointwise_conv2.bias" in state_dict, + ) + _copy_conv1d_weight( + self.pointwise_conv1, + state_dict[f"{prefix}.pointwise_conv1.weight"], + bias=state_dict.get(f"{prefix}.pointwise_conv1.bias"), + ) + _copy_conv1d_weight( + self.depthwise_conv, + state_dict[f"{prefix}.depthwise_conv.weight"], + bias=state_dict.get(f"{prefix}.depthwise_conv.bias"), + ) + _copy_conv1d_weight( + self.pointwise_conv2, + state_dict[f"{prefix}.pointwise_conv2.weight"], + bias=state_dict.get(f"{prefix}.pointwise_conv2.bias"), + ) self.batch_norm.weight.data.copy_(state_dict[f"{prefix}.batch_norm.weight"].to(dtype=self.batch_norm.weight.dtype)) self.batch_norm.bias.data.copy_(state_dict[f"{prefix}.batch_norm.bias"].to(dtype=self.batch_norm.bias.dtype)) self.batch_norm.running_mean.data.copy_(