Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cactus-engine/src/embed.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ static std::vector<float> compute_mel_from_wav(const std::string& wav_path, size

int norm_type = is_v3 ? 1 : 1;
int scale_type = is_v3 ? 2 : 2;
cactus::audio::pad_or_trim_whisper_waveform(waveform_16k);
std::vector<float> mel = cactus::audio::compute_spectrogram_graph(
waveform_16k, cfg, num_mel_filters, 0.0f, 8000.0f, 16000, norm_type, scale_type);
if (mel.empty()) return mel;
Expand Down
14 changes: 14 additions & 0 deletions cactus-engine/src/engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <unordered_set>
#include <memory>
#include <cstdint>
#include <atomic>

#include "cactus_graph.h"

Expand Down Expand Up @@ -605,6 +606,11 @@ class Model {
float* out_token_time_start = nullptr, float* out_token_time_end = nullptr);

std::vector<uint32_t> transcribe_parakeet_tdt(const std::vector<float>& audio_features);
std::vector<uint32_t> transcribe_whisper_seq2seq(const std::vector<float>& audio_features,
const std::vector<uint32_t>& decoder_prompt_tokens,
size_t max_tokens,
const std::vector<std::vector<uint32_t>>& stop_token_sequences,
const std::atomic<bool>* should_stop = nullptr);

std::vector<float> get_embeddings(const std::vector<uint32_t>& tokens, bool pooled = true,
bool normalize = false, const std::string& profile_file = "");
Expand Down Expand Up @@ -669,6 +675,7 @@ class Model {
};

void copy_cache_state(const Component& src, Component& dst);
void reset_component_cache_states(Component& comp);

bool load_manifest();
bool setup_tokenizer();
Expand All @@ -688,6 +695,11 @@ class Model {
bool run_chunk_prefill_path(const std::vector<uint32_t>& tokens,
const std::vector<std::string>& image_paths,
const std::vector<float>& audio_features);
bool build_lm_encoder_outputs_dynamic_gemma4(
const std::vector<uint32_t>& tokens,
std::map<std::string, std::vector<uint8_t>>& store_bytes,
std::map<std::string, Precision>& store_prec,
std::map<std::string, std::vector<size_t>>& store_shape);

std::string bundle_dir_;
std::map<std::string, Component> components_;
Expand All @@ -698,6 +710,8 @@ class Model {
Component* lm_encoder_media_step_ = nullptr;
Component* decoder_prefill_chunk_ = nullptr;
Component* lm_encoder_ = nullptr;
Component* lm_encoder_text_chunk_ = nullptr;
Component* lm_encoder_media_chunk_ = nullptr;

std::string family_;

Expand Down
339 changes: 319 additions & 20 deletions cactus-engine/src/model.cpp

Large diffs are not rendered by default.

31 changes: 28 additions & 3 deletions cactus-engine/src/transcribe.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ int cactus_preprocess_audio_features(
cactus::audio::trim_mel_frames(features, bins, valid_frames);
frames = features.size() / bins;
} else {
cactus::audio::pad_or_trim_whisper_waveform(audio_samples);
auto cfg = cactus::audio::get_whisper_spectrogram_config();
const bool is_whisper_v3 = bins > 80;
if (is_whisper_v3) cactus::audio::apply_whisper_v3_overrides(cfg);
Expand Down Expand Up @@ -192,6 +193,7 @@ int cactus_transcribe(
handle_error_response("No audio input provided", response_buffer, buffer_size);
return -1;
}
const size_t original_audio_sample_count = audio_samples.size();

const size_t mel_bins = std::max<size_t>(1, static_cast<size_t>(handle->model->get_config().num_mel_bins));
std::vector<float> audio_features;
Expand All @@ -207,6 +209,7 @@ int cactus_transcribe(
if (valid_frames == 0) valid_frames = 1;
cactus::audio::trim_mel_frames(audio_features, mel_bins, valid_frames);
} else {
cactus::audio::pad_or_trim_whisper_waveform(audio_samples);
auto cfg = cactus::audio::get_whisper_spectrogram_config();
const bool is_whisper_v3 = mel_bins > 80;
if (is_whisper_v3) cactus::audio::apply_whisper_v3_overrides(cfg);
Expand Down Expand Up @@ -253,7 +256,7 @@ int cactus_transcribe(
append_exact_stop_sequence("<pad>");

const float audio_length_sec =
static_cast<float>(audio_samples.size()) / static_cast<float>(cactus::audio::WHISPER_SAMPLE_RATE);
static_cast<float>(original_audio_sample_count) / static_cast<float>(cactus::audio::WHISPER_SAMPLE_RATE);
if (options.max_tokens == 100 && (!options_json || std::string(options_json).find("\"max_tokens\"") == std::string::npos)) {
options.max_tokens = std::max<size_t>(100, static_cast<size_t>(audio_length_sec * (is_parakeet ? 30.0f : 20.0f)));
}
Expand Down Expand Up @@ -282,13 +285,35 @@ int cactus_transcribe(
callback(piece.c_str(), tok, user_data);
}
}
} else if (is_whisper) {
generated_tokens = handle->model->transcribe_whisper_seq2seq(
audio_features,
tokens,
options.max_tokens,
stop_token_sequences,
&handle->should_stop);
auto t_first = std::chrono::high_resolution_clock::now();
time_to_first_token =
std::chrono::duration_cast<std::chrono::microseconds>(t_first - start_time).count() / 1000.0;
for (const auto& stop_seq : stop_token_sequences) {
if (stop_seq.empty() || generated_tokens.size() < stop_seq.size()) continue;
if (std::equal(stop_seq.rbegin(), stop_seq.rend(), generated_tokens.rbegin())) {
generated_tokens.resize(generated_tokens.size() - stop_seq.size());
break;
}
}
final_text = tokenizer->decode(generated_tokens);
if (callback) {
for (uint32_t tok : generated_tokens) {
std::string piece = tokenizer->decode({tok});
callback(piece.c_str(), tok, user_data);
}
}
} else {
for (size_t i = 0; i < options.max_tokens; ++i) {
if (handle->should_stop) break;

float token_entropy = 0.0f;
float tok_time_start = 0.0f;
float tok_time_end = 0.0f;
uint32_t next_token = handle->model->decode_with_audio(
tokens, audio_features,
options.temperature, options.top_p, options.top_k,
Expand Down
20 changes: 20 additions & 0 deletions cactus-engine/src/utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,15 @@ namespace audio {

static constexpr size_t WHISPER_TARGET_FRAMES = 3000;
static constexpr int WHISPER_SAMPLE_RATE = 16000;
static constexpr size_t WHISPER_TARGET_SAMPLES = WHISPER_SAMPLE_RATE * 30;

inline void pad_or_trim_whisper_waveform(std::vector<float>& samples) {
if (samples.size() > WHISPER_TARGET_SAMPLES) {
samples.resize(WHISPER_TARGET_SAMPLES);
} else if (samples.size() < WHISPER_TARGET_SAMPLES) {
samples.resize(WHISPER_TARGET_SAMPLES, 0.0f);
}
}

inline cactus::engine::SpectrogramConfig get_whisper_spectrogram_config() {
cactus::engine::SpectrogramConfig cfg{};
Expand Down Expand Up @@ -313,6 +322,17 @@ inline std::vector<float> normalize_whisper_mel(std::vector<float>& mel, size_t
if (mel.empty() || n_mels == 0) return mel;
size_t n_frames = mel.size() / n_mels;

if (n_frames > WHISPER_TARGET_FRAMES) {
std::vector<float> trimmed(n_mels * WHISPER_TARGET_FRAMES);
for (size_t m = 0; m < n_mels; ++m) {
const float* src = &mel[m * n_frames];
float* dst = &trimmed[m * WHISPER_TARGET_FRAMES];
std::copy(src, src + WHISPER_TARGET_FRAMES, dst);
}
mel = std::move(trimmed);
n_frames = WHISPER_TARGET_FRAMES;
}

float max_val = -std::numeric_limits<float>::infinity();
for (float v : mel) if (v > max_val) max_val = v;

Expand Down
10 changes: 9 additions & 1 deletion python/cactus/convert/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from .export.validate import validate_qdq
from .model_adapters.detection import SUPPORTED_FAMILIES, detect_family
from .model_adapters.adapters import adapter_for_family
from .model_adapters.nemo import ensure_parakeet_tdt_nemo_source
from .quantization.cq import quantize_hadamard, quantize_orthogonal, write_cq_tensor
from .compat import patch_transformers_import_compat

Expand All @@ -48,6 +49,9 @@ def _load_hf(model_id_or_path: str, device: str):
warnings.filterwarnings("ignore", message=".*You are using a model of type.*")
for note in patch_transformers_import_compat():
print(f"note={note}")
nemo_export = ensure_parakeet_tdt_nemo_source(model_id_or_path, cache_dir=_hf_cache_dir())
if nemo_export is not None:
model_id_or_path = nemo_export
try:
from transformers import AutoConfig, AutoModel
except Exception as exc: # pragma: no cover
Expand Down Expand Up @@ -123,6 +127,9 @@ def _bits_for_component(component: str, args: argparse.Namespace) -> int:


def _load_checkpoint_state_dict(model_id_or_path: str) -> dict[str, Any] | None:
nemo_export = ensure_parakeet_tdt_nemo_source(model_id_or_path, cache_dir=_hf_cache_dir())
if nemo_export is not None:
model_id_or_path = nemo_export
root = Path(model_id_or_path)
if not root.exists() or not root.is_dir():
try:
Expand Down Expand Up @@ -370,6 +377,7 @@ def convert(args: argparse.Namespace) -> None:
out_dir.mkdir(parents=True, exist_ok=True)

cfg, processor, model = _load_hf(args.model, args.device)
runtime_source = ensure_parakeet_tdt_nemo_source(args.model, cache_dir=_hf_cache_dir()) or args.model
family = detect_family(cfg, args.model_family)
adapter = adapter_for_family(family)
checkpoint_state = _load_checkpoint_state_dict(args.model)
Expand All @@ -382,7 +390,7 @@ def convert(args: argparse.Namespace) -> None:
model_config = adapter.runtime_config(cfg)
model_config["model_type"] = adapter.runtime_model_type()
write_config_txt({**_config_dict(cfg), **model_config}, out_dir)
copy_runtime_files(args.model, out_dir, token=getattr(args, "token", None), cache_dir=_hf_cache_dir())
copy_runtime_files(runtime_source, out_dir, token=getattr(args, "token", None), cache_dir=_hf_cache_dir())
try:
from .cactus_adapters.tokenizer import convert_hf_tokenizer

Expand Down
1 change: 1 addition & 0 deletions python/cactus/convert/export/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"merges.txt",
"tokenizer.json",
"tokenizer.model",
"vocab.txt",
"tokenizer_config.json",
"special_tokens_map.json",
"added_tokens.json",
Expand Down
37 changes: 19 additions & 18 deletions python/cactus/convert/model_adapters/adapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -389,24 +389,25 @@ def runtime_config(self, cfg: Any) -> dict[str, Any]:
def normalize_state_dict(self, state_dict: dict[str, Any]) -> NormalizedState:
augmented = dict(state_dict)
provenance: dict[str, TensorProvenance] = {}
i = 0
while True:
ih_key = f"decoder.lstm.bias_ih_l{i}"
hh_key = f"decoder.lstm.bias_hh_l{i}"
out_key = f"decoder.lstm.bias_l{i}"
if ih_key not in state_dict and hh_key not in state_dict:
break
sources = [k for k in (ih_key, hh_key) if k in state_dict]
if ih_key in state_dict and hh_key in state_dict:
augmented[out_key] = state_dict[ih_key] + state_dict[hh_key]
elif ih_key in state_dict:
augmented[out_key] = state_dict[ih_key]
else:
augmented[out_key] = state_dict[hh_key]
augmented.pop(ih_key, None)
augmented.pop(hh_key, None)
provenance[out_key] = TensorProvenance(sources, "parakeet_tdt_lstm_bias_sum", "hf_key")
i += 1
for prefix in ("decoder.lstm", "decoder.prediction.dec_rnn.lstm"):
i = 0
while True:
ih_key = f"{prefix}.bias_ih_l{i}"
hh_key = f"{prefix}.bias_hh_l{i}"
out_key = f"{prefix}.bias_l{i}"
if ih_key not in state_dict and hh_key not in state_dict:
break
sources = [k for k in (ih_key, hh_key) if k in state_dict]
if ih_key in state_dict and hh_key in state_dict:
augmented[out_key] = state_dict[ih_key] + state_dict[hh_key]
elif ih_key in state_dict:
augmented[out_key] = state_dict[ih_key]
else:
augmented[out_key] = state_dict[hh_key]
augmented.pop(ih_key, None)
augmented.pop(hh_key, None)
provenance[out_key] = TensorProvenance(sources, "parakeet_tdt_lstm_bias_sum", "hf_key")
i += 1
return NormalizedState(augmented, provenance)


Expand Down
4 changes: 4 additions & 0 deletions python/cactus/convert/model_adapters/naming.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,10 @@ def _parakeet_tdt_predictor_match(name: str) -> str | None:
if new:
suffix = {"weight_ih": "weight_ih.weights", "weight_hh": "weight_hh.weights", "bias": "bias.weights"}[new.group(1)]
return f"tdt_predictor_lstm_{int(new.group(2))}_{suffix}"
nemo = re.fullmatch(r"decoder\.prediction\.dec_rnn\.lstm\.(weight_ih|weight_hh|bias)_l(\d+)", name)
if nemo:
suffix = {"weight_ih": "weight_ih.weights", "weight_hh": "weight_hh.weights", "bias": "bias.weights"}[nemo.group(1)]
return f"tdt_predictor_lstm_{int(nemo.group(2))}_{suffix}"
return None


Expand Down
Loading
Loading