diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 0225d792f..b1ddab19f 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -730,6 +730,7 @@ class Model { struct ParakeetTdtStreamState { bool initialized = false; + bool internal = false; uint32_t last_token = 0; size_t time_index = 0; std::vector> dec_state; @@ -744,6 +745,10 @@ class Model { bool is_final = true, size_t end_frame = 0, const std::atomic* should_stop = nullptr); + std::vector transcribe_parakeet_tdt_longform(const std::vector& audio_features, + size_t window_frames, size_t mels, + size_t source_frames, + const std::atomic* should_stop); std::vector transcribe_whisper_seq2seq(const std::vector& audio_features, const std::vector& decoder_prompt_tokens, size_t max_tokens, diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 2fc60d1e4..92e3a2b23 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -4747,6 +4747,10 @@ std::vector Model::transcribe_parakeet_tdt(const std::vector& a const size_t expected_frames = feat_desc.shape[1]; const size_t expected_mels = feat_desc.shape[2]; const size_t source_frames = expected_mels > 0 ? audio_features.size() / expected_mels : 0; + if (stream == nullptr && source_frames > expected_frames) { + return transcribe_parakeet_tdt_longform(audio_features, expected_frames, expected_mels, + source_frames, should_stop); + } const size_t copy_frames = std::min(source_frames, expected_frames); std::vector transposed(expected_frames * expected_mels, 0.0f); for (size_t t = 0; t < copy_frames; ++t) { @@ -4919,7 +4923,7 @@ std::vector Model::transcribe_parakeet_tdt(const std::vector& a float best_token_score = -std::numeric_limits::infinity(); for (size_t i = 0; i < token_class_count; ++i) { float v = get_logit(i); - if (stream && !tdt_vocab_bias.empty()) { + if (stream && !stream->internal && !tdt_vocab_bias.empty()) { auto it = tdt_vocab_bias.find(static_cast(i)); if (it != tdt_vocab_bias.end()) v += it->second; } @@ -4992,6 +4996,58 @@ std::vector Model::transcribe_parakeet_tdt(const std::vector& a return emitted; } +std::vector Model::transcribe_parakeet_tdt_longform(const std::vector& audio_features, + size_t window_frames, size_t mels, + size_t source_frames, + const std::atomic* should_stop) { + // Audio longer than the encoder window: slice the precomputed mel features into + // overlapping windows and carry decoder state across them, reusing the streaming + // path's left-context / word-boundary-confirm / rollback scheme (see stream.cpp). + const size_t sub = std::max(1, config_.subsampling_factor); + // Left context must stay a multiple of the subsampling factor so the decode + // cursor lands exactly on an encoder frame. + const size_t left_ctx = (std::min(window_frames / 2, static_cast(100 * sub)) / sub) * sub; + const size_t right_ctx = 100; // 1 s look-ahead encoded but not committed on non-final windows + CACTUS_LOG_INFO("model", "Parakeet long-form: " << source_frames << " mel frames exceed encoder window " + << window_frames << "; decoding in overlapping windows"); + + std::vector all_tokens; + ParakeetTdtStreamState st; + st.internal = true; + std::vector window; + size_t mel_cursor = 0; + while (mel_cursor < source_frames) { + if (should_stop && should_stop->load()) break; + const size_t win_start = mel_cursor > left_ctx ? mel_cursor - left_ctx : 0; + const size_t win_end = std::min(win_start + window_frames, source_frames); + const bool is_last = win_end == source_frames; + const size_t frames = win_end - win_start; + const size_t end_frame = (is_last || frames <= right_ctx) ? 0 : (frames - right_ctx) / sub; + window.assign(frames * mels, 0.0f); + for (size_t m = 0; m < mels; ++m) { + const float* src = audio_features.data() + m * source_frames + win_start; + std::copy(src, src + frames, window.data() + m * frames); + } + st.time_index = (mel_cursor - win_start) / sub; + std::vector tokens = + transcribe_parakeet_tdt(window, &st, is_last, end_frame, should_stop); + all_tokens.insert(all_tokens.end(), tokens.begin(), tokens.end()); + if (is_last || (should_stop && should_stop->load())) break; + size_t next_cursor = win_start + st.time_index * sub; + if (next_cursor <= mel_cursor) { + // Nothing was confirmed (silence, or no word boundary in the window): + // finalize this window so the cursor can advance. + st.time_index = (mel_cursor - win_start) / sub; + tokens = transcribe_parakeet_tdt(window, &st, true, 0, should_stop); + all_tokens.insert(all_tokens.end(), tokens.begin(), tokens.end()); + next_cursor = win_start + st.time_index * sub; + if (next_cursor <= mel_cursor) break; + } + mel_cursor = std::min(next_cursor, win_end); + } + return all_tokens; +} + uint32_t Model::decode_with_images(const std::vector& tokens, const std::vector& /*image_paths*/, float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, float min_p, float repetition_penalty) { diff --git a/cactus-engine/tests/test_stt.cpp b/cactus-engine/tests/test_stt.cpp index d66ef894d..ca582a915 100644 --- a/cactus-engine/tests/test_stt.cpp +++ b/cactus-engine/tests/test_stt.cpp @@ -64,9 +64,47 @@ bool test_transcription() { return passed; } +bool test_transcription_long_form() { + std::cout << "\n╔══════════════════════════════════════════╗\n" + << "║ LONG-FORM TRANSCRIPTION TEST ║\n" + << "╚══════════════════════════════════════════╝\n"; + + cactus_model_t model = cactus_init(g_transcription_model_path, nullptr, false); + if (!model) { + std::cerr << "[✗] Failed to initialize model\n"; + return false; + } + + // test_long.wav is 75.7 s — far beyond one encoder window (~20 s). A truncated + // decode yields well under 100 words; a full decode yields several hundred. + std::string audio_path = std::string(g_assets_path) + "/test_long.wav"; + std::vector response(1 << 17, 0); + + int rc = cactus_transcribe(model, audio_path.c_str(), nullptr, + response.data(), response.size(), g_options, + nullptr, nullptr, nullptr, 0); + std::string transcript = json_string(std::string(response.data()), "response"); + size_t words = 0; + bool in_word = false; + for (char c : transcript) { + bool space = (c == ' ' || c == '\n' || c == '\t'); + if (!space && !in_word) ++words; + in_word = !space; + } + std::cout << "├─ Words: " << words << "\n"; + std::cout << "├─ Transcript: " << transcript.substr(0, 120) << "...\n"; + cactus_destroy(model); + + bool passed = rc > 0 && words > 120; + if (!passed) std::cerr << "[✗] Long-form transcription truncated or failed (words=" << words << ")\n"; + std::cout << "└─ Status: " << (passed ? "PASSED ✓" : "FAILED ✗") << "\n"; + return passed; +} + int main() { TestUtils::TestRunner runner("STT Tests"); runner.run_test("transcription", test_transcription()); + runner.run_test("transcription_long_form", test_transcription_long_form()); runner.print_summary(); return runner.all_passed() ? 0 : 1; }