Fix PocketTTS prompt graph overflow

CMakeLists.txt

@@ -348,6 +348,14 @@ add_library(engine_runtime STATIC
     src/models/voxtral_realtime/text_decoder.cpp
     src/models/voxtral_realtime/session.cpp
     src/models/voxtral_realtime/loader.cpp
+    src/models/fish_audio/ar.cpp
+    src/models/fish_audio/assets.cpp
+    src/models/fish_audio/codec.cpp
+    src/models/fish_audio/generator.cpp
+    src/models/fish_audio/loader.cpp
+    src/models/fish_audio/prompt_builder.cpp
+    src/models/fish_audio/session.cpp
+    src/models/fish_audio/tokenizer_text.cpp
     src/models/heartmula/assets.cpp
     src/models/heartmula/codec.cpp
     src/models/heartmula/generator.cpp
@@ -371,6 +379,15 @@ add_library(engine_runtime STATIC
     src/models/higgs_audio_stt/postprocess.cpp
     src/models/higgs_audio_stt/session.cpp
     src/models/higgs_audio_stt/loader.cpp
+    src/models/higgs_audio_tts/ar.cpp
+    src/models/higgs_audio_tts/assets.cpp
+    src/models/higgs_audio_tts/codec.cpp
+    src/models/higgs_audio_tts/codebooks.cpp
+    src/models/higgs_audio_tts/generator.cpp
+    src/models/higgs_audio_tts/loader.cpp
+    src/models/higgs_audio_tts/sampler.cpp
+    src/models/higgs_audio_tts/session.cpp
+    src/models/higgs_audio_tts/tokenizer_text.cpp
     src/models/irodori_tts/assets.cpp
     src/models/irodori_tts/codec.cpp
     src/models/irodori_tts/condition_encoder.cpp
@@ -723,6 +740,7 @@ if (ENGINE_BUILD_WARMBENCH)
     add_engine_warmbench(chatterbox_warm_bench tests/chatterbox/chatterbox_warm_bench.cpp)
     add_engine_warmbench(citrinet_asr_warm_bench tests/citrinet_asr/citrinet_asr_warm_bench.cpp)
     add_engine_warmbench(higgs_audio_stt_warm_bench tests/higgs_audio_stt/higgs_audio_stt_warm_bench.cpp)
+    add_engine_warmbench(higgs_audio_tts_warm_bench tests/higgs_audio_tts/higgs_audio_tts_warm_bench.cpp)
     add_engine_warmbench(hviske_asr_warm_bench tests/hviske_asr/hviske_asr_warm_bench.cpp)
     add_engine_warmbench(index_tts2_warm_bench tests/index_tts2/index_tts2_warm_bench.cpp)
     add_engine_warmbench(irodori_tts_warm_bench tests/irodori_tts/irodori_tts_warm_bench.cpp)
@@ -881,6 +899,16 @@ if (ENGINE_BUILD_TESTS)
         COMMAND encoder_module_test
     )
 
+    add_engine_unittest(
+        qwen_decoder_packed_projection_test
+        tests/unittests/test_qwen_decoder_packed_projections.cpp
+    )
+
+    add_test(
+        NAME qwen_decoder_packed_projection_test
+        COMMAND qwen_decoder_packed_projection_test
+    )
+
     add_engine_unittest(conv_transpose_fast_path_test tests/unittests/test_conv_transpose_fast_path.cpp)
 
     add_test(

README.md

@@ -51,8 +51,10 @@ audio.cpp would not be moving this quickly without generous contributors bringin
 | **ace_step** | music generation, music editing | 50+ langs | ACE-Step 1.5 Turbo and Base with acestep-5Hz-lm-1.7B |
 | **chatterbox** | TTS, voice cloning, voice conversion | ar, da, de, el, en, es, fi, fr, hi, it, ko, ms, nl, no, pl, pt, sv, sw, tr | Chatterbox with 0.5B backbone |
 | **citrinet_asr** | ASR | en | Citrinet-256 |
+| **fish_audio** | TTS, voice cloning | auto, en, zh | Fish Audio S2 Pro |
 | **heartmula** | music generation | zh, en, ja, ko, es | HeartMuLa-oss-3B with HeartCodec-oss |
 | **higgs_audio_stt** | ASR | en | Higgs Audio v3 STT |
+| **higgs_audio_tts** | TTS, voice cloning | auto | Higgs Audio v3 TTS 4B |
 | **htdemucs** | source separation | lang agnostic | HTDemucs, HTDemucs_ft |
 | **hviske_asr** | ASR | da | Hviske v5.3 |
 | **marblenet_vad** | VAD | lang agnostic | MarbleNet VAD |
@@ -90,8 +92,6 @@ Community model ports live under `community_models` to make the ownership bounda
 | **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support |
 | **vietneu_tts** | TTS, voice cloning | vi, en | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](vietneu_tts.md) TTS and voice cloning support |
 
-WIP (loaders not registered in this release tree — catalog entries are `UnsupportedSource`): Kokoro 82M bf16, Higgs Audio v3 TTS 4B, Parakeet TDT 0.6B v3, Fish Audio S2 Pro. See [docs/maintainers/loader_and_catalog.md](docs/maintainers/loader_and_catalog.md).
-
 PocketTTS language selection is a model-load option. When the model path points at the PocketTTS root, the loader uses `english` unless you pass `--load-option language=<name>`. Kyutai's normal non-English PocketTTS releases are smaller distilled language models intended for the fast PocketTTS path. The `_24l` variants are larger 24-layer, undistilled preview models that can sound better but are slower. Kyutai currently publishes French only as `french_24l`, not as a normal distilled `french` language directory, so French is not listed as a normal PocketTTS language here.
 
 ## Docker
@@ -373,20 +373,22 @@ Recommended top-level install packages:
 
 `Yes` means Hugging Face has a ready-to-use repo that the framework can download as-is. `No` means the tool must assemble, convert, or post-process files before the framework can use them. Packages whose loaders are not registered in this release tree are listed as **Unavailable** (see [docs/maintainers/loader_and_catalog.md](docs/maintainers/loader_and_catalog.md)).
 
+For shared audio.cpp GGUF packages, the model manager installs the default Q8_0 GGUF. Other precision variants can be downloaded directly from [audio-cpp/audio.cpp-gguf](https://huggingface.co/audio-cpp/audio.cpp-gguf); see [docs/gguf.md](docs/gguf.md) for GGUF support status.
+
 | Package id | Model | HF ready-to-use repo |
 |---|---|---|
 | `ace_step` | ACE-Step 1.5 Turbo/Base | No |
 | `chatterbox` | Chatterbox | **Yes** |
 | `citrinet_asr` | Citrinet ASR converted layout | No |
+| `fish_audio_s2_pro` | Fish Audio S2 Pro GGUF Q8_0 | **Yes** |
 | `heartmula` | HeartMuLa | No |
 | `higgs_audio_stt` | Higgs Audio STT | No |
-| `higgs_audio_v3_tts_4b` | Higgs Audio v3 TTS 4B | Unavailable (loader not in this tree) |
+| `higgs_audio_v3_tts_4b` | Higgs Audio v3 TTS 4B GGUF Q8_0 | **Yes** |
 | `htdemucs` | HTDemucs | No |
 | `hviske_asr` | Hviske ASR | **Yes** |
 | `irodori_tts_500m_v3` | Irodori-TTS 500M v3 | No |
 | `irodori_tts_600m_v3_voice_design` | Irodori-TTS 600M v3 VoiceDesign | No |
 | `index_tts2` | IndexTTS-2 | **Yes** |
-| `kokoro_82m_bf16` | Kokoro 82M bf16 | Unavailable (loader not in this tree) |
 | `marblenet_vad` | MarbleNet VAD converted layout | No |
 | `mel_band_roformer` | Mel-Band RoFormer MLX | **Yes** |
 | `miocodec_25hz_44k_v2` | MioCodec 25Hz 44.1kHz v2 | No |
@@ -399,7 +401,6 @@ Recommended top-level install packages:
 | `nemotron_asr` | Nemotron ASR | **Yes** |
 | `omnivoice` | OmniVoice | **Yes** |
 | `outetts_1_0_1b` | OuteTTS 1.0 1B with IBM DAC codec and Qwen3-aligned voice cloning | No |
-| `parakeet_tdt_0_6b_v3` | Parakeet TDT 0.6B v3 | Unavailable (loader not in this tree) |
 | `pocket_tts` | PocketTTS | **Yes** |
 | `qwen3_asr_0_6b` | Qwen3 ASR 0.6B | **Yes** |
 | `qwen3_asr_1_7b_hf` | Qwen3 ASR 1.7B HF | **Yes** |
@@ -419,7 +420,7 @@ Recommended top-level install packages:
 | `vibevoice_1_5b` | VibeVoice 1.5B | No |
 | `vibevoice_7b` | VibeVoice 7B | No |
 | `vibevoice_asr` | VibeVoice ASR | No |
-| `voxtral_realtime` | Voxtral Mini 4B Realtime | **Yes** |
+| `voxtral_realtime` | Voxtral Mini 4B Realtime GGUF Q8_0 | **Yes** |
 | `voxcpm2` | VoxCPM2 | No |
 
 > [!WARNING]
@@ -612,7 +613,6 @@ For TTS-family models, the measured one-shot RTF is:
 | model | audio len (s) | wall time (s) | RTF | x faster than real time |
 |---|---:|---:|---:|---:|
 | chatterbox | 9.72 | 2.45 | 0.252 | 3.97x |
-| kokoro tts | 10.15 | 0.64 | 0.063 | 15.90x |
 | miotts | 20.40 | 3.30 | 0.162 | 6.18x |
 | moss_tts_local | 9.60 | 0.97 | 0.101 | 9.91x |
 | omnivoice | 9.00 | 1.32 | 0.146 | 6.84x |
@@ -627,7 +627,6 @@ For long-form TTS tests, each run uses the same 6,026-character, 1,028-word inpu
 | model | audio len (s) | wall time (s) | RTF | x faster than real time |
 |---|---:|---:|---:|---:|
 | chatterbox | 391.24 | 58.57 | 0.150 | 6.68x |
-| kokoro tts | 371.17 | 7.19 | 0.019 | 51.60x |
 | index tts2 | 422.12 | 139.95 | 0.332 | 3.02x |
 | miotts | 399.16 | 66.59 | 0.167 | 5.99x |
 | moss_tts_nano | 391.20 | 43.16 | 0.110 | 9.06x |

docs/asr.md

@@ -255,12 +255,12 @@ audiocpp_cli --task asr --family vibevoice_asr --model models/VibeVoice-ASR --ba
 
 ## Voxtral Realtime
 
-Voxtral Realtime is a Mistral realtime ASR model with offline and streaming sessions. It accepts the native Hugging Face model directory and standalone audio.cpp GGUF packages.
+Voxtral Realtime is a Mistral realtime ASR model with offline and streaming sessions. The model manager installs the Q8_0 standalone GGUF package by default; native Hugging Face directories and other standalone GGUF variants can also be used when provided directly.
 
 | Field | Value |
 |---|---|
 | Family | `voxtral_realtime` |
-| Model directory | `models/Voxtral-Mini-4B-Realtime-2602` or a standalone Voxtral GGUF |
+| Model path | `models/Voxtral-Mini-4B-Realtime-2602-GGUF/voxtral-mini-4b-realtime-2602-q8_0.gguf` when installed through the model manager |
 | Task | `asr` |
 | Modes | `offline`, `streaming` |
 | Output | Transcription text |
@@ -270,19 +270,19 @@ Voxtral Realtime is a Mistral realtime ASR model with offline and streaming sess
 Offline CLI:
 
 ```bash
-audiocpp_cli --task asr --family voxtral_realtime --model <VOXTRAL_MODEL> --backend cuda --threads 8 --audio assets/resources/sample.wav --text-out transcript.txt
+audiocpp_cli --task asr --family voxtral_realtime --model models/Voxtral-Mini-4B-Realtime-2602-GGUF/voxtral-mini-4b-realtime-2602-q8_0.gguf --backend cuda --threads 8 --audio assets/resources/sample.wav --text-out transcript.txt
 ```
 
 Sampling and token-cap options can be passed through request options:
 
 ```bash
-audiocpp_cli --task asr --family voxtral_realtime --model <VOXTRAL_MODEL> --backend cuda --threads 8 --audio assets/resources/sample.wav --text-out transcript.txt --request-option max_new_tokens=256 --do-sample false --temperature 1.0 --top-p 1.0 --top-k 50 --seed 1234
+audiocpp_cli --task asr --family voxtral_realtime --model models/Voxtral-Mini-4B-Realtime-2602-GGUF/voxtral-mini-4b-realtime-2602-q8_0.gguf --backend cuda --threads 8 --audio assets/resources/sample.wav --text-out transcript.txt --request-option max_new_tokens=256 --do-sample false --temperature 1.0 --top-p 1.0 --top-k 50 --seed 1234
 ```
 
 Streaming CLI:
 
 ```bash
-audiocpp_cli --task asr --family voxtral_realtime --model <VOXTRAL_MODEL> --backend cuda --threads 8 --mode streaming --audio assets/resources/sample.wav --text-out transcript.txt
+audiocpp_cli --task asr --family voxtral_realtime --model models/Voxtral-Mini-4B-Realtime-2602-GGUF/voxtral-mini-4b-realtime-2602-q8_0.gguf --backend cuda --threads 8 --mode streaming --audio assets/resources/sample.wav --text-out transcript.txt
 ```
 
 Streaming server config:
@@ -299,7 +299,7 @@ Streaming server config:
     {
       "id": "voxtral-stream",
       "family": "voxtral_realtime",
-      "path": "/path/to/Voxtral-Mini-4B-Realtime-2602",
+      "path": "/path/to/voxtral-mini-4b-realtime-2602-q8_0.gguf",
       "task": "asr",
       "mode": "streaming"
     }

docs/gguf.md

@@ -259,8 +259,10 @@ Status labels:
 | `ace_step` | No | --- | --- | --- | --- |
 | `chatterbox` | No | --- | --- | --- | --- |
 | `citrinet_asr` | Done | Pass | --- | --- | Pass |
+| `fish_audio` | Done | Pass | --- | Pass | Pass |
 | `heartmula` | No | --- | --- | --- | --- |
 | `higgs_audio_stt` | Done | Pass | --- | Pass | Pass |
+| `higgs_audio_tts` | Done | Pass | --- | Pass | Pass |
 | `htdemucs` | Done | Pass | --- | Pass | Pass (drift) |
 | `hviske_asr` | Done | Pass | --- | --- | Pass |
 | `index_tts2` | Done | Pass | Pass | Pass (drift) | No (similarity drift, frame drift, text minor drift) |

docs/tts.md

@@ -10,6 +10,8 @@
 | OmniVoice | `omnivoice` | `tts` | [OmniVoice](#omnivoice) |
 | PocketTTS | `pocket_tts` | `tts` | [PocketTTS](#pockettts) |
 | VoxCPM2 | `voxcpm2` | `tts`, `vdes` | [VoxCPM2](#voxcpm2) |
+| Higgs Audio v3 TTS | `higgs_audio_tts` | `tts` | [Higgs Audio v3 TTS](#higgs-audio-v3-tts) |
+| Fish Audio S2 Pro | `fish_audio` | `tts` | [Fish Audio S2 Pro](#fish-audio-s2-pro) |
 | IndexTTS2 | `index_tts2` | `tts` | [IndexTTS2](#indextts2) |
 | Irodori-TTS | `irodori_tts` | `tts`, `vdes` | [Irodori-TTS](#irodori-tts) |
 | OuteTTS | `outetts` | `tts`, `clon` | [OuteTTS](#outetts) |
@@ -297,6 +299,89 @@ audiocpp_cli --task tts --family voxcpm2 --model models/VoxCPM2 --backend cuda -
 | `--num-inference-steps` | integer | `10` | Flow matching steps. |
 | `--guidance-scale` | float | `2.0` | CFG strength. |
 
+## Higgs Audio v3 TTS
+
+Higgs Audio v3 TTS is a voice-clone TTS model. The current integration uses the framework chunker for long text and keeps the reference prompt state in the model session.
+
+| Field | Value |
+|---|---|
+| Family | `higgs_audio_tts` |
+| Model path | `models/Higgs-Audio-v3-TTS-4B-GGUF/higgs-audio-v3-tts-4b-q8_0.gguf` when installed through the model manager |
+| Task | `tts` |
+| Modes | `offline` |
+| Languages | Model auto-handles supported languages |
+| Voice input | Reference WAV through `--voice-ref`; transcript through `--reference-text` when known |
+| Built-in voices | Not exposed |
+
+```bash
+audiocpp_cli --task tts --family higgs_audio_tts --model models/Higgs-Audio-v3-TTS-4B-GGUF/higgs-audio-v3-tts-4b-q8_0.gguf --backend cuda --text "Hello from Higgs Audio." --voice-ref assets/resources/b.wav --reference-text "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you." --out out.wav
+```
+
+The model manager installs the Q8_0 standalone GGUF package by default:
+
+```bash
+python3 tools/model_manager.py install --models-root models higgs_audio_v3_tts_4b
+```
+
+| Option | Values | Default | Meaning |
+|---|---|---:|---|
+| `--voice-ref` | WAV path | required | Reference speaker audio. |
+| `--reference-text` | text | empty string | Transcript for reference audio. |
+| `--text-chunk-size` | integer chars | `1024` | Long-form chunk size. |
+| `--max-tokens` | integer | `2048` | Maximum generated AR tokens per chunk. |
+| `--temperature` | float | `0.8` | AR sampling temperature. |
+| `--top-k` | integer | `30` | AR top-k sampling limit. The narrower default is less prone to premature EOC than the Python client's `50`. |
+| `--top-p` | float | `0.8` | AR nucleus sampling limit. The Python client's unfiltered equivalent is `1.0`. |
+| `--repetition-penalty` | float | `1.1` | Accepted for Python API compatibility; Higgs audio-code sampling does not consume it. |
+
+## Fish Audio S2 Pro
+
+Fish Audio S2 Pro is a TTS and reference voice-clone model. The integration uses the framework text chunker for long-form input, caches prepared reference audio in the session, and supports GGUF loading through the package spec path.
+
+| Field | Value |
+|---|---|
+| Family | `fish_audio` |
+| Model path | `models/Fish-Audio-S2-Pro-GGUF/fish-audio-s2-pro-q8_0.gguf` when installed through the model manager |
+| Task | `tts` |
+| Modes | `offline` |
+| Languages | Model auto-handles language; tested paths cover English and Chinese-style prompts |
+| Voice input | Optional reference WAV through `--voice-ref`; transcript through `--reference-text` when known |
+| Built-in voices | Not exposed |
+
+Text-to-speech:
+
+```bash
+audiocpp_cli --task tts --family fish_audio --model models/Fish-Audio-S2-Pro-GGUF/fish-audio-s2-pro-q8_0.gguf --backend cuda --text "Hello from Fish Audio." --out out.wav
+```
+
+Reference voice clone:
+
+```bash
+audiocpp_cli --task tts --family fish_audio --model models/Fish-Audio-S2-Pro-GGUF/fish-audio-s2-pro-q8_0.gguf --backend cuda --text "The final render is ready for review." --voice-ref assets/resources/b.wav --reference-text "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you." --out out.wav
+```
+
+The model manager installs the Q8_0 standalone GGUF package by default:
+
+```bash
+python3 tools/model_manager.py install --models-root models fish_audio_s2_pro
+```
+
+| Option | Values | Default | Meaning |
+|---|---|---:|---|
+| `--voice-ref` | WAV path | not set | Reference speaker audio for voice cloning. |
+| `--reference-text` | text | empty string | Transcript for reference audio. |
+| `--max-new-tokens` | integer | `1024` | Maximum generated semantic tokens per chunk. `0` uses the default. |
+| `--text-chunk-size` | integer chars | `200` | Long-form chunk size. |
+| `--text-chunk-mode` | `default`, `tag_aware`, `japanese`, `endline` | `default` | Framework text chunking mode. |
+| `--temperature` | float | `0.8` | Sampling temperature. |
+| `--top-k` | integer | `30` | Top-k sampling limit. |
+| `--top-p` | float | `0.8` | Nucleus sampling limit. |
+| `--seed` | integer | random when omitted | Sampling seed for reproducible output. |
+| `--session-option fish_audio.mem_saver=true|false` | bool | `false` | Release cached AR runtime graphs after each request. |
+| `--session-option fish_audio.reference_cache_slots=<n>` | integer | `1` | Prepared reference-audio cache slots. |
+| `--session-option fish_audio.weight_type=<type>` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | AR matmul weight storage type. |
+| `--session-option fish_audio.codec_weight_type=<type>` | `native`, `f32`, `f16`, `q8_0` | `native` | Codec conv/matmul weight storage type. |
+
 ## IndexTTS2
 
 IndexTTS2 is a Chinese and English TTS model with voice cloning and expressive emotion controls. It requires a speaker reference through the framework `--voice-ref` path.

include/engine/framework/core/backend.h

@@ -70,10 +70,16 @@ void write_tensor_f32_slice(const TensorValue & tensor, size_t element_offset, c
 void write_tensor_f32(const TensorValue & tensor, const std::vector<float> & values);
 void write_tensor_f16(const TensorValue & tensor, const float * values, size_t count);
 void write_tensor_f16(const TensorValue & tensor, const std::vector<float> & values);
+void write_tensor_bf16(const TensorValue & tensor, const float * values, size_t count);
+void write_tensor_bf16(const TensorValue & tensor, const std::vector<float> & values);
 void write_tensor_i32(const TensorValue & tensor, const int32_t * values, size_t count);
 void write_tensor_i32(const TensorValue & tensor, const std::vector<int32_t> & values);
 void read_tensor_f32_into(const ggml_tensor * tensor, std::vector<float> & values);
 std::vector<float> read_tensor_f32(const ggml_tensor * tensor);
+void read_tensor_f16_into(const ggml_tensor * tensor, std::vector<float> & values);
+std::vector<float> read_tensor_f16(const ggml_tensor * tensor);
+void read_tensor_bf16_into(const ggml_tensor * tensor, std::vector<float> & values);
+std::vector<float> read_tensor_bf16(const ggml_tensor * tensor);
 void read_tensor_i32_into(const ggml_tensor * tensor, std::vector<int32_t> & values);
 std::vector<int32_t> read_tensor_i32(const ggml_tensor * tensor);
 

include/engine/framework/modules/attention/qwen_causal_decoder.h

@@ -79,6 +79,11 @@ std::vector<int32_t> qwen_position_ids(int64_t steps, int64_t offset = 0);
 
 std::vector<ggml_fp16_t> qwen_causal_prefill_mask_values(int64_t batch_size, int64_t steps);
 
+std::vector<ggml_fp16_t> qwen_causal_suffix_mask_values(
+    int64_t batch_size,
+    int64_t query_steps,
+    int64_t prefix_steps);
+
 void write_qwen_causal_prefill_mask(
     ggml_tensor * tensor,
     int64_t batch_size,