From f37c5ae71c3233bd0809100d8d6744c9488549a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 16:43:21 +0000 Subject: [PATCH 1/7] chore(native): sync native release v0.4.1-1 --- CHANGELOG.md | 6 + README.md | 2 +- lib/src/backends/llama_cpp/bindings.dart | 138 +++++++++++------- lib/src/hook/native_release_pins.dart | 2 +- .../llamadart_llama_cpp_flutter/CHANGELOG.md | 4 + .../llamadart_llama_cpp_flutter/README.md | 2 +- .../llamadart_llama_cpp_flutter/Package.swift | 4 +- website/docs/getting-started/installation.md | 6 +- website/docs/platforms/support-matrix.md | 6 +- 9 files changed, 104 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c86439c29..04fdec71e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## Unreleased +* Updated the default llama.cpp native runtime pin to + `leehack/llamadart-native@v0.4.1-1`, keeping the `v0.4.1` llama.cpp + ABI/bindings while picking up wrapper-only native fixes. Refreshed + the `llamadart_llama_cpp_flutter` Apple SwiftPM checksum and + aligned current README/website native override docs. + - Extend the GGUF speech-to-text validation pack with four synthetic edge fixtures built in-process, so no extra audio is stored: generated digital silence, plus a truncated RIFF, a stereo 44.1 kHz re-encode and a 33-second diff --git a/README.md b/README.md index dee222a86..3c7e09779 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Current default runtime pins: | Runtime | Pin | | --- | --- | -| Native llama.cpp / GGUF | `leehack/llamadart-native@v0.4.1` | +| Native llama.cpp / GGUF | `leehack/llamadart-native@v0.4.1-1` | | Native LiteRT-LM / `.litertlm` | `leehack/litert-lm-native@v0.17.0-6` | | Web llama.cpp / GGUF | `leehack/llama-web-bridge-assets@v0.1.44` | | Web LiteRT-LM / `.litertlm` | `@litert-lm/core@0.15.0` | diff --git a/lib/src/backends/llama_cpp/bindings.dart b/lib/src/backends/llama_cpp/bindings.dart index 7635b0b03..31d10f32b 100644 --- a/lib/src/backends/llama_cpp/bindings.dart +++ b/lib/src/backends/llama_cpp/bindings.dart @@ -6298,6 +6298,15 @@ external int llama_dart_tts_api_version(); @ffi.Native)>() external void llama_dart_tts_cancel(ffi.Pointer tts); +@ffi.Native< + ffi.Bool Function(ffi.Pointer, ffi.Bool, ffi.Pointer) +>() +external bool llama_dart_tts_eval_callback( + ffi.Pointer tensor, + bool ask, + ffi.Pointer user_data, +); + @ffi.Native)>() external void llama_dart_tts_free(ffi.Pointer tts); @@ -6403,6 +6412,23 @@ llama_dart_tts_status llama_dart_tts_reset(ffi.Pointer tts) { return llama_dart_tts_status.fromValue(_llama_dart_tts_reset(tts)); } +@ffi.Native< + ffi.Int Function(ffi.Pointer, ffi.Pointer) +>(symbol: 'llama_dart_tts_set_cancel_flag') +external int _llama_dart_tts_set_cancel_flag( + ffi.Pointer tts, + ffi.Pointer flag, +); + +llama_dart_tts_status llama_dart_tts_set_cancel_flag( + ffi.Pointer tts, + ffi.Pointer flag, +) { + return llama_dart_tts_status.fromValue( + _llama_dart_tts_set_cancel_flag(tts, flag), + ); +} + @ffi.Native< ffi.Int Function( ffi.Pointer, @@ -8719,7 +8745,7 @@ external int mtmd_tokenize_from_parts( bool add_special, ); -typedef FILE = __sFILE; +typedef FILE = _IO_FILE; const int GGML_BACKEND_META_MAX_DEVICES = 16; @@ -8858,88 +8884,90 @@ final class UnnamedUnion extends ffi.Union { external ffi.Array val_str; } -typedef __darwin_off_t = __int64_t; -typedef __int64_t = ffi.LongLong; -typedef Dart__int64_t = int; +final class _IO_FILE extends ffi.Struct { + @ffi.Int() + external int _flags; -final class __sFILE extends ffi.Struct { - external ffi.Pointer _p; + external ffi.Pointer _IO_read_ptr; - @ffi.Int() - external int _r; + external ffi.Pointer _IO_read_end; - @ffi.Int() - external int _w; + external ffi.Pointer _IO_read_base; - @ffi.Short() - external int _flags; + external ffi.Pointer _IO_write_base; - @ffi.Short() - external int _file; + external ffi.Pointer _IO_write_ptr; - external __sbuf _bf; + external ffi.Pointer _IO_write_end; - @ffi.Int() - external int _lbfsize; + external ffi.Pointer _IO_buf_base; - external ffi.Pointer _cookie; + external ffi.Pointer _IO_buf_end; - external ffi.Pointer< - ffi.NativeFunction)> - > - _close; + external ffi.Pointer _IO_save_base; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, ffi.Int) - > - > - _read; + external ffi.Pointer _IO_backup_base; - external ffi.Pointer< - ffi.NativeFunction, fpos_t, ffi.Int)> - > - _seek; + external ffi.Pointer _IO_save_end; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, ffi.Int) - > - > - _write; + external ffi.Pointer<_IO_marker> _markers; - external __sbuf _ub; + external ffi.Pointer<_IO_FILE> _chain; - external ffi.Pointer<__sFILEX> _extra; + @ffi.Int() + external int _fileno; @ffi.Int() - external int _ur; + external int _flags2; - @ffi.Array.multi([3]) - external ffi.Array _ubuf; + @__off_t() + external int _old_offset; - @ffi.Array.multi([1]) - external ffi.Array _nbuf; + @ffi.UnsignedShort() + external int _cur_column; - external __sbuf _lb; + @ffi.SignedChar() + external int _vtable_offset; - @ffi.Int() - external int _blksize; + @ffi.Array.multi([1]) + external ffi.Array _shortbuf; + + external ffi.Pointer<_IO_lock_t> _lock; - @fpos_t() + @__off64_t() external int _offset; -} -final class __sFILEX extends ffi.Opaque {} + external ffi.Pointer<_IO_codecvt> _codecvt; + + external ffi.Pointer<_IO_wide_data> _wide_data; + + external ffi.Pointer<_IO_FILE> _freeres_list; + + external ffi.Pointer _freeres_buf; -final class __sbuf extends ffi.Struct { - external ffi.Pointer _base; + @ffi.Size() + external int __pad5; @ffi.Int() - external int _size; + external int _mode; + + @ffi.Array.multi([20]) + external ffi.Array _unused2; } -typedef fpos_t = __darwin_off_t; +final class _IO_codecvt extends ffi.Opaque {} + +typedef _IO_lock_t = ffi.Void; +typedef Dart_IO_lock_t = void; + +final class _IO_marker extends ffi.Opaque {} + +final class _IO_wide_data extends ffi.Opaque {} + +typedef __off64_t = ffi.Long; +typedef Dart__off64_t = int; +typedef __off_t = ffi.Long; +typedef Dart__off_t = int; typedef ggml_abort_callback = ffi.Pointer>; typedef ggml_abort_callbackFunction = diff --git a/lib/src/hook/native_release_pins.dart b/lib/src/hook/native_release_pins.dart index 92ef44d98..38ee2afcf 100644 --- a/lib/src/hook/native_release_pins.dart +++ b/lib/src/hook/native_release_pins.dart @@ -5,7 +5,7 @@ library; /// `leehack/llamadart-native` release tag downloaded for the llama.cpp runtime. -const llamaCppTag = 'v0.4.1'; +const llamaCppTag = 'v0.4.1-1'; /// `leehack/litert-lm-native` release tag downloaded for the LiteRT-LM runtime. const liteRtLmReleaseTag = 'v0.17.0-6'; diff --git a/packages/llamadart_llama_cpp_flutter/CHANGELOG.md b/packages/llamadart_llama_cpp_flutter/CHANGELOG.md index 7bbc8f80a..6350b70be 100644 --- a/packages/llamadart_llama_cpp_flutter/CHANGELOG.md +++ b/packages/llamadart_llama_cpp_flutter/CHANGELOG.md @@ -1,3 +1,7 @@ +## Unreleased + +* Updated Apple SwiftPM native pin to `leehack/llamadart-native@v0.4.1-1`. + ## 0.0.19 * Updated Apple SwiftPM native pin to `leehack/llamadart-native@v0.4.1`. diff --git a/packages/llamadart_llama_cpp_flutter/README.md b/packages/llamadart_llama_cpp_flutter/README.md index 13566e3f1..993820443 100644 --- a/packages/llamadart_llama_cpp_flutter/README.md +++ b/packages/llamadart_llama_cpp_flutter/README.md @@ -20,7 +20,7 @@ dependencies: This package has no runtime Dart API of its own. Import `package:llamadart` normally from the core package and use `LlamaBackend()` / `LlamaEngine` there. -The Apple SwiftPM manifest pins `leehack/llamadart-native@v0.4.1`. +The Apple SwiftPM manifest pins `leehack/llamadart-native@v0.4.1-1`. Source for this package lives in `packages/llamadart_llama_cpp_flutter` in the diff --git a/packages/llamadart_llama_cpp_flutter/darwin/llamadart_llama_cpp_flutter/Package.swift b/packages/llamadart_llama_cpp_flutter/darwin/llamadart_llama_cpp_flutter/Package.swift index bd96187d9..8f7d808d4 100644 --- a/packages/llamadart_llama_cpp_flutter/darwin/llamadart_llama_cpp_flutter/Package.swift +++ b/packages/llamadart_llama_cpp_flutter/darwin/llamadart_llama_cpp_flutter/Package.swift @@ -4,7 +4,7 @@ import PackageDescription let packageRoot = URL(fileURLWithPath: #filePath).deletingLastPathComponent() let artifactsRoot = packageRoot.appendingPathComponent("Artifacts") -let llamaCppTag = "v0.4.1" +let llamaCppTag = "v0.4.1-1" func localArtifactPath(_ name: String) -> String? { let path = artifactsRoot.appendingPathComponent(name).path @@ -47,7 +47,7 @@ let package = Package( repository: "leehack/llamadart-native", artifactName: "llamadart-native-apple-xcframework-\(llamaCppTag).zip", tag: llamaCppTag, - checksum: "16175a4084a14af63cc66e78b9b2a8173d2e14db86775af280c859438bd8616a" + checksum: "bab353ce17dfbac9afe65ade73751d67f110b1807a1fb365b4d1df2db25a5418" ), .target( name: "llamadart_llama_cpp_flutter", diff --git a/website/docs/getting-started/installation.md b/website/docs/getting-started/installation.md index 90681fedd..d300edd9a 100644 --- a/website/docs/getting-started/installation.md +++ b/website/docs/getting-started/installation.md @@ -88,7 +88,7 @@ hooks: llamadart: # Optional. Defaults to llamadart's tested native runtime pin. # Use a leehack/llamadart-native release tag when testing another build. - llamadart_native_tag: v0.4.1 + llamadart_native_tag: v0.4.1-1 # Optional. GitHub repository slug or github.com URL. llamadart_native_repository: leehack/llamadart-native @@ -115,7 +115,7 @@ per-target module list. Native source overrides are for compatibility testing. They do not regenerate Dart FFI bindings or symbol lookups, so the selected binary still must be ABI- and symbol-compatible with the default -`leehack/llamadart-native@v0.4.1` runtime. +`leehack/llamadart-native@v0.4.1-1` runtime. Available native tags are published on the [`leehack/llamadart-native` releases page](https://github.com/leehack/llamadart-native/releases). @@ -143,7 +143,7 @@ gh release list --repo leehack/llamadart-native --limit 20 Before overriding, confirm the release includes the asset for your target. The hook downloads files named `llamadart-native--.tar.gz`, for example -`llamadart-native-windows-x64-v0.4.1.tar.gz`. +`llamadart-native-windows-x64-v0.4.1-1.tar.gz`. For local testing, `llamadart_native_path` may point directly at a bundle archive, at an extracted bundle directory, or at a directory containing `//`, `/`, or the expected archive file. diff --git a/website/docs/platforms/support-matrix.md b/website/docs/platforms/support-matrix.md index 204a12c75..48aa4990d 100644 --- a/website/docs/platforms/support-matrix.md +++ b/website/docs/platforms/support-matrix.md @@ -8,7 +8,7 @@ backend-module configuration for `llamadart`. The native-assets hook currently pins `llamadart-native` tag -`v0.4.1` and +`v0.4.1-1` and `litert-lm-native` release `v0.17.0-6` (`lib/src/hook/native_release_pins.dart`). Apps can override the llama.cpp native GitHub source with `hooks.user_defines.llamadart.llamadart_native_tag` and @@ -306,7 +306,7 @@ device/model bundle, use `cpu` or `gpu` for that artifact. a web load failure as a package bug. The [WebGPU Bridge](./webgpu-bridge) page has the browser-console probe and Flutter Web smoke-test path. -## Current llama.cpp module availability by bundle (`v0.4.1`) +## Current llama.cpp module availability by bundle (`v0.4.1-1`) | Bundle key | Available backend modules in bundle | | --- | --- | @@ -354,7 +354,7 @@ hooks: user_defines: llamadart: # Optional. Defaults to llamadart's tested native runtime pin. - llamadart_native_tag: v0.4.1 + llamadart_native_tag: v0.4.1-1 # Optional. GitHub repository slug or github.com URL. llamadart_native_repository: leehack/llamadart-native From 3de7c7e34ee2d43131022a16c4c6013581a9f506 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Wed, 23 Sep 2026 12:17:47 -0400 Subject: [PATCH 2/7] fix: cancel native TTS inside a step through a main-isolate flag For [llamadart-native#86](https://github.com/leehack/llamadart-native/issues/86) and [#325](https://github.com/leehack/llamadart/issues/325). A text-to-speech cancel reached the worker only as a message, so it waited in the worker isolate's queue until the native step in progress returned. On CPU that step can be the whole Qwen3-TTS audio decode. When the runtime exports `llama_dart_tts_eval_callback`, every mtmd context now gets it as `cb_eval`, so a cancelled task stops the audio decode at its next chunk boundary. The backend allocates a one-byte cancel flag per synthesis, raises it on cancel and still sends the cancel message. The worker reads the flag before native task setup and, when the runtime exports `llama_dart_tts_set_cancel_flag`, attaches it after startup so native code reads it during a step. The flag is freed after the worker's terminal response, after a dispose acknowledgement, or when its request was never sent; a dispose frees only the flag of a synthesis claimed before it began. Both exports are optional lookups: v0.4.1 and older runtimes keep today's behaviour, and a runtime without the text-to-speech API gets no callback. Qwen3-TTS 1.7B, tts pack on an M4 Max CPU: in-flight cancels took 17.8-4376.9 ms on main with v0.4.1 and 2.3-118.6 ms with this change on llamadart-native 14d4f0b. --- CHANGELOG.md | 7 + .../backends/llama_cpp/llama_cpp_backend.dart | 50 +++- .../backends/llama_cpp/llama_cpp_service.dart | 78 ++++- lib/src/backends/llama_cpp/worker.dart | 1 + .../backends/llama_cpp/worker_messages.dart | 8 + .../e2e/backends/text_to_speech_e2e_test.dart | 94 ++++++ .../native_symbol_integration_test.dart | 38 +++ .../llama_cpp/llama_cpp_backend_test.dart | 280 +++++++++++++++++- test/unit/backends/llama_cpp/worker_test.dart | 11 +- website/docs/guides/text-to-speech.md | 7 +- 10 files changed, 564 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04fdec71e..5d1c5382a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ the `llamadart_llama_cpp_flutter` Apple SwiftPM checksum and aligned current README/website native override docs. +- Stop a Qwen3-TTS audio decode at its next chunk boundary when native + text-to-speech is cancelled, instead of finishing the native step in + progress first, when the llamadart-native runtime exports + `llama_dart_tts_set_cancel_flag` and `llama_dart_tts_eval_callback`. Older + runtimes, including the pinned v0.4.1, keep the previous behaviour + ([llamadart-native#86](https://github.com/leehack/llamadart-native/issues/86), + [#325](https://github.com/leehack/llamadart/issues/325)). - Extend the GGUF speech-to-text validation pack with four synthetic edge fixtures built in-process, so no extra audio is stored: generated digital silence, plus a truncated RIFF, a stereo 44.1 kHz re-encode and a 33-second diff --git a/lib/src/backends/llama_cpp/llama_cpp_backend.dart b/lib/src/backends/llama_cpp/llama_cpp_backend.dart index 56c9471eb..fe2c68262 100644 --- a/lib/src/backends/llama_cpp/llama_cpp_backend.dart +++ b/lib/src/backends/llama_cpp/llama_cpp_backend.dart @@ -43,23 +43,30 @@ class NativeLlamaBackend int _lifecycleEpoch = 0; final LlamaWorkerEntrypoint _workerEntrypoint; final Duration _workerStartupTimeout; + final Allocator _textToSpeechCancelFlagAllocator; Pointer? _activeCancelToken; void Function()? _activeGenerationCleanup; void Function()? _activeFreeToken; bool _textToSpeechActive = false; bool _textToSpeechCancelRequested = false; bool _textToSpeechRequestSent = false; + _NativeCancelFlag? _textToSpeechCancelFlag; bool _isReady = false; LlamaLogLevel _currentLogLevel = LlamaLogLevel.warn; /// Creates a new [NativeLlamaBackend] and initializes its ports. + /// + /// [textToSpeechCancelFlagAllocator] allocates and frees the one-byte cancel + /// flag of each text-to-speech synthesis. NativeLlamaBackend({ SendPort? initialSendPort, LlamaWorkerEntrypoint workerEntrypoint = llamaWorkerEntry, Duration workerStartupTimeout = const Duration(seconds: 30), + Allocator textToSpeechCancelFlagAllocator = malloc, }) : _workerEntrypoint = workerEntrypoint, - _workerStartupTimeout = workerStartupTimeout { + _workerStartupTimeout = workerStartupTimeout, + _textToSpeechCancelFlagAllocator = textToSpeechCancelFlagAllocator { if (initialSendPort != null) { _sendPort = initialSendPort; _isReady = true; @@ -757,6 +764,9 @@ class NativeLlamaBackend } Future _disposeWorker() async { + // A synthesis claimed after this point sends its flag to the next worker, + // so this dispose must not free it. + final textToSpeechCancelFlag = _textToSpeechCancelFlag; _lifecycleEpoch += 1; _isReady = false; final startup = _isolateStart; @@ -791,6 +801,7 @@ class NativeLlamaBackend _workerLogPort = null; // Worker is gone; free the token if a terminal response did not already. _activeFreeToken?.call(); + textToSpeechCancelFlag?.free(); _activeCancelToken = null; _activeGenerationCleanup = null; _activeFreeToken = null; @@ -885,10 +896,13 @@ class NativeLlamaBackend _textToSpeechActive = true; _textToSpeechCancelRequested = false; _textToSpeechRequestSent = false; + final cancelFlag = _NativeCancelFlag(_textToSpeechCancelFlagAllocator); + _textToSpeechCancelFlag = cancelFlag; try { await _ensureIsolate(); } catch (_) { _textToSpeechActive = false; + cancelFlag.free(); rethrow; } final rp = ReceivePort(); @@ -898,6 +912,7 @@ class NativeLlamaBackend contextHandle, mmContextHandle, request, + cancelFlag.address, rp.sendPort, ), ); @@ -941,6 +956,7 @@ class NativeLlamaBackend return await completer.future; } finally { _textToSpeechActive = false; + cancelFlag.free(); await subscription.cancel(); rp.close(); } @@ -952,6 +968,7 @@ class NativeLlamaBackend return; } _textToSpeechCancelRequested = true; + _textToSpeechCancelFlag?.raise(); if (_textToSpeechRequestSent) { _sendPort?.send(TextToSpeechCancelRequest()); } @@ -1019,3 +1036,34 @@ class NativeLlamaBackend throw Exception("Unknown response during chat template application"); } } + +/// A one-byte cancel flag that the worker isolate reads, including from native +/// code while a text-to-speech step runs. +/// +/// The backend frees it after the worker's terminal response for the request +/// that carries it, after the worker acknowledges a dispose that began after +/// the flag was allocated, or when that request was never sent; never on +/// cancel or on a timer. +final class _NativeCancelFlag { + final Allocator _allocator; + Pointer? _pointer; + + /// Allocates the flag and lowers it. + _NativeCancelFlag(this._allocator) : _pointer = _allocator()..value = 0; + + /// The flag's native address. + int get address => _pointer!.address; + + /// Raises the flag unless it was freed. + void raise() => _pointer?.value = 1; + + /// Frees the flag; later calls do nothing. + void free() { + final pointer = _pointer; + if (pointer == null) { + return; + } + _pointer = null; + _allocator.free(pointer); + } +} diff --git a/lib/src/backends/llama_cpp/llama_cpp_service.dart b/lib/src/backends/llama_cpp/llama_cpp_service.dart index 7b7ac27a1..ee89a2505 100644 --- a/lib/src/backends/llama_cpp/llama_cpp_service.dart +++ b/lib/src/backends/llama_cpp/llama_cpp_service.dart @@ -65,6 +65,10 @@ typedef _TtsStepDart = int Function(Pointer, Pointer); typedef _TtsCancelNative = Void Function(Pointer); typedef _TtsCancelDart = void Function(Pointer); +typedef _TtsSetCancelFlagNative = + Int32 Function(Pointer, Pointer); +typedef _TtsSetCancelFlagDart = + int Function(Pointer, Pointer); typedef _TtsResetNative = Int32 Function(Pointer); typedef _TtsResetDart = int Function(Pointer); typedef _TtsGetOutputInfoNative = @@ -2963,6 +2967,14 @@ class LlamaCppService { throw LlamaUnsupportedException(_ttsUnavailableMessage()); } + ggml_backend_sched_eval_callback _ttsEvalCallback() { + try { + return _resolveTtsApi().evalCallback; + } on LlamaUnsupportedException { + return nullptr; + } + } + String _ttsUnavailableMessage() { return 'Native text-to-speech is unavailable in this runtime bundle ' '(missing llama_dart_tts_* ABI v$LLAMA_DART_TTS_API_VERSION symbols). ' @@ -6860,9 +6872,11 @@ class LlamaCppService { final mmProjPathPtr = mmProjPath.toNativeUtf8(); Pointer mmCtx = nullptr; try { - final ctxParams = _mtmdContextParamsDefault(); - ctxParams.use_gpu = _modelToMtmdUseGpu[modelHandle] ?? true; - mmCtx = _mtmdInitFromFile(mmProjPathPtr.cast(), model.pointer, ctxParams); + mmCtx = _mtmdInitFromFile( + mmProjPathPtr.cast(), + model.pointer, + _mtmdContextParamsFor(modelHandle), + ); } finally { malloc.free(mmProjPathPtr); } @@ -6877,6 +6891,33 @@ class LlamaCppService { return handle; } + mtmd_context_params _mtmdContextParamsFor(int modelHandle) { + final ctxParams = _mtmdContextParamsDefault(); + ctxParams.use_gpu = _modelToMtmdUseGpu[modelHandle] ?? true; + ctxParams.cb_eval = _ttsEvalCallback(); + return ctxParams; + } + + /// The `cb_eval` address [createMultimodalContext] passes to mtmd: that of + /// `llama_dart_tts_eval_callback` in the library providing the text-to- + /// speech API, or 0 when that API is unavailable or its library does not + /// export the callback. + int debugMtmdEvalCallbackAddressForTesting() => + _mtmdContextParamsFor(-1).cb_eval.address; + + /// Whether the service's text-to-speech API lookup finds + /// `llama_dart_tts_eval_callback` and `llama_dart_tts_set_cancel_flag` in + /// [library]; null when [library] lacks the text-to-speech API. + static ({bool evalCallback, bool setCancelFlag})? + debugTtsCancelExportsForTesting(DynamicLibrary library) { + final api = _TtsApi.tryLoad(library); + if (api == null) return null; + return ( + evalCallback: api.evalCallback != nullptr, + setCancelFlag: api.setCancelFlag != null, + ); + } + /// Frees the multimodal context (projector). void freeMultimodalContext(int mmContextHandle) { final mmCtx = _mtmdContexts.remove(mmContextHandle); @@ -7690,12 +7731,21 @@ class LlamaCppService { } /// Synthesizes complete float32 PCM with the stable native TTS wrapper. + /// + /// [cancelFlagAddress] is the address of a one-byte flag that another + /// isolate sets to nonzero to cancel. It is read before the native task is + /// created and, when the runtime exports `llama_dart_tts_set_cancel_flag`, + /// attached to the task after startup so native code reads it while a step + /// runs. It must stay allocated until this future completes; it is last read + /// before then. Future synthesizeTextToSpeech( int contextHandle, int mmContextHandle, - BackendTextToSpeechRequest request, { + BackendTextToSpeechRequest request, + int cancelFlagAddress, { void Function(BackendTextToSpeechProgress progress)? onProgress, }) async { + final cancelFlag = Pointer.fromAddress(cancelFlagAddress); final context = _contexts[contextHandle]; final mtmd = _mtmdContexts[mmContextHandle]; if (context == null || @@ -7768,7 +7818,7 @@ class LlamaCppService { // Yields so a cancel already queued for this synthesis can be processed // before the native task setup, which cannot be interrupted. await Future.delayed(Duration.zero); - if (_ttsCancelPending) { + if (_ttsCancelPending || cancelFlag.value != 0) { _ttsCancelPending = false; throw LlamaTextToSpeechException( 'Text-to-speech synthesis was cancelled.', @@ -7811,6 +7861,7 @@ class LlamaCppService { api: api, task: task, ); + api.setCancelFlag?.call(task, cancelFlag); while (true) { progress.ref.struct_size = sizeOf(); @@ -8036,6 +8087,8 @@ class _TtsApi { final _TtsGetOutputInfoDart getOutputInfo; final _TtsReadPcmDart readPcm; final _TtsLastErrorDart lastError; + final _TtsSetCancelFlagDart? setCancelFlag; + final ggml_backend_sched_eval_callback evalCallback; const _TtsApi({ required this.apiVersion, @@ -8050,10 +8103,14 @@ class _TtsApi { required this.getOutputInfo, required this.readPcm, required this.lastError, + required this.setCancelFlag, + required this.evalCallback, }); static _TtsApi? tryLoad(DynamicLibrary library) { try { + const setCancelFlagSymbol = 'llama_dart_tts_set_cancel_flag'; + const evalCallbackSymbol = 'llama_dart_tts_eval_callback'; return _TtsApi( apiVersion: library .lookupFunction<_TtsApiVersionNative, _TtsApiVersionDart>( @@ -8095,6 +8152,17 @@ class _TtsApi { .lookupFunction<_TtsLastErrorNative, _TtsLastErrorDart>( 'llama_dart_tts_last_error', ), + setCancelFlag: library.providesSymbol(setCancelFlagSymbol) + ? library.lookupFunction< + _TtsSetCancelFlagNative, + _TtsSetCancelFlagDart + >(setCancelFlagSymbol) + : null, + evalCallback: library.providesSymbol(evalCallbackSymbol) + ? library.lookup< + NativeFunction + >(evalCallbackSymbol) + : nullptr, ); } catch (_) { return null; diff --git a/lib/src/backends/llama_cpp/worker.dart b/lib/src/backends/llama_cpp/worker.dart index 9a6be06ad..cf550400e 100644 --- a/lib/src/backends/llama_cpp/worker.dart +++ b/lib/src/backends/llama_cpp/worker.dart @@ -289,6 +289,7 @@ void runLlamaWorkerForTesting( message.contextHandle, message.mmContextHandle, message.request, + message.cancelFlagAddress, onProgress: (progress) { message.sendPort.send( TextToSpeechProgressResponse(progress), diff --git a/lib/src/backends/llama_cpp/worker_messages.dart b/lib/src/backends/llama_cpp/worker_messages.dart index c259634e2..c6afa57d7 100644 --- a/lib/src/backends/llama_cpp/worker_messages.dart +++ b/lib/src/backends/llama_cpp/worker_messages.dart @@ -352,11 +352,19 @@ class TextToSpeechSynthesizeRequest extends WorkerRequest { /// Synthesis inputs and sampling parameters. final BackendTextToSpeechRequest request; + /// Address of the one-byte cancel flag the sender owns. + /// + /// The sender sets the byte to nonzero to cancel and keeps it allocated + /// until this request's terminal response arrives or the worker acknowledges + /// a [DisposeRequest]. + final int cancelFlagAddress; + /// Creates a TTS synthesis request. TextToSpeechSynthesizeRequest( this.contextHandle, this.mmContextHandle, this.request, + this.cancelFlagAddress, super.sendPort, ); } diff --git a/test/e2e/backends/text_to_speech_e2e_test.dart b/test/e2e/backends/text_to_speech_e2e_test.dart index db6ee24b5..951c1728a 100644 --- a/test/e2e/backends/text_to_speech_e2e_test.dart +++ b/test/e2e/backends/text_to_speech_e2e_test.dart @@ -3,14 +3,30 @@ @Timeout(Duration(minutes: 15)) library; +import 'dart:ffi'; import 'dart:io'; +import 'package:ffi/ffi.dart'; import 'package:llamadart/llamadart.dart'; +import 'package:llamadart/src/backends/llama_cpp/llama_cpp_service.dart'; import 'package:test/test.dart'; const _modelPathKey = 'LLAMADART_TTS_MODEL_PATH'; const _mmprojPathKey = 'LLAMADART_TTS_MMPROJ_PATH'; const _outputPathKey = 'LLAMADART_TTS_OUTPUT_PATH'; +const _serviceRequest = BackendTextToSpeechRequest( + text: 'Hello from llamadart.', + language: 'english', + maxFrames: 64, + seed: 1, +); +final _cancelled = throwsA( + isA().having( + (error) => error.message, + 'message', + 'Text-to-speech synthesis was cancelled.', + ), +); void main() { test('synthesizes a playable WAV through the public API', () async { @@ -86,6 +102,84 @@ void main() { await engine.dispose(); } }); + + test('a cancel flag raised before synthesis starts cancels it', () async { + await _withTextToSpeechService((service, context, projector) async { + final flag = calloc()..value = 1; + try { + final progress = []; + await expectLater( + service.synthesizeTextToSpeech( + context, + projector, + _serviceRequest, + flag.address, + onProgress: progress.add, + ), + _cancelled, + ); + expect(progress, isEmpty); + } finally { + calloc.free(flag); + } + }); + }); + + test('a cancel flag raised between steps stops the next step only when the ' + 'runtime can attach it', () async { + await _withTextToSpeechService((service, context, projector) async { + final attachable = service.debugMtmdEvalCallbackAddressForTesting() != 0; + final flag = calloc(); + try { + var steps = 0; + final synthesis = service.synthesizeTextToSpeech( + context, + projector, + _serviceRequest, + flag.address, + onProgress: (_) { + steps += 1; + flag.value = 1; + }, + ); + if (attachable) { + await expectLater(synthesis, _cancelled); + expect(steps, 1); + } else { + expect((await synthesis).samples, isNotEmpty); + expect(steps, greaterThan(1)); + } + } finally { + calloc.free(flag); + } + }); + }); +} + +Future _withTextToSpeechService( + Future Function(LlamaCppService service, int context, int projector) + body, +) async { + final modelPath = _requiredFile(_modelPathKey); + final mmprojPath = _requiredFile(_mmprojPathKey); + if (modelPath == null || mmprojPath == null) { + return; + } + final service = LlamaCppService()..setLogLevel(LlamaLogLevel.none); + try { + service.initializeBackend(); + const modelParams = ModelParams( + contextSize: 4096, + preferredBackend: GpuBackend.cpu, + gpuLayers: 0, + ); + final model = service.loadModel(modelPath, modelParams); + final context = service.createContext(model, modelParams); + final projector = service.createMultimodalContext(model, mmprojPath); + await body(service, context, projector); + } finally { + service.dispose(); + } } String? _requiredFile(String environmentKey) { diff --git a/test/integration/backends/llama_cpp/native_symbol_integration_test.dart b/test/integration/backends/llama_cpp/native_symbol_integration_test.dart index 17a67434f..7346e476a 100644 --- a/test/integration/backends/llama_cpp/native_symbol_integration_test.dart +++ b/test/integration/backends/llama_cpp/native_symbol_integration_test.dart @@ -68,6 +68,11 @@ const _mtmdChunkEvalSymbols = [ 'mtmd_helper_decode_image_chunk', ]; +const _ttsCancelSymbols = [ + 'llama_dart_tts_eval_callback', + 'llama_dart_tts_set_cancel_flag', +]; + const _aloraMetadataSymbols = [ 'llama_adapter_get_alora_n_invocation_tokens', 'llama_adapter_get_alora_invocation_tokens', @@ -978,6 +983,39 @@ void main() { ); }); + test('mtmd contexts get the TTS eval callback only when exported', () { + final wrapper = _llamadartWrapperLibraryFileOrNull(); + expect( + wrapper, + isNotNull, + reason: 'Expected the llama.cpp wrapper library.', + ); + final library = ffi.DynamicLibrary.open(wrapper!.path); + final exports = LlamaCppService.debugTtsCancelExportsForTesting(library); + final address = LlamaCppService() + .debugMtmdEvalCallbackAddressForTesting(); + final exported = _ttsCancelSymbols + .where((symbol) => _fileContainsAscii(wrapper, symbol)) + .toList(growable: false); + + if (!_fileContainsAscii(wrapper, 'llama_dart_tts_step')) { + expect(exports, isNull); + expect(address, 0); + return; + } + if (exported.isEmpty) { + expect(exports, (evalCallback: false, setCancelFlag: false)); + expect(address, 0); + return; + } + expect(exported, _ttsCancelSymbols); + expect(exports, (evalCallback: true, setCancelFlag: true)); + expect( + address, + library.lookup('llama_dart_tts_eval_callback').address, + ); + }); + test('Verify core llama symbols are resolvable', () { expect(() => llama_backend_init(), returnsNormally); expect(llama_version().cast().toDartString(), isNotEmpty); diff --git a/test/unit/backends/llama_cpp/llama_cpp_backend_test.dart b/test/unit/backends/llama_cpp/llama_cpp_backend_test.dart index 8440dc4d3..8333f8865 100644 --- a/test/unit/backends/llama_cpp/llama_cpp_backend_test.dart +++ b/test/unit/backends/llama_cpp/llama_cpp_backend_test.dart @@ -2,9 +2,12 @@ library; import 'dart:async'; +import 'dart:ffi'; +import 'dart:io'; import 'dart:isolate'; import 'dart:typed_data'; +import 'package:ffi/ffi.dart'; import 'package:llamadart/src/backends/backend.dart'; import 'package:llamadart/src/backends/llama_cpp/llama_cpp_backend.dart'; import 'package:llamadart/src/backends/llama_cpp/llama_cpp_service.dart'; @@ -338,6 +341,164 @@ void main() { }, ); + test( + 'a cancel before the request is sent is already in its flag', + () async { + harness.holdTextToSpeech = true; + final pending = backend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Cancel early.'), + ); + backend.cancelTextToSpeech(); + await harness.textToSpeechStarted.future; + await Future.delayed(Duration.zero); + expect(harness.cancelFlagAtRequest, 1); + expect( + harness.received.whereType(), + hasLength(1), + ); + harness.finishHeldTextToSpeech(); + await pending; + }, + ); + + test( + 'frees the cancel flag on the terminal response, not on cancel', + () async { + final allocator = _RecordingAllocator(); + addTearDown(allocator.release); + final flagBackend = NativeLlamaBackend( + initialSendPort: harness.sendPort, + textToSpeechCancelFlagAllocator: allocator, + ); + harness.holdTextToSpeech = true; + final pending = flagBackend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Hold.'), + ); + await harness.textToSpeechStarted.future; + final address = harness.received + .whereType() + .last + .cancelFlagAddress; + expect(allocator.allocated, [address]); + expect(harness.cancelFlagAtRequest, 0); + + flagBackend.cancelTextToSpeech(); + await Future.delayed(Duration.zero); + expect(Pointer.fromAddress(address).value, 1); + expect(allocator.freed, isEmpty); + + harness.finishHeldTextToSpeech(); + await pending; + expect(allocator.freed, [address]); + await flagBackend.dispose(); + expect(allocator.freed, [address]); + }, + ); + + test( + 'dispose frees a running synthesis cancel flag after the worker acks', + () async { + final allocator = _RecordingAllocator(); + addTearDown(allocator.release); + final flagBackend = NativeLlamaBackend( + initialSendPort: harness.sendPort, + textToSpeechCancelFlagAllocator: allocator, + ); + harness.holdTextToSpeech = true; + final pending = flagBackend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Hold.'), + ); + await harness.textToSpeechStarted.future; + final flag = Pointer.fromAddress( + harness.received + .whereType() + .last + .cancelFlagAddress, + ); + + harness.holdNextDispose = true; + final disposed = flagBackend.dispose(); + await harness.disposeReceived.future; + expect(flag.value, 1); + expect(allocator.freed, isEmpty); + + harness.acknowledgeHeldDispose(); + await disposed; + expect(allocator.freed, [flag.address]); + + flag.value = 0; + flagBackend.cancelTextToSpeech(); + expect(flag.value, 0); + + harness.finishHeldTextToSpeech(); + await pending; + expect(allocator.freed, [flag.address]); + }, + ); + + test('a failed worker startup frees the cancel flag', () async { + final allocator = _RecordingAllocator(); + addTearDown(allocator.release); + final flagBackend = NativeLlamaBackend( + workerEntrypoint: _failingInitializationWorkerEntry, + textToSpeechCancelFlagAllocator: allocator, + ); + addTearDown(flagBackend.dispose); + + await expectLater( + flagBackend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Never sent.'), + ), + throwsA(isA()), + ); + expect(allocator.allocated, hasLength(1)); + expect(allocator.freed, allocator.allocated); + }); + + test( + 'dispose keeps the cancel flag of a synthesis claimed during it', + () async { + final allocator = _RecordingAllocator(); + addTearDown(allocator.release); + final flagBackend = NativeLlamaBackend( + workerEntrypoint: _delayedTextToSpeechRejectingWorkerEntry, + textToSpeechCancelFlagAllocator: allocator, + ); + addTearDown(flagBackend.dispose); + final startup = flagBackend.modelLoad( + 'never.gguf', + const ModelParams(), + ); + final disposed = flagBackend.dispose(); + final synthesis = flagBackend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Next worker.'), + ); + await expectLater( + startup, + throwsA(isA()), + ); + await disposed; + expect(allocator.allocated, hasLength(1)); + expect(allocator.freed, isEmpty); + + await expectLater( + synthesis, + throwsA(isA()), + ); + expect(allocator.freed, allocator.allocated); + }, + ); + test('preserves speech error subtypes from the worker', () async { await expectLater( backend.synthesizeTextToSpeech( @@ -469,6 +630,41 @@ void main() { }); }); + test( + 'cancel reaches a worker blocked in a native step through the flag', + () async { + final events = ReceivePort(); + final eventQueue = StreamIterator(events); + final isolate = await Isolate.spawn(_flagPollingWorker, events.sendPort); + addTearDown(() { + isolate.kill(priority: Isolate.immediate); + events.close(); + }); + expect(await eventQueue.moveNext(), isTrue); + final backend = NativeLlamaBackend( + initialSendPort: eventQueue.current as SendPort, + ); + final pending = backend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Blocked.'), + ); + expect(await eventQueue.moveNext(), isTrue); + expect(eventQueue.current, 'in-step'); + backend.cancelTextToSpeech(); + await expectLater( + pending, + throwsA( + isA().having( + (error) => error.message, + 'message', + 'flag observed', + ), + ), + ); + }, + ); + test('modelFree and contextFree are no-op without worker port', () async { final backend = NativeLlamaBackend(); @@ -893,6 +1089,26 @@ void main() { }); } +void _delayedTextToSpeechRejectingWorkerEntry(SendPort initialSendPort) { + final receivePort = ReceivePort(); + initialSendPort.send(receivePort.sendPort); + receivePort.listen((message) async { + switch (message) { + case WorkerHandshake(): + await Future.delayed(const Duration(milliseconds: 50)); + message.sendPort.send(DoneResponse()); + case TextToSpeechSynthesizeRequest(): + message.sendPort.send( + ErrorResponse('rejected', kind: WorkerErrorKind.textToSpeech), + ); + case DisposeRequest(): + message.sendPort.send(null); + receivePort.close(); + Isolate.exit(); + } + }); +} + void _failingInitializationWorkerEntry(SendPort initialSendPort) { runLlamaWorkerForTesting( initialSendPort, @@ -1081,6 +1297,10 @@ class _FakeWorkerHarness { bool holdTextToSpeech = false; Completer textToSpeechStarted = Completer(); TextToSpeechSynthesizeRequest? _heldTextToSpeech; + int? cancelFlagAtRequest; + bool holdNextDispose = false; + final Completer disposeReceived = Completer(); + DisposeRequest? _heldDispose; _FakeWorkerHarness() { _port.listen((message) { @@ -1246,6 +1466,9 @@ class _FakeWorkerHarness { ), ); case TextToSpeechSynthesizeRequest(): + cancelFlagAtRequest = Pointer.fromAddress( + message.cancelFlagAddress, + ).value; if (!textToSpeechStarted.isCompleted) { textToSpeechStarted.complete(); } @@ -1313,7 +1536,13 @@ class _FakeWorkerHarness { message.sendPort.send(DoneResponse()); } case DisposeRequest(): - message.sendPort.send(DoneResponse()); + if (holdNextDispose) { + holdNextDispose = false; + _heldDispose = message; + disposeReceived.complete(); + } else { + message.sendPort.send(DoneResponse()); + } case WorkerHandshake(): // Not expected in these tests. } @@ -1322,6 +1551,11 @@ class _FakeWorkerHarness { SendPort get sendPort => _port.sendPort; + void acknowledgeHeldDispose() { + _heldDispose?.sendPort.send(DoneResponse()); + _heldDispose = null; + } + void finishHeldTextToSpeech() { final request = _heldTextToSpeech; if (request == null) { @@ -1357,3 +1591,47 @@ class _FakeWorkerHarness { _port.close(); } } + +void _flagPollingWorker(SendPort events) { + final requests = ReceivePort(); + events.send(requests.sendPort); + requests.listen((message) { + if (message is! TextToSpeechSynthesizeRequest) { + return; + } + final flag = Pointer.fromAddress(message.cancelFlagAddress); + events.send('in-step'); + final watch = Stopwatch()..start(); + while (flag.value == 0 && watch.elapsed < const Duration(seconds: 10)) { + sleep(const Duration(milliseconds: 1)); + } + message.sendPort.send( + ErrorResponse( + flag.value == 0 ? 'flag never observed' : 'flag observed', + kind: WorkerErrorKind.textToSpeech, + ), + ); + }); +} + +final class _RecordingAllocator implements Allocator { + final List allocated = []; + final List freed = []; + + @override + Pointer allocate(int byteCount, {int? alignment}) { + final pointer = malloc.allocate(byteCount, alignment: alignment); + pointer.asTypedList(byteCount).fillRange(0, byteCount, 0x5a); + allocated.add(pointer.address); + return pointer.cast(); + } + + @override + void free(Pointer pointer) => freed.add(pointer.address); + + void release() { + for (final address in allocated) { + malloc.free(Pointer.fromAddress(address)); + } + } +} diff --git a/test/unit/backends/llama_cpp/worker_test.dart b/test/unit/backends/llama_cpp/worker_test.dart index c617fc969..7e44a6b55 100644 --- a/test/unit/backends/llama_cpp/worker_test.dart +++ b/test/unit/backends/llama_cpp/worker_test.dart @@ -271,10 +271,12 @@ void main() { 2, 3, const BackendTextToSpeechRequest(text: 'Hello.'), + 0x5a5a0, responsePort.sendPort, ), ); await service.synthesisStarted.future; + expect(service.receivedCancelFlagAddress, 0x5a5a0); worker.sendPort.send(TextToSpeechCancelRequest()); await service.cancelObserved.future; service.releaseSynthesis(); @@ -323,6 +325,7 @@ void main() { 2, 3, const BackendTextToSpeechRequest(text: 'Hello.'), + 0x5a5a0, sendPort, ), ); @@ -700,6 +703,7 @@ class _BlockingTextToSpeechService extends LlamaCppService { final Completer cancelObserved = Completer(); final Completer _releaseSynthesis = Completer(); int cancelCalls = 0; + int? receivedCancelFlagAddress; void releaseSynthesis() { if (!_releaseSynthesis.isCompleted) { @@ -717,9 +721,11 @@ class _BlockingTextToSpeechService extends LlamaCppService { Future synthesizeTextToSpeech( int contextHandle, int mmContextHandle, - BackendTextToSpeechRequest request, { + BackendTextToSpeechRequest request, + int cancelFlagAddress, { void Function(BackendTextToSpeechProgress progress)? onProgress, }) async { + receivedCancelFlagAddress = cancelFlagAddress; if (!synthesisStarted.isCompleted) { synthesisStarted.complete(); } @@ -768,7 +774,8 @@ class _ThrowingTextToSpeechService extends LlamaCppService { Future synthesizeTextToSpeech( int contextHandle, int mmContextHandle, - BackendTextToSpeechRequest request, { + BackendTextToSpeechRequest request, + int cancelFlagAddress, { void Function(BackendTextToSpeechProgress progress)? onProgress, }) async { throw exception; diff --git a/website/docs/guides/text-to-speech.md b/website/docs/guides/text-to-speech.md index cec3034ad..07260f43a 100644 --- a/website/docs/guides/text-to-speech.md +++ b/website/docs/guides/text-to-speech.md @@ -129,7 +129,12 @@ responsible for its own files, byte buffers, permissions, and disclosures. ## Cancellation, concurrency, and buffering Call `task.cancel()` to request cooperative cancellation. Cancelling only the -event-stream subscription does not cancel synthesis. +event-stream subscription does not cancel synthesis. On native llama.cpp, when +the runtime exports `llama_dart_tts_set_cancel_flag` and +`llama_dart_tts_eval_callback`, a cancel stops a Qwen3-TTS audio decode in +progress at its next chunk boundary. Older runtimes, including the pinned +v0.4.1, finish the native step in progress first, which can include the whole +audio decode. All typed STT and TTS wrappers over one `LlamaEngine` share a one-task speech lease. Do not run chat generation, transcription, or another synthesis on the From 643f72a6d0366fab6fda392fa62f195ae1ce4c83 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Wed, 23 Sep 2026 13:19:35 -0400 Subject: [PATCH 3/7] test: require both TTS cancel exports from the v0.4.1-1 runtime Every llamadart-native v0.4.1-1 bundle exports `llama_dart_tts_eval_callback` and `llama_dart_tts_set_cancel_flag`. The integration test now requires both whenever the runtime has the text-to-speech API, and still accepts a runtime without that API. `debugTtsCancelExportsForTesting` reports the text-to-speech API the service itself resolves. The e2e between-steps test picks its branch from `llama_dart_tts_set_cancel_flag`, not from the eval callback, so the pinned runtime takes the attach branch. The CHANGELOG and the text-to-speech guide name llamadart-native v0.4.1-1 or later as the floor. --- CHANGELOG.md | 5 ++- .../backends/llama_cpp/llama_cpp_service.dart | 17 +++++----- .../e2e/backends/text_to_speech_e2e_test.dart | 3 +- .../native_symbol_integration_test.dart | 31 +++++++++---------- website/docs/guides/text-to-speech.md | 10 +++--- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d1c5382a..e8d2dbc18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,8 @@ - Stop a Qwen3-TTS audio decode at its next chunk boundary when native text-to-speech is cancelled, instead of finishing the native step in - progress first, when the llamadart-native runtime exports - `llama_dart_tts_set_cancel_flag` and `llama_dart_tts_eval_callback`. Older - runtimes, including the pinned v0.4.1, keep the previous behaviour + progress first. This needs llamadart-native v0.4.1-1 or later; older + runtimes keep the previous behaviour ([llamadart-native#86](https://github.com/leehack/llamadart-native/issues/86), [#325](https://github.com/leehack/llamadart/issues/325)). - Extend the GGUF speech-to-text validation pack with four synthetic edge diff --git a/lib/src/backends/llama_cpp/llama_cpp_service.dart b/lib/src/backends/llama_cpp/llama_cpp_service.dart index ee89a2505..597e13619 100644 --- a/lib/src/backends/llama_cpp/llama_cpp_service.dart +++ b/lib/src/backends/llama_cpp/llama_cpp_service.dart @@ -6905,13 +6905,16 @@ class LlamaCppService { int debugMtmdEvalCallbackAddressForTesting() => _mtmdContextParamsFor(-1).cb_eval.address; - /// Whether the service's text-to-speech API lookup finds - /// `llama_dart_tts_eval_callback` and `llama_dart_tts_set_cancel_flag` in - /// [library]; null when [library] lacks the text-to-speech API. - static ({bool evalCallback, bool setCancelFlag})? - debugTtsCancelExportsForTesting(DynamicLibrary library) { - final api = _TtsApi.tryLoad(library); - if (api == null) return null; + /// Whether the text-to-speech API this service resolves has + /// `llama_dart_tts_eval_callback` and `llama_dart_tts_set_cancel_flag`; + /// null when that API is unavailable. + ({bool evalCallback, bool setCancelFlag})? debugTtsCancelExportsForTesting() { + final _TtsApi api; + try { + api = _resolveTtsApi(); + } on LlamaUnsupportedException { + return null; + } return ( evalCallback: api.evalCallback != nullptr, setCancelFlag: api.setCancelFlag != null, diff --git a/test/e2e/backends/text_to_speech_e2e_test.dart b/test/e2e/backends/text_to_speech_e2e_test.dart index 951c1728a..adfe280ea 100644 --- a/test/e2e/backends/text_to_speech_e2e_test.dart +++ b/test/e2e/backends/text_to_speech_e2e_test.dart @@ -128,7 +128,8 @@ void main() { test('a cancel flag raised between steps stops the next step only when the ' 'runtime can attach it', () async { await _withTextToSpeechService((service, context, projector) async { - final attachable = service.debugMtmdEvalCallbackAddressForTesting() != 0; + final attachable = + service.debugTtsCancelExportsForTesting()?.setCancelFlag ?? false; final flag = calloc(); try { var steps = 0; diff --git a/test/integration/backends/llama_cpp/native_symbol_integration_test.dart b/test/integration/backends/llama_cpp/native_symbol_integration_test.dart index 7346e476a..3df3d6642 100644 --- a/test/integration/backends/llama_cpp/native_symbol_integration_test.dart +++ b/test/integration/backends/llama_cpp/native_symbol_integration_test.dart @@ -983,36 +983,35 @@ void main() { ); }); - test('mtmd contexts get the TTS eval callback only when exported', () { + test('the TTS API has both cancel exports and mtmd contexts get its eval ' + 'callback', () { final wrapper = _llamadartWrapperLibraryFileOrNull(); expect( wrapper, isNotNull, reason: 'Expected the llama.cpp wrapper library.', ); - final library = ffi.DynamicLibrary.open(wrapper!.path); - final exports = LlamaCppService.debugTtsCancelExportsForTesting(library); - final address = LlamaCppService() - .debugMtmdEvalCallbackAddressForTesting(); - final exported = _ttsCancelSymbols - .where((symbol) => _fileContainsAscii(wrapper, symbol)) - .toList(growable: false); + final service = LlamaCppService(); + final exports = service.debugTtsCancelExportsForTesting(); + final address = service.debugMtmdEvalCallbackAddressForTesting(); - if (!_fileContainsAscii(wrapper, 'llama_dart_tts_step')) { + if (!_fileContainsAscii(wrapper!, 'llama_dart_tts_step')) { expect(exports, isNull); expect(address, 0); return; } - if (exported.isEmpty) { - expect(exports, (evalCallback: false, setCancelFlag: false)); - expect(address, 0); - return; - } - expect(exported, _ttsCancelSymbols); + expect( + _ttsCancelSymbols + .where((symbol) => _fileContainsAscii(wrapper, symbol)) + .toList(), + _ttsCancelSymbols, + ); expect(exports, (evalCallback: true, setCancelFlag: true)); expect( address, - library.lookup('llama_dart_tts_eval_callback').address, + ffi.DynamicLibrary.open( + wrapper.path, + ).lookup('llama_dart_tts_eval_callback').address, ); }); diff --git a/website/docs/guides/text-to-speech.md b/website/docs/guides/text-to-speech.md index 07260f43a..a4c2c883f 100644 --- a/website/docs/guides/text-to-speech.md +++ b/website/docs/guides/text-to-speech.md @@ -129,12 +129,10 @@ responsible for its own files, byte buffers, permissions, and disclosures. ## Cancellation, concurrency, and buffering Call `task.cancel()` to request cooperative cancellation. Cancelling only the -event-stream subscription does not cancel synthesis. On native llama.cpp, when -the runtime exports `llama_dart_tts_set_cancel_flag` and -`llama_dart_tts_eval_callback`, a cancel stops a Qwen3-TTS audio decode in -progress at its next chunk boundary. Older runtimes, including the pinned -v0.4.1, finish the native step in progress first, which can include the whole -audio decode. +event-stream subscription does not cancel synthesis. On native llama.cpp with +llamadart-native v0.4.1-1 or later, a cancel stops a Qwen3-TTS audio decode in +progress at its next chunk boundary. Older runtimes finish the native step in +progress first, which can include the whole audio decode. All typed STT and TTS wrappers over one `LlamaEngine` share a one-task speech lease. Do not run chat generation, transcription, or another synthesis on the From b61136de3924b10c92774a24482979ccd39ae23a Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Wed, 23 Sep 2026 13:35:17 -0400 Subject: [PATCH 4/7] chore: approve the v0.4.1-1 native pin beside the v0.1.44 Web bridge The v0.4.1-1 sync left two gates red. performance-tuning.md still called v0.4.1 the package-pinned runtime, and the Web/native gate required the native pin to equal the native release the v0.1.44 bridge assets were qualified against (v0.4.1). v0.4.1-1 rebuilds v0.4.1 from the same upstream llama.cpp commit; both release manifests record b29c606e28. The gate now records v0.4.1-1 as the approved native pin, separate from the qualified anchor that `--verify-manifest` still checks against the published bridge manifest, and the two bridge docs name both. Any other native pin still fails the gate. --- doc/webgpu_bridge.md | 13 ++++++----- .../tooling/check_webgpu_bridge_tag_test.dart | 23 +++++++++++++++---- tool/testing/check_webgpu_bridge_tag.dart | 21 ++++++++++------- website/docs/guides/performance-tuning.md | 2 +- website/docs/platforms/webgpu-bridge.md | 12 ++++++---- 5 files changed, 47 insertions(+), 24 deletions(-) diff --git a/doc/webgpu_bridge.md b/doc/webgpu_bridge.md index c60365631..bf89a372a 100644 --- a/doc/webgpu_bridge.md +++ b/doc/webgpu_bridge.md @@ -22,15 +22,16 @@ pipelines. Default pinned tag in the example is `v0.1.44`. That release embeds llama.cpp `v0.4.1`, matching the `hook/build.dart` native pin -(`v0.4.1`, both built from upstream llama.cpp `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`) +(`v0.4.1-1`, both built from upstream llama.cpp `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`) even though the bridge asset tag `v0.1.44` differs from the native runtime tag -`v0.4.1`. Provenance for this immutable consumer artifact: release `389783936`, +`v0.4.1-1`. Provenance for this immutable consumer artifact: release `389783936`, tag commit `fdafd9f8cbdb9bf99c359536595eff9a23095379`, bridge source `89178be67c3c84300bc1b129182bd5bc5a8e21fc`, and manifest SHA-256 -`8d61f453753ac7a7d839ac12318b70986a814748d86029993118c19454293aa9`. It retains -the Qwen3-ASR typed speech-to-text contract introduced in `v0.1.30` and -provisions the explicit 1 MiB Wasm stack needed for memory64 context -construction in direct and worker modes. The chat bootstrap opts +`8d61f453753ac7a7d839ac12318b70986a814748d86029993118c19454293aa9`. The bridge +assets were qualified against native `v0.4.1`; `v0.4.1-1` rebuilds it from the +same upstream commit. It retains the Qwen3-ASR typed speech-to-text contract +introduced in `v0.1.30` and provisions the explicit 1 MiB Wasm stack needed for +memory64 context construction in direct and worker modes. The chat bootstrap opts `SpeechToTextEngine` into that contract from the immutable tag; older or custom assets remain disabled unless the host explicitly sets `window.__llamadartBridgeSpeechToTextSupported = true` after equivalent diff --git a/test/unit/tooling/check_webgpu_bridge_tag_test.dart b/test/unit/tooling/check_webgpu_bridge_tag_test.dart index 0ed779fda..751ac5568 100644 --- a/test/unit/tooling/check_webgpu_bridge_tag_test.dart +++ b/test/unit/tooling/check_webgpu_bridge_tag_test.dart @@ -696,18 +696,33 @@ void main() { ); }); - test('a different wrapper in the same family is not the approved anchor', () { + test('a different wrapper in the same family is not the approved pin', () { final root = _fakeRuntimeRepo(bridgeTag: 'v0.2.0', nativeTag: 'v0.2.0-2'); expect( findBridgeRuntimeDrift(root, 'v0.2.0'), contains( - '$nativeLlamaCppTagPath pins v0.2.0-2, expected the bridge-qualified ' - 'native anchor $bridgeNativeReleaseTag', + '$nativeLlamaCppTagPath pins v0.2.0-2, expected the bridge-approved ' + 'native pin $bridgeApprovedNativePin', ), ); }); + test( + 'the qualified anchor is not accepted in place of the approved pin', + () { + final root = _fakeRuntimeRepo( + bridgeTag: bridgeLlamaCppTag, + nativeTag: bridgeNativeReleaseTag, + ); + + expect(findBridgeRuntimeDrift(root, bridgeLlamaCppTag), [ + '$nativeLlamaCppTagPath pins $bridgeNativeReleaseTag, expected the ' + 'bridge-approved native pin $bridgeApprovedNativePin', + ]); + }, + ); + test('stale divergence prose is reported', () { final root = _fakeRuntimeRepo( bridgeTag: 'v0.2.0', @@ -724,7 +739,7 @@ void main() { test('parity prose is accepted for matching upstream tags', () { final root = _fakeRuntimeRepo( bridgeTag: bridgeLlamaCppTag, - nativeTag: bridgeNativeReleaseTag, + nativeTag: bridgeApprovedNativePin, parityWording: true, ); diff --git a/tool/testing/check_webgpu_bridge_tag.dart b/tool/testing/check_webgpu_bridge_tag.dart index dbf127a71..c06108ac7 100644 --- a/tool/testing/check_webgpu_bridge_tag.dart +++ b/tool/testing/check_webgpu_bridge_tag.dart @@ -383,6 +383,11 @@ const String bridgeNativeRepository = 'leehack/llamadart-native'; /// The native release tag the pinned bridge assets were qualified against. const String bridgeNativeReleaseTag = 'v0.4.1'; +/// The native pin approved beside the pinned bridge assets: either +/// [bridgeNativeReleaseTag] or a native wrapper rebuild of it from the same +/// upstream llama.cpp commit. +const String bridgeApprovedNativePin = 'v0.4.1-1'; + /// The asset repository release that published the pinned bridge assets. const String bridgeAssetsReleaseId = '389783936'; @@ -439,11 +444,11 @@ final List bridgeProvenancePins = [ BridgeTagPin( 'doc/webgpu_bridge.md', RegExp( - r'^\(`(?[^`]+)`, both built from upstream llama\.cpp ' + r'^\(`(?[^`]+)`, both built from upstream llama\.cpp ' r'`(?[^`@]+)@(?[0-9a-f]{40})`\)\r?\n' r'even though the bridge asset tag `v\d+\.\d+\.\d+` differs from the ' r'native runtime tag\r?\n' - r'`v\d+\.\d+\.\d+`\. Provenance for this immutable consumer artifact: ' + r'`v\d+\.\d+\.\d+(?:-\d+)?`\. Provenance for this immutable consumer artifact: ' r'release `(?\d+)`,\r?\n' r'tag commit `(?[0-9a-f]{40})`, bridge source\r?\n' r'`(?[0-9a-f]{40})`, and manifest SHA-256\r?\n' @@ -454,11 +459,11 @@ final List bridgeProvenancePins = [ BridgeTagPin( 'website/docs/platforms/webgpu-bridge.md', RegExp( - r'^ \(`(?[^`]+)`, both built from upstream ' + r'^ \(`(?[^`]+)`, both built from upstream ' r'`(?[^`@]+)@(?[0-9a-f]{40})`\)\r?\n' r' even though the bridge asset tag `v\d+\.\d+\.\d+` differs from the ' r'native runtime tag\r?\n' - r' `v\d+\.\d+\.\d+`\. Pinned artifact provenance: release ' + r' `v\d+\.\d+\.\d+(?:-\d+)?`\. Pinned artifact provenance: release ' r'`(?\d+)`, tag commit\r?\n' r' `(?[0-9a-f]{40})`, bridge source\r?\n' r' `(?[0-9a-f]{40})`, manifest SHA-256\r?\n' @@ -470,7 +475,7 @@ final List bridgeProvenancePins = [ /// The expected value behind each [bridgeProvenancePins] group name. Map get bridgeProvenanceValues => { - 'nativeReleaseTag': bridgeNativeReleaseTag, + 'nativePin': bridgeApprovedNativePin, 'upstreamTag': bridgeLlamaCppTag, 'upstreamCommit': bridgeLlamaCppCommit, 'releaseId': bridgeAssetsReleaseId, @@ -532,10 +537,10 @@ List findBridgeRuntimeDrift(Directory repoRoot, String bridgeTag) { 'expected stable vMAJOR.MINOR.PATCH(-N) or nightly bNNNN(-N|-llamadart.N)', ); } else { - if (nativeTag != bridgeNativeReleaseTag) { + if (nativeTag != bridgeApprovedNativePin) { problems.add( - '$nativeLlamaCppTagPath pins $nativeTag, expected the bridge-qualified ' - 'native anchor $bridgeNativeReleaseTag', + '$nativeLlamaCppTagPath pins $nativeTag, expected the bridge-approved ' + 'native pin $bridgeApprovedNativePin', ); } if (normalizedNativeFamily != bridgeTag) { diff --git a/website/docs/guides/performance-tuning.md b/website/docs/guides/performance-tuning.md index aa592c672..4cdc66f52 100644 --- a/website/docs/guides/performance-tuning.md +++ b/website/docs/guides/performance-tuning.md @@ -148,7 +148,7 @@ Guidelines: - DSpark is an experimental, opt-in native llama.cpp external draft-model strategy. Use `SpeculativeDecodingConfig.draftDspark(draftModelPath: ...)`; it is never selected automatically, maps to upstream `draft-dspark`, requires - at least the `b10356-llamadart.1` wrapper fix; the package-pinned `v0.4.1` + at least the `b10356-llamadart.1` wrapper fix; the package-pinned `v0.4.1-1` runtime satisfies that ABI, supports speculators-format checkpoints, and adds LFM2 target/draft support. DSpark remains subject to target/draft/backend compatibility. diff --git a/website/docs/platforms/webgpu-bridge.md b/website/docs/platforms/webgpu-bridge.md index 4b478d7a6..b526144a1 100644 --- a/website/docs/platforms/webgpu-bridge.md +++ b/website/docs/platforms/webgpu-bridge.md @@ -202,15 +202,17 @@ cannot report success before the bridge exposes `prefetchModelToCache(...)`. remains unsupported; use memory64. - `v0.1.39+` remains the compatibility floor for bridge asset capabilities. - The pinned `v0.1.44` bridge assets embed llama.cpp `v0.4.1`, matching the native runtime - (`v0.4.1`, both built from upstream `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`) + (`v0.4.1-1`, both built from upstream `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`) even though the bridge asset tag `v0.1.44` differs from the native runtime tag - `v0.4.1`. Pinned artifact provenance: release `389783936`, tag commit + `v0.4.1-1`. Pinned artifact provenance: release `389783936`, tag commit `fdafd9f8cbdb9bf99c359536595eff9a23095379`, bridge source `89178be67c3c84300bc1b129182bd5bc5a8e21fc`, manifest SHA-256 `8d61f453753ac7a7d839ac12318b70986a814748d86029993118c19454293aa9`. The - bridge assets provision an explicit 1 MiB stack for both wasm32 and memory64, - preventing graph-parameter growth from overflowing Emscripten's 64 KiB - default during memory64 Qwen3-ASR context construction. + bridge assets were qualified against native `v0.4.1`; `v0.4.1-1` rebuilds it + from the same upstream commit. The bridge assets provision an explicit 1 MiB + stack for both wasm32 and memory64, preventing graph-parameter growth from + overflowing Emscripten's 64 KiB default during memory64 Qwen3-ASR context + construction. - `v0.1.12+` bridge assets forward native-compatible `ModelParams` load tuning fields, including multi-sequence slots, KV cache type, flash attention, RoPE overrides, split mode, and main GPU. From dc5e953857ae59b06c19f615b9c9b521ae2151a4 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Wed, 23 Sep 2026 13:36:17 -0400 Subject: [PATCH 5/7] test: require no startup diagnostics from the TTS API lookup Every mtmd context creation now resolves the text-to-speech API, including for vision and speech-to-text. On Windows that lookup preloads each existing absolute wrapper candidate it tries, and a failed preload records a startup diagnostic. The integration test now requires the lookup to record none on the runtime it runs against. --- .../backends/llama_cpp/native_symbol_integration_test.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/backends/llama_cpp/native_symbol_integration_test.dart b/test/integration/backends/llama_cpp/native_symbol_integration_test.dart index 3df3d6642..162ab1c29 100644 --- a/test/integration/backends/llama_cpp/native_symbol_integration_test.dart +++ b/test/integration/backends/llama_cpp/native_symbol_integration_test.dart @@ -994,6 +994,7 @@ void main() { final service = LlamaCppService(); final exports = service.debugTtsCancelExportsForTesting(); final address = service.debugMtmdEvalCallbackAddressForTesting(); + expect(service.getStartupDiagnostics(), isEmpty); if (!_fileContainsAscii(wrapper!, 'llama_dart_tts_step')) { expect(exports, isNull); From e2e9fe957b8b8d7c7595a46dee1843ff1d6cbbe3 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Wed, 23 Sep 2026 15:12:20 -0400 Subject: [PATCH 6/7] docs: link the TTS tracking issue from the TTS cancel changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d62475ea..a5ec06e53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ progress first. This needs llamadart-native v0.4.1-1 or later; older runtimes keep the previous behaviour ([llamadart-native#86](https://github.com/leehack/llamadart-native/issues/86), - [#325](https://github.com/leehack/llamadart/issues/325)). + [#322](https://github.com/leehack/llamadart/issues/322)). - Add an experimental `DecisionEngine` for Laya-style decision models (a ModernBERT encoder GGUF plus a safetensors head) on native llama.cpp, with typed `ChoiceKey`, `ScoreKey` and `NoulKey` questions From 4c14e3c38891e415cbb316c4b6babd2444b7e363 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Wed, 23 Sep 2026 16:14:35 -0400 Subject: [PATCH 7/7] docs: mirror the v0.4.1-1 pin and TTS cancel entries in recent releases --- website/docs/changelog/recent-releases.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/website/docs/changelog/recent-releases.md b/website/docs/changelog/recent-releases.md index 73e986915..127352928 100644 --- a/website/docs/changelog/recent-releases.md +++ b/website/docs/changelog/recent-releases.md @@ -9,6 +9,17 @@ For canonical full release notes, use: ## Unreleased +- Updated the default llama.cpp native runtime pin to + `leehack/llamadart-native@v0.4.1-1`, keeping the `v0.4.1` llama.cpp + ABI/bindings while picking up wrapper-only native fixes. Refreshed the + `llamadart_llama_cpp_flutter` Apple SwiftPM checksum and aligned current + README/website native override docs. +- Stop a Qwen3-TTS audio decode at its next chunk boundary when native + text-to-speech is cancelled, instead of finishing the native step in + progress first. This needs llamadart-native v0.4.1-1 or later; older + runtimes keep the previous behaviour + ([llamadart-native#86](https://github.com/leehack/llamadart-native/issues/86), + [#322](https://github.com/leehack/llamadart/issues/322)). - Add an experimental `DecisionEngine` for Laya-style decision models (a ModernBERT encoder GGUF plus a safetensors head) on native llama.cpp, with typed `ChoiceKey`, `ScoreKey` and `NoulKey` questions