fix: stop a cancelled Qwen3-TTS audio decode at its next chunk on native v0.4.1-1

CHANGELOG.md

@@ -1,5 +1,17 @@
 ## Unreleased
 
+* Updated the default llama.cpp native runtime pin to
+  `leehack/llamadart-native@v0.4.1-1`, keeping the `v0.4.1` llama.cpp
+  ABI/bindings while picking up wrapper-only native fixes. Refreshed
+  the `llamadart_llama_cpp_flutter` Apple SwiftPM checksum and
+  aligned current README/website native override docs.
+
+- Stop a Qwen3-TTS audio decode at its next chunk boundary when native
+  text-to-speech is cancelled, instead of finishing the native step in
+  progress first. This needs llamadart-native v0.4.1-1 or later; older
+  runtimes keep the previous behaviour
+  ([llamadart-native#86](https://github.com/leehack/llamadart-native/issues/86),
+  [#322](https://github.com/leehack/llamadart/issues/322)).
 - Add an experimental `DecisionEngine` for Laya-style decision models (a
   ModernBERT encoder GGUF plus a safetensors head) on native llama.cpp, with
   typed `ChoiceKey`, `ScoreKey` and `NoulKey` questions

README.md

@@ -156,7 +156,7 @@ Current default runtime pins:
 
 | Runtime | Pin |
 | --- | --- |
-| Native llama.cpp / GGUF | `leehack/llamadart-native@v0.4.1` |
+| Native llama.cpp / GGUF | `leehack/llamadart-native@v0.4.1-1` |
 | Native LiteRT-LM / `.litertlm` | `leehack/litert-lm-native@v0.17.0-6` |
 | Web llama.cpp / GGUF | `leehack/llama-web-bridge-assets@v0.1.44` |
 | Web LiteRT-LM / `.litertlm` | `@litert-lm/core@0.15.0` |

doc/webgpu_bridge.md

@@ -22,15 +22,16 @@ pipelines.
 Default pinned tag in the example is `v0.1.44`.
 
 That release embeds llama.cpp `v0.4.1`, matching the `hook/build.dart` native pin
-(`v0.4.1`, both built from upstream llama.cpp `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`)
+(`v0.4.1-1`, both built from upstream llama.cpp `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`)
 even though the bridge asset tag `v0.1.44` differs from the native runtime tag
-`v0.4.1`. Provenance for this immutable consumer artifact: release `389783936`,
+`v0.4.1-1`. Provenance for this immutable consumer artifact: release `389783936`,
 tag commit `fdafd9f8cbdb9bf99c359536595eff9a23095379`, bridge source
 `89178be67c3c84300bc1b129182bd5bc5a8e21fc`, and manifest SHA-256
-`8d61f453753ac7a7d839ac12318b70986a814748d86029993118c19454293aa9`. It retains
-the Qwen3-ASR typed speech-to-text contract introduced in `v0.1.30` and
-provisions the explicit 1 MiB Wasm stack needed for memory64 context
-construction in direct and worker modes. The chat bootstrap opts
+`8d61f453753ac7a7d839ac12318b70986a814748d86029993118c19454293aa9`. The bridge
+assets were qualified against native `v0.4.1`; `v0.4.1-1` rebuilds it from the
+same upstream commit. It retains the Qwen3-ASR typed speech-to-text contract
+introduced in `v0.1.30` and provisions the explicit 1 MiB Wasm stack needed for
+memory64 context construction in direct and worker modes. The chat bootstrap opts
 `SpeechToTextEngine` into that contract from the immutable tag; older or custom
 assets remain disabled unless the host explicitly sets
 `window.__llamadartBridgeSpeechToTextSupported = true` after equivalent

lib/src/backends/llama_cpp/bindings.dart

@@ -6298,6 +6298,15 @@ external int llama_dart_tts_api_version();
 @ffi.Native<ffi.Void Function(ffi.Pointer<llama_dart_tts>)>()
 external void llama_dart_tts_cancel(ffi.Pointer<llama_dart_tts> tts);
 
+@ffi.Native<
+  ffi.Bool Function(ffi.Pointer<ggml_tensor>, ffi.Bool, ffi.Pointer<ffi.Void>)
+>()
+external bool llama_dart_tts_eval_callback(
+  ffi.Pointer<ggml_tensor> tensor,
+  bool ask,
+  ffi.Pointer<ffi.Void> user_data,
+);
+
 @ffi.Native<ffi.Void Function(ffi.Pointer<llama_dart_tts>)>()
 external void llama_dart_tts_free(ffi.Pointer<llama_dart_tts> tts);
 
@@ -6403,6 +6412,23 @@ llama_dart_tts_status llama_dart_tts_reset(ffi.Pointer<llama_dart_tts> tts) {
   return llama_dart_tts_status.fromValue(_llama_dart_tts_reset(tts));
 }
 
+@ffi.Native<
+  ffi.Int Function(ffi.Pointer<llama_dart_tts>, ffi.Pointer<ffi.Int8>)
+>(symbol: 'llama_dart_tts_set_cancel_flag')
+external int _llama_dart_tts_set_cancel_flag(
+  ffi.Pointer<llama_dart_tts> tts,
+  ffi.Pointer<ffi.Int8> flag,
+);
+
+llama_dart_tts_status llama_dart_tts_set_cancel_flag(
+  ffi.Pointer<llama_dart_tts> tts,
+  ffi.Pointer<ffi.Int8> flag,
+) {
+  return llama_dart_tts_status.fromValue(
+    _llama_dart_tts_set_cancel_flag(tts, flag),
+  );
+}
+
 @ffi.Native<
   ffi.Int Function(
     ffi.Pointer<llama_dart_tts>,
@@ -8719,7 +8745,7 @@ external int mtmd_tokenize_from_parts(
   bool add_special,
 );
 
-typedef FILE = __sFILE;
+typedef FILE = _IO_FILE;
 
 const int GGML_BACKEND_META_MAX_DEVICES = 16;
 
@@ -8858,88 +8884,90 @@ final class UnnamedUnion extends ffi.Union {
   external ffi.Array<ffi.Char> val_str;
 }
 
-typedef __darwin_off_t = __int64_t;
-typedef __int64_t = ffi.LongLong;
-typedef Dart__int64_t = int;
+final class _IO_FILE extends ffi.Struct {
+  @ffi.Int()
+  external int _flags;
 
-final class __sFILE extends ffi.Struct {
-  external ffi.Pointer<ffi.UnsignedChar> _p;
+  external ffi.Pointer<ffi.Char> _IO_read_ptr;
 
-  @ffi.Int()
-  external int _r;
+  external ffi.Pointer<ffi.Char> _IO_read_end;
 
-  @ffi.Int()
-  external int _w;
+  external ffi.Pointer<ffi.Char> _IO_read_base;
 
-  @ffi.Short()
-  external int _flags;
+  external ffi.Pointer<ffi.Char> _IO_write_base;
 
-  @ffi.Short()
-  external int _file;
+  external ffi.Pointer<ffi.Char> _IO_write_ptr;
 
-  external __sbuf _bf;
+  external ffi.Pointer<ffi.Char> _IO_write_end;
 
-  @ffi.Int()
-  external int _lbfsize;
+  external ffi.Pointer<ffi.Char> _IO_buf_base;
 
-  external ffi.Pointer<ffi.Void> _cookie;
+  external ffi.Pointer<ffi.Char> _IO_buf_end;
 
-  external ffi.Pointer<
-    ffi.NativeFunction<ffi.Int Function(ffi.Pointer<ffi.Void>)>
-  >
-  _close;
+  external ffi.Pointer<ffi.Char> _IO_save_base;
 
-  external ffi.Pointer<
-    ffi.NativeFunction<
-      ffi.Int Function(ffi.Pointer<ffi.Void>, ffi.Pointer<ffi.Char>, ffi.Int)
-    >
-  >
-  _read;
+  external ffi.Pointer<ffi.Char> _IO_backup_base;
 
-  external ffi.Pointer<
-    ffi.NativeFunction<fpos_t Function(ffi.Pointer<ffi.Void>, fpos_t, ffi.Int)>
-  >
-  _seek;
+  external ffi.Pointer<ffi.Char> _IO_save_end;
 
-  external ffi.Pointer<
-    ffi.NativeFunction<
-      ffi.Int Function(ffi.Pointer<ffi.Void>, ffi.Pointer<ffi.Char>, ffi.Int)
-    >
-  >
-  _write;
+  external ffi.Pointer<_IO_marker> _markers;
 
-  external __sbuf _ub;
+  external ffi.Pointer<_IO_FILE> _chain;
 
-  external ffi.Pointer<__sFILEX> _extra;
+  @ffi.Int()
+  external int _fileno;
 
   @ffi.Int()
-  external int _ur;
+  external int _flags2;
 
-  @ffi.Array.multi([3])
-  external ffi.Array<ffi.UnsignedChar> _ubuf;
+  @__off_t()
+  external int _old_offset;
 
-  @ffi.Array.multi([1])
-  external ffi.Array<ffi.UnsignedChar> _nbuf;
+  @ffi.UnsignedShort()
+  external int _cur_column;
 
-  external __sbuf _lb;
+  @ffi.SignedChar()
+  external int _vtable_offset;
 
-  @ffi.Int()
-  external int _blksize;
+  @ffi.Array.multi([1])
+  external ffi.Array<ffi.Char> _shortbuf;
+
+  external ffi.Pointer<_IO_lock_t> _lock;
 
-  @fpos_t()
+  @__off64_t()
   external int _offset;
-}
 
-final class __sFILEX extends ffi.Opaque {}
+  external ffi.Pointer<_IO_codecvt> _codecvt;
+
+  external ffi.Pointer<_IO_wide_data> _wide_data;
+
+  external ffi.Pointer<_IO_FILE> _freeres_list;
+
+  external ffi.Pointer<ffi.Void> _freeres_buf;
 
-final class __sbuf extends ffi.Struct {
-  external ffi.Pointer<ffi.UnsignedChar> _base;
+  @ffi.Size()
+  external int __pad5;
 
   @ffi.Int()
-  external int _size;
+  external int _mode;
+
+  @ffi.Array.multi([20])
+  external ffi.Array<ffi.Char> _unused2;
 }
 
-typedef fpos_t = __darwin_off_t;
+final class _IO_codecvt extends ffi.Opaque {}
+
+typedef _IO_lock_t = ffi.Void;
+typedef Dart_IO_lock_t = void;
+
+final class _IO_marker extends ffi.Opaque {}
+
+final class _IO_wide_data extends ffi.Opaque {}
+
+typedef __off64_t = ffi.Long;
+typedef Dart__off64_t = int;
+typedef __off_t = ffi.Long;
+typedef Dart__off_t = int;
 typedef ggml_abort_callback =
     ffi.Pointer<ffi.NativeFunction<ggml_abort_callbackFunction>>;
 typedef ggml_abort_callbackFunction =

lib/src/backends/llama_cpp/llama_cpp_backend.dart

@@ -44,23 +44,30 @@ class NativeLlamaBackend
   int _lifecycleEpoch = 0;
   final LlamaWorkerEntrypoint _workerEntrypoint;
   final Duration _workerStartupTimeout;
+  final Allocator _textToSpeechCancelFlagAllocator;
   Pointer<Int8>? _activeCancelToken;
   void Function()? _activeGenerationCleanup;
   void Function()? _activeFreeToken;
   bool _textToSpeechActive = false;
   bool _textToSpeechCancelRequested = false;
   bool _textToSpeechRequestSent = false;
+  _NativeCancelFlag? _textToSpeechCancelFlag;
 
   bool _isReady = false;
   LlamaLogLevel _currentLogLevel = LlamaLogLevel.warn;
 
   /// Creates a new [NativeLlamaBackend] and initializes its ports.
+  ///
+  /// [textToSpeechCancelFlagAllocator] allocates and frees the one-byte cancel
+  /// flag of each text-to-speech synthesis.
   NativeLlamaBackend({
     SendPort? initialSendPort,
     LlamaWorkerEntrypoint workerEntrypoint = llamaWorkerEntry,
     Duration workerStartupTimeout = const Duration(seconds: 30),
+    Allocator textToSpeechCancelFlagAllocator = malloc,
   }) : _workerEntrypoint = workerEntrypoint,
-       _workerStartupTimeout = workerStartupTimeout {
+       _workerStartupTimeout = workerStartupTimeout,
+       _textToSpeechCancelFlagAllocator = textToSpeechCancelFlagAllocator {
     if (initialSendPort != null) {
       _sendPort = initialSendPort;
       _isReady = true;
@@ -758,6 +765,9 @@ class NativeLlamaBackend
   }
 
   Future<void> _disposeWorker() async {
+    // A synthesis claimed after this point sends its flag to the next worker,
+    // so this dispose must not free it.
+    final textToSpeechCancelFlag = _textToSpeechCancelFlag;
     _lifecycleEpoch += 1;
     _isReady = false;
     final startup = _isolateStart;
@@ -792,6 +802,7 @@ class NativeLlamaBackend
     _workerLogPort = null;
     // Worker is gone; free the token if a terminal response did not already.
     _activeFreeToken?.call();
+    textToSpeechCancelFlag?.free();
     _activeCancelToken = null;
     _activeGenerationCleanup = null;
     _activeFreeToken = null;
@@ -886,10 +897,13 @@ class NativeLlamaBackend
     _textToSpeechActive = true;
     _textToSpeechCancelRequested = false;
     _textToSpeechRequestSent = false;
+    final cancelFlag = _NativeCancelFlag(_textToSpeechCancelFlagAllocator);
+    _textToSpeechCancelFlag = cancelFlag;
     try {
       await _ensureIsolate();
     } catch (_) {
       _textToSpeechActive = false;
+      cancelFlag.free();
       rethrow;
     }
     final rp = ReceivePort();
@@ -899,6 +913,7 @@ class NativeLlamaBackend
         contextHandle,
         mmContextHandle,
         request,
+        cancelFlag.address,
         rp.sendPort,
       ),
     );
@@ -942,6 +957,7 @@ class NativeLlamaBackend
       return await completer.future;
     } finally {
       _textToSpeechActive = false;
+      cancelFlag.free();
       await subscription.cancel();
       rp.close();
     }
@@ -953,6 +969,7 @@ class NativeLlamaBackend
       return;
     }
     _textToSpeechCancelRequested = true;
+    _textToSpeechCancelFlag?.raise();
     if (_textToSpeechRequestSent) {
       _sendPort?.send(TextToSpeechCancelRequest());
     }
@@ -1096,3 +1113,34 @@ class NativeLlamaBackend
     throw Exception("Unknown response during chat template application");
   }
 }
+
+/// A one-byte cancel flag that the worker isolate reads, including from native
+/// code while a text-to-speech step runs.
+///
+/// The backend frees it after the worker's terminal response for the request
+/// that carries it, after the worker acknowledges a dispose that began after
+/// the flag was allocated, or when that request was never sent; never on
+/// cancel or on a timer.
+final class _NativeCancelFlag {
+  final Allocator _allocator;
+  Pointer<Int8>? _pointer;
+
+  /// Allocates the flag and lowers it.
+  _NativeCancelFlag(this._allocator) : _pointer = _allocator<Int8>()..value = 0;
+
+  /// The flag's native address.
+  int get address => _pointer!.address;
+
+  /// Raises the flag unless it was freed.
+  void raise() => _pointer?.value = 1;
+
+  /// Frees the flag; later calls do nothing.
+  void free() {
+    final pointer = _pointer;
+    if (pointer == null) {
+      return;
+    }
+    _pointer = null;
+    _allocator.free(pointer);
+  }
+}