diff --git a/CHANGELOG.md b/CHANGELOG.md index 597f315de..a5ec06e53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ ## Unreleased +* Updated the default llama.cpp native runtime pin to + `leehack/llamadart-native@v0.4.1-1`, keeping the `v0.4.1` llama.cpp + ABI/bindings while picking up wrapper-only native fixes. Refreshed + the `llamadart_llama_cpp_flutter` Apple SwiftPM checksum and + aligned current README/website native override docs. + +- Stop a Qwen3-TTS audio decode at its next chunk boundary when native + text-to-speech is cancelled, instead of finishing the native step in + progress first. This needs llamadart-native v0.4.1-1 or later; older + runtimes keep the previous behaviour + ([llamadart-native#86](https://github.com/leehack/llamadart-native/issues/86), + [#322](https://github.com/leehack/llamadart/issues/322)). - Add an experimental `DecisionEngine` for Laya-style decision models (a ModernBERT encoder GGUF plus a safetensors head) on native llama.cpp, with typed `ChoiceKey`, `ScoreKey` and `NoulKey` questions diff --git a/README.md b/README.md index 65bca1f27..a81ee8a7f 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ Current default runtime pins: | Runtime | Pin | | --- | --- | -| Native llama.cpp / GGUF | `leehack/llamadart-native@v0.4.1` | +| Native llama.cpp / GGUF | `leehack/llamadart-native@v0.4.1-1` | | Native LiteRT-LM / `.litertlm` | `leehack/litert-lm-native@v0.17.0-6` | | Web llama.cpp / GGUF | `leehack/llama-web-bridge-assets@v0.1.44` | | Web LiteRT-LM / `.litertlm` | `@litert-lm/core@0.15.0` | diff --git a/doc/webgpu_bridge.md b/doc/webgpu_bridge.md index c60365631..bf89a372a 100644 --- a/doc/webgpu_bridge.md +++ b/doc/webgpu_bridge.md @@ -22,15 +22,16 @@ pipelines. Default pinned tag in the example is `v0.1.44`. That release embeds llama.cpp `v0.4.1`, matching the `hook/build.dart` native pin -(`v0.4.1`, both built from upstream llama.cpp `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`) +(`v0.4.1-1`, both built from upstream llama.cpp `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`) even though the bridge asset tag `v0.1.44` differs from the native runtime tag -`v0.4.1`. Provenance for this immutable consumer artifact: release `389783936`, +`v0.4.1-1`. Provenance for this immutable consumer artifact: release `389783936`, tag commit `fdafd9f8cbdb9bf99c359536595eff9a23095379`, bridge source `89178be67c3c84300bc1b129182bd5bc5a8e21fc`, and manifest SHA-256 -`8d61f453753ac7a7d839ac12318b70986a814748d86029993118c19454293aa9`. It retains -the Qwen3-ASR typed speech-to-text contract introduced in `v0.1.30` and -provisions the explicit 1 MiB Wasm stack needed for memory64 context -construction in direct and worker modes. The chat bootstrap opts +`8d61f453753ac7a7d839ac12318b70986a814748d86029993118c19454293aa9`. The bridge +assets were qualified against native `v0.4.1`; `v0.4.1-1` rebuilds it from the +same upstream commit. It retains the Qwen3-ASR typed speech-to-text contract +introduced in `v0.1.30` and provisions the explicit 1 MiB Wasm stack needed for +memory64 context construction in direct and worker modes. The chat bootstrap opts `SpeechToTextEngine` into that contract from the immutable tag; older or custom assets remain disabled unless the host explicitly sets `window.__llamadartBridgeSpeechToTextSupported = true` after equivalent diff --git a/lib/src/backends/llama_cpp/bindings.dart b/lib/src/backends/llama_cpp/bindings.dart index 7635b0b03..31d10f32b 100644 --- a/lib/src/backends/llama_cpp/bindings.dart +++ b/lib/src/backends/llama_cpp/bindings.dart @@ -6298,6 +6298,15 @@ external int llama_dart_tts_api_version(); @ffi.Native)>() external void llama_dart_tts_cancel(ffi.Pointer tts); +@ffi.Native< + ffi.Bool Function(ffi.Pointer, ffi.Bool, ffi.Pointer) +>() +external bool llama_dart_tts_eval_callback( + ffi.Pointer tensor, + bool ask, + ffi.Pointer user_data, +); + @ffi.Native)>() external void llama_dart_tts_free(ffi.Pointer tts); @@ -6403,6 +6412,23 @@ llama_dart_tts_status llama_dart_tts_reset(ffi.Pointer tts) { return llama_dart_tts_status.fromValue(_llama_dart_tts_reset(tts)); } +@ffi.Native< + ffi.Int Function(ffi.Pointer, ffi.Pointer) +>(symbol: 'llama_dart_tts_set_cancel_flag') +external int _llama_dart_tts_set_cancel_flag( + ffi.Pointer tts, + ffi.Pointer flag, +); + +llama_dart_tts_status llama_dart_tts_set_cancel_flag( + ffi.Pointer tts, + ffi.Pointer flag, +) { + return llama_dart_tts_status.fromValue( + _llama_dart_tts_set_cancel_flag(tts, flag), + ); +} + @ffi.Native< ffi.Int Function( ffi.Pointer, @@ -8719,7 +8745,7 @@ external int mtmd_tokenize_from_parts( bool add_special, ); -typedef FILE = __sFILE; +typedef FILE = _IO_FILE; const int GGML_BACKEND_META_MAX_DEVICES = 16; @@ -8858,88 +8884,90 @@ final class UnnamedUnion extends ffi.Union { external ffi.Array val_str; } -typedef __darwin_off_t = __int64_t; -typedef __int64_t = ffi.LongLong; -typedef Dart__int64_t = int; +final class _IO_FILE extends ffi.Struct { + @ffi.Int() + external int _flags; -final class __sFILE extends ffi.Struct { - external ffi.Pointer _p; + external ffi.Pointer _IO_read_ptr; - @ffi.Int() - external int _r; + external ffi.Pointer _IO_read_end; - @ffi.Int() - external int _w; + external ffi.Pointer _IO_read_base; - @ffi.Short() - external int _flags; + external ffi.Pointer _IO_write_base; - @ffi.Short() - external int _file; + external ffi.Pointer _IO_write_ptr; - external __sbuf _bf; + external ffi.Pointer _IO_write_end; - @ffi.Int() - external int _lbfsize; + external ffi.Pointer _IO_buf_base; - external ffi.Pointer _cookie; + external ffi.Pointer _IO_buf_end; - external ffi.Pointer< - ffi.NativeFunction)> - > - _close; + external ffi.Pointer _IO_save_base; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, ffi.Int) - > - > - _read; + external ffi.Pointer _IO_backup_base; - external ffi.Pointer< - ffi.NativeFunction, fpos_t, ffi.Int)> - > - _seek; + external ffi.Pointer _IO_save_end; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, ffi.Int) - > - > - _write; + external ffi.Pointer<_IO_marker> _markers; - external __sbuf _ub; + external ffi.Pointer<_IO_FILE> _chain; - external ffi.Pointer<__sFILEX> _extra; + @ffi.Int() + external int _fileno; @ffi.Int() - external int _ur; + external int _flags2; - @ffi.Array.multi([3]) - external ffi.Array _ubuf; + @__off_t() + external int _old_offset; - @ffi.Array.multi([1]) - external ffi.Array _nbuf; + @ffi.UnsignedShort() + external int _cur_column; - external __sbuf _lb; + @ffi.SignedChar() + external int _vtable_offset; - @ffi.Int() - external int _blksize; + @ffi.Array.multi([1]) + external ffi.Array _shortbuf; + + external ffi.Pointer<_IO_lock_t> _lock; - @fpos_t() + @__off64_t() external int _offset; -} -final class __sFILEX extends ffi.Opaque {} + external ffi.Pointer<_IO_codecvt> _codecvt; + + external ffi.Pointer<_IO_wide_data> _wide_data; + + external ffi.Pointer<_IO_FILE> _freeres_list; + + external ffi.Pointer _freeres_buf; -final class __sbuf extends ffi.Struct { - external ffi.Pointer _base; + @ffi.Size() + external int __pad5; @ffi.Int() - external int _size; + external int _mode; + + @ffi.Array.multi([20]) + external ffi.Array _unused2; } -typedef fpos_t = __darwin_off_t; +final class _IO_codecvt extends ffi.Opaque {} + +typedef _IO_lock_t = ffi.Void; +typedef Dart_IO_lock_t = void; + +final class _IO_marker extends ffi.Opaque {} + +final class _IO_wide_data extends ffi.Opaque {} + +typedef __off64_t = ffi.Long; +typedef Dart__off64_t = int; +typedef __off_t = ffi.Long; +typedef Dart__off_t = int; typedef ggml_abort_callback = ffi.Pointer>; typedef ggml_abort_callbackFunction = diff --git a/lib/src/backends/llama_cpp/llama_cpp_backend.dart b/lib/src/backends/llama_cpp/llama_cpp_backend.dart index 7757e5229..3e259e247 100644 --- a/lib/src/backends/llama_cpp/llama_cpp_backend.dart +++ b/lib/src/backends/llama_cpp/llama_cpp_backend.dart @@ -44,23 +44,30 @@ class NativeLlamaBackend int _lifecycleEpoch = 0; final LlamaWorkerEntrypoint _workerEntrypoint; final Duration _workerStartupTimeout; + final Allocator _textToSpeechCancelFlagAllocator; Pointer? _activeCancelToken; void Function()? _activeGenerationCleanup; void Function()? _activeFreeToken; bool _textToSpeechActive = false; bool _textToSpeechCancelRequested = false; bool _textToSpeechRequestSent = false; + _NativeCancelFlag? _textToSpeechCancelFlag; bool _isReady = false; LlamaLogLevel _currentLogLevel = LlamaLogLevel.warn; /// Creates a new [NativeLlamaBackend] and initializes its ports. + /// + /// [textToSpeechCancelFlagAllocator] allocates and frees the one-byte cancel + /// flag of each text-to-speech synthesis. NativeLlamaBackend({ SendPort? initialSendPort, LlamaWorkerEntrypoint workerEntrypoint = llamaWorkerEntry, Duration workerStartupTimeout = const Duration(seconds: 30), + Allocator textToSpeechCancelFlagAllocator = malloc, }) : _workerEntrypoint = workerEntrypoint, - _workerStartupTimeout = workerStartupTimeout { + _workerStartupTimeout = workerStartupTimeout, + _textToSpeechCancelFlagAllocator = textToSpeechCancelFlagAllocator { if (initialSendPort != null) { _sendPort = initialSendPort; _isReady = true; @@ -758,6 +765,9 @@ class NativeLlamaBackend } Future _disposeWorker() async { + // A synthesis claimed after this point sends its flag to the next worker, + // so this dispose must not free it. + final textToSpeechCancelFlag = _textToSpeechCancelFlag; _lifecycleEpoch += 1; _isReady = false; final startup = _isolateStart; @@ -792,6 +802,7 @@ class NativeLlamaBackend _workerLogPort = null; // Worker is gone; free the token if a terminal response did not already. _activeFreeToken?.call(); + textToSpeechCancelFlag?.free(); _activeCancelToken = null; _activeGenerationCleanup = null; _activeFreeToken = null; @@ -886,10 +897,13 @@ class NativeLlamaBackend _textToSpeechActive = true; _textToSpeechCancelRequested = false; _textToSpeechRequestSent = false; + final cancelFlag = _NativeCancelFlag(_textToSpeechCancelFlagAllocator); + _textToSpeechCancelFlag = cancelFlag; try { await _ensureIsolate(); } catch (_) { _textToSpeechActive = false; + cancelFlag.free(); rethrow; } final rp = ReceivePort(); @@ -899,6 +913,7 @@ class NativeLlamaBackend contextHandle, mmContextHandle, request, + cancelFlag.address, rp.sendPort, ), ); @@ -942,6 +957,7 @@ class NativeLlamaBackend return await completer.future; } finally { _textToSpeechActive = false; + cancelFlag.free(); await subscription.cancel(); rp.close(); } @@ -953,6 +969,7 @@ class NativeLlamaBackend return; } _textToSpeechCancelRequested = true; + _textToSpeechCancelFlag?.raise(); if (_textToSpeechRequestSent) { _sendPort?.send(TextToSpeechCancelRequest()); } @@ -1096,3 +1113,34 @@ class NativeLlamaBackend throw Exception("Unknown response during chat template application"); } } + +/// A one-byte cancel flag that the worker isolate reads, including from native +/// code while a text-to-speech step runs. +/// +/// The backend frees it after the worker's terminal response for the request +/// that carries it, after the worker acknowledges a dispose that began after +/// the flag was allocated, or when that request was never sent; never on +/// cancel or on a timer. +final class _NativeCancelFlag { + final Allocator _allocator; + Pointer? _pointer; + + /// Allocates the flag and lowers it. + _NativeCancelFlag(this._allocator) : _pointer = _allocator()..value = 0; + + /// The flag's native address. + int get address => _pointer!.address; + + /// Raises the flag unless it was freed. + void raise() => _pointer?.value = 1; + + /// Frees the flag; later calls do nothing. + void free() { + final pointer = _pointer; + if (pointer == null) { + return; + } + _pointer = null; + _allocator.free(pointer); + } +} diff --git a/lib/src/backends/llama_cpp/llama_cpp_service.dart b/lib/src/backends/llama_cpp/llama_cpp_service.dart index b9f09d4d5..0bb36e476 100644 --- a/lib/src/backends/llama_cpp/llama_cpp_service.dart +++ b/lib/src/backends/llama_cpp/llama_cpp_service.dart @@ -68,6 +68,10 @@ typedef _TtsStepDart = int Function(Pointer, Pointer); typedef _TtsCancelNative = Void Function(Pointer); typedef _TtsCancelDart = void Function(Pointer); +typedef _TtsSetCancelFlagNative = + Int32 Function(Pointer, Pointer); +typedef _TtsSetCancelFlagDart = + int Function(Pointer, Pointer); typedef _TtsResetNative = Int32 Function(Pointer); typedef _TtsResetDart = int Function(Pointer); typedef _TtsGetOutputInfoNative = @@ -2968,6 +2972,14 @@ class LlamaCppService { throw LlamaUnsupportedException(_ttsUnavailableMessage()); } + ggml_backend_sched_eval_callback _ttsEvalCallback() { + try { + return _resolveTtsApi().evalCallback; + } on LlamaUnsupportedException { + return nullptr; + } + } + String _ttsUnavailableMessage() { return 'Native text-to-speech is unavailable in this runtime bundle ' '(missing llama_dart_tts_* ABI v$LLAMA_DART_TTS_API_VERSION symbols). ' @@ -6874,9 +6886,11 @@ class LlamaCppService { final mmProjPathPtr = mmProjPath.toNativeUtf8(); Pointer mmCtx = nullptr; try { - final ctxParams = _mtmdContextParamsDefault(); - ctxParams.use_gpu = _modelToMtmdUseGpu[modelHandle] ?? true; - mmCtx = _mtmdInitFromFile(mmProjPathPtr.cast(), model.pointer, ctxParams); + mmCtx = _mtmdInitFromFile( + mmProjPathPtr.cast(), + model.pointer, + _mtmdContextParamsFor(modelHandle), + ); } finally { malloc.free(mmProjPathPtr); } @@ -6891,6 +6905,36 @@ class LlamaCppService { return handle; } + mtmd_context_params _mtmdContextParamsFor(int modelHandle) { + final ctxParams = _mtmdContextParamsDefault(); + ctxParams.use_gpu = _modelToMtmdUseGpu[modelHandle] ?? true; + ctxParams.cb_eval = _ttsEvalCallback(); + return ctxParams; + } + + /// The `cb_eval` address [createMultimodalContext] passes to mtmd: that of + /// `llama_dart_tts_eval_callback` in the library providing the text-to- + /// speech API, or 0 when that API is unavailable or its library does not + /// export the callback. + int debugMtmdEvalCallbackAddressForTesting() => + _mtmdContextParamsFor(-1).cb_eval.address; + + /// Whether the text-to-speech API this service resolves has + /// `llama_dart_tts_eval_callback` and `llama_dart_tts_set_cancel_flag`; + /// null when that API is unavailable. + ({bool evalCallback, bool setCancelFlag})? debugTtsCancelExportsForTesting() { + final _TtsApi api; + try { + api = _resolveTtsApi(); + } on LlamaUnsupportedException { + return null; + } + return ( + evalCallback: api.evalCallback != nullptr, + setCancelFlag: api.setCancelFlag != null, + ); + } + /// Frees the multimodal context (projector). void freeMultimodalContext(int mmContextHandle) { final mmCtx = _mtmdContexts.remove(mmContextHandle); @@ -8198,12 +8242,21 @@ class LlamaCppService { } /// Synthesizes complete float32 PCM with the stable native TTS wrapper. + /// + /// [cancelFlagAddress] is the address of a one-byte flag that another + /// isolate sets to nonzero to cancel. It is read before the native task is + /// created and, when the runtime exports `llama_dart_tts_set_cancel_flag`, + /// attached to the task after startup so native code reads it while a step + /// runs. It must stay allocated until this future completes; it is last read + /// before then. Future synthesizeTextToSpeech( int contextHandle, int mmContextHandle, - BackendTextToSpeechRequest request, { + BackendTextToSpeechRequest request, + int cancelFlagAddress, { void Function(BackendTextToSpeechProgress progress)? onProgress, }) async { + final cancelFlag = Pointer.fromAddress(cancelFlagAddress); final context = _contexts[contextHandle]; final mtmd = _mtmdContexts[mmContextHandle]; if (context == null || @@ -8276,7 +8329,7 @@ class LlamaCppService { // Yields so a cancel already queued for this synthesis can be processed // before the native task setup, which cannot be interrupted. await Future.delayed(Duration.zero); - if (_ttsCancelPending) { + if (_ttsCancelPending || cancelFlag.value != 0) { _ttsCancelPending = false; throw LlamaTextToSpeechException( 'Text-to-speech synthesis was cancelled.', @@ -8319,6 +8372,7 @@ class LlamaCppService { api: api, task: task, ); + api.setCancelFlag?.call(task, cancelFlag); while (true) { progress.ref.struct_size = sizeOf(); @@ -8544,6 +8598,8 @@ class _TtsApi { final _TtsGetOutputInfoDart getOutputInfo; final _TtsReadPcmDart readPcm; final _TtsLastErrorDart lastError; + final _TtsSetCancelFlagDart? setCancelFlag; + final ggml_backend_sched_eval_callback evalCallback; const _TtsApi({ required this.apiVersion, @@ -8558,10 +8614,14 @@ class _TtsApi { required this.getOutputInfo, required this.readPcm, required this.lastError, + required this.setCancelFlag, + required this.evalCallback, }); static _TtsApi? tryLoad(DynamicLibrary library) { try { + const setCancelFlagSymbol = 'llama_dart_tts_set_cancel_flag'; + const evalCallbackSymbol = 'llama_dart_tts_eval_callback'; return _TtsApi( apiVersion: library .lookupFunction<_TtsApiVersionNative, _TtsApiVersionDart>( @@ -8603,6 +8663,17 @@ class _TtsApi { .lookupFunction<_TtsLastErrorNative, _TtsLastErrorDart>( 'llama_dart_tts_last_error', ), + setCancelFlag: library.providesSymbol(setCancelFlagSymbol) + ? library.lookupFunction< + _TtsSetCancelFlagNative, + _TtsSetCancelFlagDart + >(setCancelFlagSymbol) + : null, + evalCallback: library.providesSymbol(evalCallbackSymbol) + ? library.lookup< + NativeFunction + >(evalCallbackSymbol) + : nullptr, ); } catch (_) { return null; diff --git a/lib/src/backends/llama_cpp/worker.dart b/lib/src/backends/llama_cpp/worker.dart index b2dd81d4a..2b3904a7c 100644 --- a/lib/src/backends/llama_cpp/worker.dart +++ b/lib/src/backends/llama_cpp/worker.dart @@ -289,6 +289,7 @@ void runLlamaWorkerForTesting( message.contextHandle, message.mmContextHandle, message.request, + message.cancelFlagAddress, onProgress: (progress) { message.sendPort.send( TextToSpeechProgressResponse(progress), diff --git a/lib/src/backends/llama_cpp/worker_messages.dart b/lib/src/backends/llama_cpp/worker_messages.dart index d3a68360c..c6fe50ff6 100644 --- a/lib/src/backends/llama_cpp/worker_messages.dart +++ b/lib/src/backends/llama_cpp/worker_messages.dart @@ -352,11 +352,19 @@ class TextToSpeechSynthesizeRequest extends WorkerRequest { /// Synthesis inputs and sampling parameters. final BackendTextToSpeechRequest request; + /// Address of the one-byte cancel flag the sender owns. + /// + /// The sender sets the byte to nonzero to cancel and keeps it allocated + /// until this request's terminal response arrives or the worker acknowledges + /// a [DisposeRequest]. + final int cancelFlagAddress; + /// Creates a TTS synthesis request. TextToSpeechSynthesizeRequest( this.contextHandle, this.mmContextHandle, this.request, + this.cancelFlagAddress, super.sendPort, ); } diff --git a/lib/src/hook/native_release_pins.dart b/lib/src/hook/native_release_pins.dart index 92ef44d98..38ee2afcf 100644 --- a/lib/src/hook/native_release_pins.dart +++ b/lib/src/hook/native_release_pins.dart @@ -5,7 +5,7 @@ library; /// `leehack/llamadart-native` release tag downloaded for the llama.cpp runtime. -const llamaCppTag = 'v0.4.1'; +const llamaCppTag = 'v0.4.1-1'; /// `leehack/litert-lm-native` release tag downloaded for the LiteRT-LM runtime. const liteRtLmReleaseTag = 'v0.17.0-6'; diff --git a/packages/llamadart_llama_cpp_flutter/CHANGELOG.md b/packages/llamadart_llama_cpp_flutter/CHANGELOG.md index 7bbc8f80a..6350b70be 100644 --- a/packages/llamadart_llama_cpp_flutter/CHANGELOG.md +++ b/packages/llamadart_llama_cpp_flutter/CHANGELOG.md @@ -1,3 +1,7 @@ +## Unreleased + +* Updated Apple SwiftPM native pin to `leehack/llamadart-native@v0.4.1-1`. + ## 0.0.19 * Updated Apple SwiftPM native pin to `leehack/llamadart-native@v0.4.1`. diff --git a/packages/llamadart_llama_cpp_flutter/README.md b/packages/llamadart_llama_cpp_flutter/README.md index 13566e3f1..993820443 100644 --- a/packages/llamadart_llama_cpp_flutter/README.md +++ b/packages/llamadart_llama_cpp_flutter/README.md @@ -20,7 +20,7 @@ dependencies: This package has no runtime Dart API of its own. Import `package:llamadart` normally from the core package and use `LlamaBackend()` / `LlamaEngine` there. -The Apple SwiftPM manifest pins `leehack/llamadart-native@v0.4.1`. +The Apple SwiftPM manifest pins `leehack/llamadart-native@v0.4.1-1`. Source for this package lives in `packages/llamadart_llama_cpp_flutter` in the diff --git a/packages/llamadart_llama_cpp_flutter/darwin/llamadart_llama_cpp_flutter/Package.swift b/packages/llamadart_llama_cpp_flutter/darwin/llamadart_llama_cpp_flutter/Package.swift index bd96187d9..8f7d808d4 100644 --- a/packages/llamadart_llama_cpp_flutter/darwin/llamadart_llama_cpp_flutter/Package.swift +++ b/packages/llamadart_llama_cpp_flutter/darwin/llamadart_llama_cpp_flutter/Package.swift @@ -4,7 +4,7 @@ import PackageDescription let packageRoot = URL(fileURLWithPath: #filePath).deletingLastPathComponent() let artifactsRoot = packageRoot.appendingPathComponent("Artifacts") -let llamaCppTag = "v0.4.1" +let llamaCppTag = "v0.4.1-1" func localArtifactPath(_ name: String) -> String? { let path = artifactsRoot.appendingPathComponent(name).path @@ -47,7 +47,7 @@ let package = Package( repository: "leehack/llamadart-native", artifactName: "llamadart-native-apple-xcframework-\(llamaCppTag).zip", tag: llamaCppTag, - checksum: "16175a4084a14af63cc66e78b9b2a8173d2e14db86775af280c859438bd8616a" + checksum: "bab353ce17dfbac9afe65ade73751d67f110b1807a1fb365b4d1df2db25a5418" ), .target( name: "llamadart_llama_cpp_flutter", diff --git a/test/e2e/backends/text_to_speech_e2e_test.dart b/test/e2e/backends/text_to_speech_e2e_test.dart index db6ee24b5..adfe280ea 100644 --- a/test/e2e/backends/text_to_speech_e2e_test.dart +++ b/test/e2e/backends/text_to_speech_e2e_test.dart @@ -3,14 +3,30 @@ @Timeout(Duration(minutes: 15)) library; +import 'dart:ffi'; import 'dart:io'; +import 'package:ffi/ffi.dart'; import 'package:llamadart/llamadart.dart'; +import 'package:llamadart/src/backends/llama_cpp/llama_cpp_service.dart'; import 'package:test/test.dart'; const _modelPathKey = 'LLAMADART_TTS_MODEL_PATH'; const _mmprojPathKey = 'LLAMADART_TTS_MMPROJ_PATH'; const _outputPathKey = 'LLAMADART_TTS_OUTPUT_PATH'; +const _serviceRequest = BackendTextToSpeechRequest( + text: 'Hello from llamadart.', + language: 'english', + maxFrames: 64, + seed: 1, +); +final _cancelled = throwsA( + isA().having( + (error) => error.message, + 'message', + 'Text-to-speech synthesis was cancelled.', + ), +); void main() { test('synthesizes a playable WAV through the public API', () async { @@ -86,6 +102,85 @@ void main() { await engine.dispose(); } }); + + test('a cancel flag raised before synthesis starts cancels it', () async { + await _withTextToSpeechService((service, context, projector) async { + final flag = calloc()..value = 1; + try { + final progress = []; + await expectLater( + service.synthesizeTextToSpeech( + context, + projector, + _serviceRequest, + flag.address, + onProgress: progress.add, + ), + _cancelled, + ); + expect(progress, isEmpty); + } finally { + calloc.free(flag); + } + }); + }); + + test('a cancel flag raised between steps stops the next step only when the ' + 'runtime can attach it', () async { + await _withTextToSpeechService((service, context, projector) async { + final attachable = + service.debugTtsCancelExportsForTesting()?.setCancelFlag ?? false; + final flag = calloc(); + try { + var steps = 0; + final synthesis = service.synthesizeTextToSpeech( + context, + projector, + _serviceRequest, + flag.address, + onProgress: (_) { + steps += 1; + flag.value = 1; + }, + ); + if (attachable) { + await expectLater(synthesis, _cancelled); + expect(steps, 1); + } else { + expect((await synthesis).samples, isNotEmpty); + expect(steps, greaterThan(1)); + } + } finally { + calloc.free(flag); + } + }); + }); +} + +Future _withTextToSpeechService( + Future Function(LlamaCppService service, int context, int projector) + body, +) async { + final modelPath = _requiredFile(_modelPathKey); + final mmprojPath = _requiredFile(_mmprojPathKey); + if (modelPath == null || mmprojPath == null) { + return; + } + final service = LlamaCppService()..setLogLevel(LlamaLogLevel.none); + try { + service.initializeBackend(); + const modelParams = ModelParams( + contextSize: 4096, + preferredBackend: GpuBackend.cpu, + gpuLayers: 0, + ); + final model = service.loadModel(modelPath, modelParams); + final context = service.createContext(model, modelParams); + final projector = service.createMultimodalContext(model, mmprojPath); + await body(service, context, projector); + } finally { + service.dispose(); + } } String? _requiredFile(String environmentKey) { diff --git a/test/integration/backends/llama_cpp/native_symbol_integration_test.dart b/test/integration/backends/llama_cpp/native_symbol_integration_test.dart index 17a67434f..162ab1c29 100644 --- a/test/integration/backends/llama_cpp/native_symbol_integration_test.dart +++ b/test/integration/backends/llama_cpp/native_symbol_integration_test.dart @@ -68,6 +68,11 @@ const _mtmdChunkEvalSymbols = [ 'mtmd_helper_decode_image_chunk', ]; +const _ttsCancelSymbols = [ + 'llama_dart_tts_eval_callback', + 'llama_dart_tts_set_cancel_flag', +]; + const _aloraMetadataSymbols = [ 'llama_adapter_get_alora_n_invocation_tokens', 'llama_adapter_get_alora_invocation_tokens', @@ -978,6 +983,39 @@ void main() { ); }); + test('the TTS API has both cancel exports and mtmd contexts get its eval ' + 'callback', () { + final wrapper = _llamadartWrapperLibraryFileOrNull(); + expect( + wrapper, + isNotNull, + reason: 'Expected the llama.cpp wrapper library.', + ); + final service = LlamaCppService(); + final exports = service.debugTtsCancelExportsForTesting(); + final address = service.debugMtmdEvalCallbackAddressForTesting(); + expect(service.getStartupDiagnostics(), isEmpty); + + if (!_fileContainsAscii(wrapper!, 'llama_dart_tts_step')) { + expect(exports, isNull); + expect(address, 0); + return; + } + expect( + _ttsCancelSymbols + .where((symbol) => _fileContainsAscii(wrapper, symbol)) + .toList(), + _ttsCancelSymbols, + ); + expect(exports, (evalCallback: true, setCancelFlag: true)); + expect( + address, + ffi.DynamicLibrary.open( + wrapper.path, + ).lookup('llama_dart_tts_eval_callback').address, + ); + }); + test('Verify core llama symbols are resolvable', () { expect(() => llama_backend_init(), returnsNormally); expect(llama_version().cast().toDartString(), isNotEmpty); diff --git a/test/unit/backends/llama_cpp/llama_cpp_backend_test.dart b/test/unit/backends/llama_cpp/llama_cpp_backend_test.dart index d5ae89f39..c3c0fd0b4 100644 --- a/test/unit/backends/llama_cpp/llama_cpp_backend_test.dart +++ b/test/unit/backends/llama_cpp/llama_cpp_backend_test.dart @@ -2,9 +2,12 @@ library; import 'dart:async'; +import 'dart:ffi'; +import 'dart:io'; import 'dart:isolate'; import 'dart:typed_data'; +import 'package:ffi/ffi.dart'; import 'package:llamadart/src/backends/backend.dart'; import 'package:llamadart/src/backends/llama_cpp/llama_cpp_backend.dart'; import 'package:llamadart/src/backends/llama_cpp/llama_cpp_service.dart'; @@ -339,6 +342,164 @@ void main() { }, ); + test( + 'a cancel before the request is sent is already in its flag', + () async { + harness.holdTextToSpeech = true; + final pending = backend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Cancel early.'), + ); + backend.cancelTextToSpeech(); + await harness.textToSpeechStarted.future; + await Future.delayed(Duration.zero); + expect(harness.cancelFlagAtRequest, 1); + expect( + harness.received.whereType(), + hasLength(1), + ); + harness.finishHeldTextToSpeech(); + await pending; + }, + ); + + test( + 'frees the cancel flag on the terminal response, not on cancel', + () async { + final allocator = _RecordingAllocator(); + addTearDown(allocator.release); + final flagBackend = NativeLlamaBackend( + initialSendPort: harness.sendPort, + textToSpeechCancelFlagAllocator: allocator, + ); + harness.holdTextToSpeech = true; + final pending = flagBackend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Hold.'), + ); + await harness.textToSpeechStarted.future; + final address = harness.received + .whereType() + .last + .cancelFlagAddress; + expect(allocator.allocated, [address]); + expect(harness.cancelFlagAtRequest, 0); + + flagBackend.cancelTextToSpeech(); + await Future.delayed(Duration.zero); + expect(Pointer.fromAddress(address).value, 1); + expect(allocator.freed, isEmpty); + + harness.finishHeldTextToSpeech(); + await pending; + expect(allocator.freed, [address]); + await flagBackend.dispose(); + expect(allocator.freed, [address]); + }, + ); + + test( + 'dispose frees a running synthesis cancel flag after the worker acks', + () async { + final allocator = _RecordingAllocator(); + addTearDown(allocator.release); + final flagBackend = NativeLlamaBackend( + initialSendPort: harness.sendPort, + textToSpeechCancelFlagAllocator: allocator, + ); + harness.holdTextToSpeech = true; + final pending = flagBackend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Hold.'), + ); + await harness.textToSpeechStarted.future; + final flag = Pointer.fromAddress( + harness.received + .whereType() + .last + .cancelFlagAddress, + ); + + harness.holdNextDispose = true; + final disposed = flagBackend.dispose(); + await harness.disposeReceived.future; + expect(flag.value, 1); + expect(allocator.freed, isEmpty); + + harness.acknowledgeHeldDispose(); + await disposed; + expect(allocator.freed, [flag.address]); + + flag.value = 0; + flagBackend.cancelTextToSpeech(); + expect(flag.value, 0); + + harness.finishHeldTextToSpeech(); + await pending; + expect(allocator.freed, [flag.address]); + }, + ); + + test('a failed worker startup frees the cancel flag', () async { + final allocator = _RecordingAllocator(); + addTearDown(allocator.release); + final flagBackend = NativeLlamaBackend( + workerEntrypoint: _failingInitializationWorkerEntry, + textToSpeechCancelFlagAllocator: allocator, + ); + addTearDown(flagBackend.dispose); + + await expectLater( + flagBackend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Never sent.'), + ), + throwsA(isA()), + ); + expect(allocator.allocated, hasLength(1)); + expect(allocator.freed, allocator.allocated); + }); + + test( + 'dispose keeps the cancel flag of a synthesis claimed during it', + () async { + final allocator = _RecordingAllocator(); + addTearDown(allocator.release); + final flagBackend = NativeLlamaBackend( + workerEntrypoint: _delayedTextToSpeechRejectingWorkerEntry, + textToSpeechCancelFlagAllocator: allocator, + ); + addTearDown(flagBackend.dispose); + final startup = flagBackend.modelLoad( + 'never.gguf', + const ModelParams(), + ); + final disposed = flagBackend.dispose(); + final synthesis = flagBackend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Next worker.'), + ); + await expectLater( + startup, + throwsA(isA()), + ); + await disposed; + expect(allocator.allocated, hasLength(1)); + expect(allocator.freed, isEmpty); + + await expectLater( + synthesis, + throwsA(isA()), + ); + expect(allocator.freed, allocator.allocated); + }, + ); + test('preserves speech error subtypes from the worker', () async { await expectLater( backend.synthesizeTextToSpeech( @@ -596,6 +757,41 @@ void main() { }); }); + test( + 'cancel reaches a worker blocked in a native step through the flag', + () async { + final events = ReceivePort(); + final eventQueue = StreamIterator(events); + final isolate = await Isolate.spawn(_flagPollingWorker, events.sendPort); + addTearDown(() { + isolate.kill(priority: Isolate.immediate); + events.close(); + }); + expect(await eventQueue.moveNext(), isTrue); + final backend = NativeLlamaBackend( + initialSendPort: eventQueue.current as SendPort, + ); + final pending = backend.synthesizeTextToSpeech( + 22, + 33, + const BackendTextToSpeechRequest(text: 'Blocked.'), + ); + expect(await eventQueue.moveNext(), isTrue); + expect(eventQueue.current, 'in-step'); + backend.cancelTextToSpeech(); + await expectLater( + pending, + throwsA( + isA().having( + (error) => error.message, + 'message', + 'flag observed', + ), + ), + ); + }, + ); + test('modelFree and contextFree are no-op without worker port', () async { final backend = NativeLlamaBackend(); @@ -1031,6 +1227,26 @@ void main() { }); } +void _delayedTextToSpeechRejectingWorkerEntry(SendPort initialSendPort) { + final receivePort = ReceivePort(); + initialSendPort.send(receivePort.sendPort); + receivePort.listen((message) async { + switch (message) { + case WorkerHandshake(): + await Future.delayed(const Duration(milliseconds: 50)); + message.sendPort.send(DoneResponse()); + case TextToSpeechSynthesizeRequest(): + message.sendPort.send( + ErrorResponse('rejected', kind: WorkerErrorKind.textToSpeech), + ); + case DisposeRequest(): + message.sendPort.send(null); + receivePort.close(); + Isolate.exit(); + } + }); +} + void _failingInitializationWorkerEntry(SendPort initialSendPort) { runLlamaWorkerForTesting( initialSendPort, @@ -1221,6 +1437,9 @@ class _FakeWorkerHarness { DisposeRequest? _heldDispose; Completer textToSpeechStarted = Completer(); TextToSpeechSynthesizeRequest? _heldTextToSpeech; + int? cancelFlagAtRequest; + bool holdNextDispose = false; + final Completer disposeReceived = Completer(); _FakeWorkerHarness() { _port.listen((message) { @@ -1386,6 +1605,9 @@ class _FakeWorkerHarness { ), ); case TextToSpeechSynthesizeRequest(): + cancelFlagAtRequest = Pointer.fromAddress( + message.cancelFlagAddress, + ).value; if (!textToSpeechStarted.isCompleted) { textToSpeechStarted.complete(); } @@ -1537,7 +1759,11 @@ class _FakeWorkerHarness { message.sendPort.send(DoneResponse()); } case DisposeRequest(): - if (holdDispose) { + if (holdNextDispose) { + holdNextDispose = false; + _heldDispose = message; + disposeReceived.complete(); + } else if (holdDispose) { _heldDispose = message; } else { message.sendPort.send(DoneResponse()); @@ -1550,6 +1776,11 @@ class _FakeWorkerHarness { SendPort get sendPort => _port.sendPort; + void acknowledgeHeldDispose() { + _heldDispose?.sendPort.send(DoneResponse()); + _heldDispose = null; + } + void releaseDispose() { holdDispose = false; _heldDispose?.sendPort.send(DoneResponse()); @@ -1591,3 +1822,47 @@ class _FakeWorkerHarness { _port.close(); } } + +void _flagPollingWorker(SendPort events) { + final requests = ReceivePort(); + events.send(requests.sendPort); + requests.listen((message) { + if (message is! TextToSpeechSynthesizeRequest) { + return; + } + final flag = Pointer.fromAddress(message.cancelFlagAddress); + events.send('in-step'); + final watch = Stopwatch()..start(); + while (flag.value == 0 && watch.elapsed < const Duration(seconds: 10)) { + sleep(const Duration(milliseconds: 1)); + } + message.sendPort.send( + ErrorResponse( + flag.value == 0 ? 'flag never observed' : 'flag observed', + kind: WorkerErrorKind.textToSpeech, + ), + ); + }); +} + +final class _RecordingAllocator implements Allocator { + final List allocated = []; + final List freed = []; + + @override + Pointer allocate(int byteCount, {int? alignment}) { + final pointer = malloc.allocate(byteCount, alignment: alignment); + pointer.asTypedList(byteCount).fillRange(0, byteCount, 0x5a); + allocated.add(pointer.address); + return pointer.cast(); + } + + @override + void free(Pointer pointer) => freed.add(pointer.address); + + void release() { + for (final address in allocated) { + malloc.free(Pointer.fromAddress(address)); + } + } +} diff --git a/test/unit/backends/llama_cpp/worker_test.dart b/test/unit/backends/llama_cpp/worker_test.dart index 256e89a9d..4d6213bbc 100644 --- a/test/unit/backends/llama_cpp/worker_test.dart +++ b/test/unit/backends/llama_cpp/worker_test.dart @@ -434,10 +434,12 @@ void main() { 2, 3, const BackendTextToSpeechRequest(text: 'Hello.'), + 0x5a5a0, responsePort.sendPort, ), ); await service.synthesisStarted.future; + expect(service.receivedCancelFlagAddress, 0x5a5a0); worker.sendPort.send(TextToSpeechCancelRequest()); await service.cancelObserved.future; service.releaseSynthesis(); @@ -486,6 +488,7 @@ void main() { 2, 3, const BackendTextToSpeechRequest(text: 'Hello.'), + 0x5a5a0, sendPort, ), ); @@ -863,6 +866,7 @@ class _BlockingTextToSpeechService extends LlamaCppService { final Completer cancelObserved = Completer(); final Completer _releaseSynthesis = Completer(); int cancelCalls = 0; + int? receivedCancelFlagAddress; void releaseSynthesis() { if (!_releaseSynthesis.isCompleted) { @@ -880,9 +884,11 @@ class _BlockingTextToSpeechService extends LlamaCppService { Future synthesizeTextToSpeech( int contextHandle, int mmContextHandle, - BackendTextToSpeechRequest request, { + BackendTextToSpeechRequest request, + int cancelFlagAddress, { void Function(BackendTextToSpeechProgress progress)? onProgress, }) async { + receivedCancelFlagAddress = cancelFlagAddress; if (!synthesisStarted.isCompleted) { synthesisStarted.complete(); } @@ -931,7 +937,8 @@ class _ThrowingTextToSpeechService extends LlamaCppService { Future synthesizeTextToSpeech( int contextHandle, int mmContextHandle, - BackendTextToSpeechRequest request, { + BackendTextToSpeechRequest request, + int cancelFlagAddress, { void Function(BackendTextToSpeechProgress progress)? onProgress, }) async { throw exception; diff --git a/test/unit/tooling/check_webgpu_bridge_tag_test.dart b/test/unit/tooling/check_webgpu_bridge_tag_test.dart index 0ed779fda..751ac5568 100644 --- a/test/unit/tooling/check_webgpu_bridge_tag_test.dart +++ b/test/unit/tooling/check_webgpu_bridge_tag_test.dart @@ -696,18 +696,33 @@ void main() { ); }); - test('a different wrapper in the same family is not the approved anchor', () { + test('a different wrapper in the same family is not the approved pin', () { final root = _fakeRuntimeRepo(bridgeTag: 'v0.2.0', nativeTag: 'v0.2.0-2'); expect( findBridgeRuntimeDrift(root, 'v0.2.0'), contains( - '$nativeLlamaCppTagPath pins v0.2.0-2, expected the bridge-qualified ' - 'native anchor $bridgeNativeReleaseTag', + '$nativeLlamaCppTagPath pins v0.2.0-2, expected the bridge-approved ' + 'native pin $bridgeApprovedNativePin', ), ); }); + test( + 'the qualified anchor is not accepted in place of the approved pin', + () { + final root = _fakeRuntimeRepo( + bridgeTag: bridgeLlamaCppTag, + nativeTag: bridgeNativeReleaseTag, + ); + + expect(findBridgeRuntimeDrift(root, bridgeLlamaCppTag), [ + '$nativeLlamaCppTagPath pins $bridgeNativeReleaseTag, expected the ' + 'bridge-approved native pin $bridgeApprovedNativePin', + ]); + }, + ); + test('stale divergence prose is reported', () { final root = _fakeRuntimeRepo( bridgeTag: 'v0.2.0', @@ -724,7 +739,7 @@ void main() { test('parity prose is accepted for matching upstream tags', () { final root = _fakeRuntimeRepo( bridgeTag: bridgeLlamaCppTag, - nativeTag: bridgeNativeReleaseTag, + nativeTag: bridgeApprovedNativePin, parityWording: true, ); diff --git a/tool/testing/check_webgpu_bridge_tag.dart b/tool/testing/check_webgpu_bridge_tag.dart index dbf127a71..c06108ac7 100644 --- a/tool/testing/check_webgpu_bridge_tag.dart +++ b/tool/testing/check_webgpu_bridge_tag.dart @@ -383,6 +383,11 @@ const String bridgeNativeRepository = 'leehack/llamadart-native'; /// The native release tag the pinned bridge assets were qualified against. const String bridgeNativeReleaseTag = 'v0.4.1'; +/// The native pin approved beside the pinned bridge assets: either +/// [bridgeNativeReleaseTag] or a native wrapper rebuild of it from the same +/// upstream llama.cpp commit. +const String bridgeApprovedNativePin = 'v0.4.1-1'; + /// The asset repository release that published the pinned bridge assets. const String bridgeAssetsReleaseId = '389783936'; @@ -439,11 +444,11 @@ final List bridgeProvenancePins = [ BridgeTagPin( 'doc/webgpu_bridge.md', RegExp( - r'^\(`(?[^`]+)`, both built from upstream llama\.cpp ' + r'^\(`(?[^`]+)`, both built from upstream llama\.cpp ' r'`(?[^`@]+)@(?[0-9a-f]{40})`\)\r?\n' r'even though the bridge asset tag `v\d+\.\d+\.\d+` differs from the ' r'native runtime tag\r?\n' - r'`v\d+\.\d+\.\d+`\. Provenance for this immutable consumer artifact: ' + r'`v\d+\.\d+\.\d+(?:-\d+)?`\. Provenance for this immutable consumer artifact: ' r'release `(?\d+)`,\r?\n' r'tag commit `(?[0-9a-f]{40})`, bridge source\r?\n' r'`(?[0-9a-f]{40})`, and manifest SHA-256\r?\n' @@ -454,11 +459,11 @@ final List bridgeProvenancePins = [ BridgeTagPin( 'website/docs/platforms/webgpu-bridge.md', RegExp( - r'^ \(`(?[^`]+)`, both built from upstream ' + r'^ \(`(?[^`]+)`, both built from upstream ' r'`(?[^`@]+)@(?[0-9a-f]{40})`\)\r?\n' r' even though the bridge asset tag `v\d+\.\d+\.\d+` differs from the ' r'native runtime tag\r?\n' - r' `v\d+\.\d+\.\d+`\. Pinned artifact provenance: release ' + r' `v\d+\.\d+\.\d+(?:-\d+)?`\. Pinned artifact provenance: release ' r'`(?\d+)`, tag commit\r?\n' r' `(?[0-9a-f]{40})`, bridge source\r?\n' r' `(?[0-9a-f]{40})`, manifest SHA-256\r?\n' @@ -470,7 +475,7 @@ final List bridgeProvenancePins = [ /// The expected value behind each [bridgeProvenancePins] group name. Map get bridgeProvenanceValues => { - 'nativeReleaseTag': bridgeNativeReleaseTag, + 'nativePin': bridgeApprovedNativePin, 'upstreamTag': bridgeLlamaCppTag, 'upstreamCommit': bridgeLlamaCppCommit, 'releaseId': bridgeAssetsReleaseId, @@ -532,10 +537,10 @@ List findBridgeRuntimeDrift(Directory repoRoot, String bridgeTag) { 'expected stable vMAJOR.MINOR.PATCH(-N) or nightly bNNNN(-N|-llamadart.N)', ); } else { - if (nativeTag != bridgeNativeReleaseTag) { + if (nativeTag != bridgeApprovedNativePin) { problems.add( - '$nativeLlamaCppTagPath pins $nativeTag, expected the bridge-qualified ' - 'native anchor $bridgeNativeReleaseTag', + '$nativeLlamaCppTagPath pins $nativeTag, expected the bridge-approved ' + 'native pin $bridgeApprovedNativePin', ); } if (normalizedNativeFamily != bridgeTag) { diff --git a/website/docs/changelog/recent-releases.md b/website/docs/changelog/recent-releases.md index 73e986915..127352928 100644 --- a/website/docs/changelog/recent-releases.md +++ b/website/docs/changelog/recent-releases.md @@ -9,6 +9,17 @@ For canonical full release notes, use: ## Unreleased +- Updated the default llama.cpp native runtime pin to + `leehack/llamadart-native@v0.4.1-1`, keeping the `v0.4.1` llama.cpp + ABI/bindings while picking up wrapper-only native fixes. Refreshed the + `llamadart_llama_cpp_flutter` Apple SwiftPM checksum and aligned current + README/website native override docs. +- Stop a Qwen3-TTS audio decode at its next chunk boundary when native + text-to-speech is cancelled, instead of finishing the native step in + progress first. This needs llamadart-native v0.4.1-1 or later; older + runtimes keep the previous behaviour + ([llamadart-native#86](https://github.com/leehack/llamadart-native/issues/86), + [#322](https://github.com/leehack/llamadart/issues/322)). - Add an experimental `DecisionEngine` for Laya-style decision models (a ModernBERT encoder GGUF plus a safetensors head) on native llama.cpp, with typed `ChoiceKey`, `ScoreKey` and `NoulKey` questions diff --git a/website/docs/getting-started/installation.md b/website/docs/getting-started/installation.md index 90681fedd..d300edd9a 100644 --- a/website/docs/getting-started/installation.md +++ b/website/docs/getting-started/installation.md @@ -88,7 +88,7 @@ hooks: llamadart: # Optional. Defaults to llamadart's tested native runtime pin. # Use a leehack/llamadart-native release tag when testing another build. - llamadart_native_tag: v0.4.1 + llamadart_native_tag: v0.4.1-1 # Optional. GitHub repository slug or github.com URL. llamadart_native_repository: leehack/llamadart-native @@ -115,7 +115,7 @@ per-target module list. Native source overrides are for compatibility testing. They do not regenerate Dart FFI bindings or symbol lookups, so the selected binary still must be ABI- and symbol-compatible with the default -`leehack/llamadart-native@v0.4.1` runtime. +`leehack/llamadart-native@v0.4.1-1` runtime. Available native tags are published on the [`leehack/llamadart-native` releases page](https://github.com/leehack/llamadart-native/releases). @@ -143,7 +143,7 @@ gh release list --repo leehack/llamadart-native --limit 20 Before overriding, confirm the release includes the asset for your target. The hook downloads files named `llamadart-native--.tar.gz`, for example -`llamadart-native-windows-x64-v0.4.1.tar.gz`. +`llamadart-native-windows-x64-v0.4.1-1.tar.gz`. For local testing, `llamadart_native_path` may point directly at a bundle archive, at an extracted bundle directory, or at a directory containing `//`, `/`, or the expected archive file. diff --git a/website/docs/guides/performance-tuning.md b/website/docs/guides/performance-tuning.md index aa592c672..4cdc66f52 100644 --- a/website/docs/guides/performance-tuning.md +++ b/website/docs/guides/performance-tuning.md @@ -148,7 +148,7 @@ Guidelines: - DSpark is an experimental, opt-in native llama.cpp external draft-model strategy. Use `SpeculativeDecodingConfig.draftDspark(draftModelPath: ...)`; it is never selected automatically, maps to upstream `draft-dspark`, requires - at least the `b10356-llamadart.1` wrapper fix; the package-pinned `v0.4.1` + at least the `b10356-llamadart.1` wrapper fix; the package-pinned `v0.4.1-1` runtime satisfies that ABI, supports speculators-format checkpoints, and adds LFM2 target/draft support. DSpark remains subject to target/draft/backend compatibility. diff --git a/website/docs/guides/text-to-speech.md b/website/docs/guides/text-to-speech.md index cec3034ad..a4c2c883f 100644 --- a/website/docs/guides/text-to-speech.md +++ b/website/docs/guides/text-to-speech.md @@ -129,7 +129,10 @@ responsible for its own files, byte buffers, permissions, and disclosures. ## Cancellation, concurrency, and buffering Call `task.cancel()` to request cooperative cancellation. Cancelling only the -event-stream subscription does not cancel synthesis. +event-stream subscription does not cancel synthesis. On native llama.cpp with +llamadart-native v0.4.1-1 or later, a cancel stops a Qwen3-TTS audio decode in +progress at its next chunk boundary. Older runtimes finish the native step in +progress first, which can include the whole audio decode. All typed STT and TTS wrappers over one `LlamaEngine` share a one-task speech lease. Do not run chat generation, transcription, or another synthesis on the diff --git a/website/docs/platforms/support-matrix.md b/website/docs/platforms/support-matrix.md index f47997706..0c307c9ab 100644 --- a/website/docs/platforms/support-matrix.md +++ b/website/docs/platforms/support-matrix.md @@ -8,7 +8,7 @@ backend-module configuration for `llamadart`. The native-assets hook currently pins `llamadart-native` tag -`v0.4.1` and +`v0.4.1-1` and `litert-lm-native` release `v0.17.0-6` (`lib/src/hook/native_release_pins.dart`). Apps can override the llama.cpp native GitHub source with `hooks.user_defines.llamadart.llamadart_native_tag` and @@ -312,7 +312,7 @@ device/model bundle, use `cpu` or `gpu` for that artifact. a web load failure as a package bug. The [WebGPU Bridge](./webgpu-bridge) page has the browser-console probe and Flutter Web smoke-test path. -## Current llama.cpp module availability by bundle (`v0.4.1`) +## Current llama.cpp module availability by bundle (`v0.4.1-1`) | Bundle key | Available backend modules in bundle | | --- | --- | @@ -360,7 +360,7 @@ hooks: user_defines: llamadart: # Optional. Defaults to llamadart's tested native runtime pin. - llamadart_native_tag: v0.4.1 + llamadart_native_tag: v0.4.1-1 # Optional. GitHub repository slug or github.com URL. llamadart_native_repository: leehack/llamadart-native diff --git a/website/docs/platforms/webgpu-bridge.md b/website/docs/platforms/webgpu-bridge.md index 4b478d7a6..b526144a1 100644 --- a/website/docs/platforms/webgpu-bridge.md +++ b/website/docs/platforms/webgpu-bridge.md @@ -202,15 +202,17 @@ cannot report success before the bridge exposes `prefetchModelToCache(...)`. remains unsupported; use memory64. - `v0.1.39+` remains the compatibility floor for bridge asset capabilities. - The pinned `v0.1.44` bridge assets embed llama.cpp `v0.4.1`, matching the native runtime - (`v0.4.1`, both built from upstream `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`) + (`v0.4.1-1`, both built from upstream `v0.4.1@b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`) even though the bridge asset tag `v0.1.44` differs from the native runtime tag - `v0.4.1`. Pinned artifact provenance: release `389783936`, tag commit + `v0.4.1-1`. Pinned artifact provenance: release `389783936`, tag commit `fdafd9f8cbdb9bf99c359536595eff9a23095379`, bridge source `89178be67c3c84300bc1b129182bd5bc5a8e21fc`, manifest SHA-256 `8d61f453753ac7a7d839ac12318b70986a814748d86029993118c19454293aa9`. The - bridge assets provision an explicit 1 MiB stack for both wasm32 and memory64, - preventing graph-parameter growth from overflowing Emscripten's 64 KiB - default during memory64 Qwen3-ASR context construction. + bridge assets were qualified against native `v0.4.1`; `v0.4.1-1` rebuilds it + from the same upstream commit. The bridge assets provision an explicit 1 MiB + stack for both wasm32 and memory64, preventing graph-parameter growth from + overflowing Emscripten's 64 KiB default during memory64 Qwen3-ASR context + construction. - `v0.1.12+` bridge assets forward native-compatible `ModelParams` load tuning fields, including multi-sequence slots, KV cache type, flash attention, RoPE overrides, split mode, and main GPU.