diff --git a/.agents/skills/add-api-tests/SKILL.md b/.agents/skills/add-api-tests/SKILL.md index 57cffd9276..dae14aa852 100644 --- a/.agents/skills/add-api-tests/SKILL.md +++ b/.agents/skills/add-api-tests/SKILL.md @@ -55,7 +55,7 @@ Key helpers: | Helper | Use | | :-------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | -| `fakeJsi.registerModel(path, program)` | Make `loadModel(path)` succeed with a given schema and `execute` | +| `fakeJsi.registerModel(source, program)` | Make `loadModel(source)` succeed with a given schema and `execute` | | `fakeJsi.registerTokenizer(path, vocabulary)` | Same for `loadTokenizer` | | `fakeJsi.registerLLMRunner(path, program)` | Same for `createLLMRunner`: a context window and the responses to generate, one per `generate` call | | `fakePhonemizer.serve(text, phonemes)` | Script the native grapheme-to-phoneme converter | diff --git a/docs/docs/03-core-and-advanced/02-models-and-tensors.md b/docs/docs/03-core-and-advanced/02-models-and-tensors.md index 9250088c08..4b14e807e6 100644 --- a/docs/docs/03-core-and-advanced/02-models-and-tensors.md +++ b/docs/docs/03-core-and-advanced/02-models-and-tensors.md @@ -249,6 +249,17 @@ const model = await wrapAsync(loadModel)('/path/to/model.pte', { }); ``` +`loadModel()` also accepts an in-memory `ArrayBuffer` or a custom +`ModelDataLoader` (`{ size(): number, load(offset: number, size: number): ArrayBuffer }`), +and an optional `dataMapPath` for models exported with an external `.ptd` +weight archive: + +```typescript +const model = await wrapAsync(loadModel)(programBuffer, { + dataMapPath: '/path/to/weights.ptd', +}); +``` + See [Worklets & Threading](./06-worklets-and-threading.md) for how [`wrapAsync`](../06-api-reference/functions/wrapAsync.md) and worklet runtimes fit together. diff --git a/packages/react-native-executorch/__tests__/core/model.test.ts b/packages/react-native-executorch/__tests__/core/model.test.ts index 5e263e016d..5d3a87f5b2 100644 --- a/packages/react-native-executorch/__tests__/core/model.test.ts +++ b/packages/react-native-executorch/__tests__/core/model.test.ts @@ -38,6 +38,40 @@ describe('loadModel', () => { model.dispose(); }); + it('loads from an in-memory ArrayBuffer and forwards dataMapPath', () => { + const schema = exported(method('forward', [f32(4)], [f32(4)])); + const modelBuffer = new Uint8Array([1, 2, 3, 4]).buffer; + const weightsBuffer = new Uint8Array([5, 6, 7, 8]).buffer; + fakeJsi.registerModel(modelBuffer, { schema }); + + const model = loadModel(modelBuffer, { dataMapPath: weightsBuffer }); + expect(model.path).toBe(''); + expect(fakeJsi.modelLoads()).toEqual([ + { source: modelBuffer, options: { dataMapPath: weightsBuffer } }, + ]); + model.dispose(); + }); + + it('loads from a custom ModelDataLoader and forwards a loader dataMapPath', () => { + const schema = exported(method('forward', [f32(4)], [f32(4)])); + const modelLoader = { + size: () => 4, + load: (_offset: number, size: number) => new ArrayBuffer(size), + }; + const weightsLoader = { + size: () => 8, + load: (_offset: number, size: number) => new ArrayBuffer(size), + }; + fakeJsi.registerModel(modelLoader, { schema }); + + const model = loadModel(modelLoader, { dataMapPath: weightsLoader }); + expect(model.path).toBe(''); + expect(fakeJsi.modelLoads()).toEqual([ + { source: modelLoader, options: { dataMapPath: weightsLoader } }, + ]); + model.dispose(); + }); + it('propagates a load failure as a thrown error', () => { expect(() => loadModel('/models/missing.pte')).toThrow(/missing.pte/); }); diff --git a/packages/react-native-executorch/__tests__/hooks/useModel.test.ts b/packages/react-native-executorch/__tests__/hooks/useModel.test.ts index 5f1b7c45f3..e4855d6e96 100644 --- a/packages/react-native-executorch/__tests__/hooks/useModel.test.ts +++ b/packages/react-native-executorch/__tests__/hooks/useModel.test.ts @@ -78,6 +78,29 @@ describe('useModel', () => { expect(instances).toHaveLength(1); }); + it('rebuilds when an ArrayBuffer inside the config changes identity', async () => { + const instances: Instance[] = []; + const factory = jest.fn(async (config: { id: string; modelPath: ArrayBuffer }) => { + const instance = { dispose: jest.fn(), id: config.id }; + instances.push(instance); + return instance; + }); + const first = new Uint8Array([1, 2]).buffer; + const second = new Uint8Array([3, 4]).buffer; + const { result, rerender } = await renderHook( + ({ config }: { config: { id: string; modelPath: ArrayBuffer } }) => useModel(factory, config), + { initialProps: { config: { id: 'a', modelPath: first } } } + ); + await waitFor(() => expect(result.current.model).toBeDefined()); + + await rerender({ config: { id: 'a', modelPath: first } }); + expect(factory).toHaveBeenCalledTimes(1); + + await rerender({ config: { id: 'b', modelPath: second } }); + await waitFor(() => expect(result.current.model?.id).toBe('b')); + expect(instances[0]!.dispose).toHaveBeenCalledTimes(1); + }); + it('disposes an instance that arrives after unmount', async () => { const gate = deferred(); const instance = { dispose: jest.fn(), id: 'late' }; diff --git a/packages/react-native-executorch/__tests__/support/fakeJsi.ts b/packages/react-native-executorch/__tests__/support/fakeJsi.ts index 3dbc9062e5..d8e6b6723d 100644 --- a/packages/react-native-executorch/__tests__/support/fakeJsi.ts +++ b/packages/react-native-executorch/__tests__/support/fakeJsi.ts @@ -14,6 +14,7 @@ */ import { TextDecoder } from 'util'; import type { ConcreteDim, ModelSpec } from '../../src/core/schema'; +import type { ModelSource } from '../../src/core/model'; import type { DType } from '../../src/core/tensor'; import { FakeTensor, tensorTracker } from './fakeTensor'; import { cv, fakePhonemizer, llm, math, speech } from './fakeOps'; @@ -43,9 +44,13 @@ export type FakeProgram = { execute?: FakeExecute; }; -const programs = new Map(); +const programs = new Map(); const liveModels = new Set(); const executions: { path: string; methodName: string }[] = []; +const modelLoads: { + source: ModelSource; + options?: { eagerLoadMethods?: boolean; dataMapPath?: ModelSource }; +}[] = []; class FakeModel { readonly path: string; @@ -227,12 +232,18 @@ const jsi = { createTensor: (shape: number[], dtype: DType) => new FakeTensor(dtype, shape), - loadModel: (path: string, _options?: { eagerLoadMethods?: boolean }) => { - const program = programs.get(path); + loadModel: ( + source: ModelSource, + options?: { eagerLoadMethods?: boolean; dataMapPath?: ModelSource } + ) => { + const program = programs.get(source); + const label = + typeof source === 'string' ? source : source instanceof ArrayBuffer ? '' : ''; if (!program) { - throw new Error(`loadModel: no program registered at '${path}' (register one via fakeJsi)`); + throw new Error(`loadModel: no program registered at '${label}' (register one via fakeJsi)`); } - return new FakeModel(path, program); + modelLoads.push({ source, options }); + return new FakeModel(label, program); }, getExecuTorchRegisteredBackends: () => [...registeredBackends], @@ -266,12 +277,12 @@ export function installFakeJsi(): void { export const fakeJsi = { /** - * Makes `loadModel(path)` succeed and return `program`. - * @param path The model path a pipeline will be pointed at. + * Makes `loadModel(source)` succeed and return `program`. + * @param source The model path or buffer a pipeline will be pointed at. * @param program The schema, backends and `execute` behavior to serve. */ - registerModel(path: string, program: FakeProgram): void { - programs.set(path, program); + registerModel(source: ModelSource, program: FakeProgram): void { + programs.set(source, program); }, /** @@ -309,6 +320,14 @@ export const fakeJsi = { return executions; }, + /** @returns Every `loadModel` call made so far, in order. */ + modelLoads(): readonly { + source: ModelSource; + options?: { eagerLoadMethods?: boolean; dataMapPath?: ModelSource }; + }[] { + return modelLoads; + }, + /** @returns Paths of models that were loaded and not disposed. */ liveModels(): string[] { return [...liveModels].sort(); @@ -336,6 +355,7 @@ export const fakeJsi = { liveModels.clear(); liveTokenizers.clear(); executions.length = 0; + modelLoads.length = 0; registeredBackends = ['XnnpackBackend', 'CoreMLBackend']; jsi.isEmulator = false; fakePhonemizer.reset(); diff --git a/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts index da24ab75fb..87832930e5 100644 --- a/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts @@ -888,3 +888,41 @@ describe('createLLMMultimodalRunner — audio input', () => { } }); }); + +describe('createLLMRunner — in-memory and external weights', () => { + it('loads from an ArrayBuffer and forwards dataMapPath', async () => { + const modelBuffer = new Uint8Array([1, 2, 3, 4]).buffer; + const weightsBuffer = new Uint8Array([5, 6, 7, 8]).buffer; + fakeJsi.registerModel(modelBuffer, { + schema: TEXT_SPEC, + execute: (methodName) => { + switch (methodName) { + case 'get_max_seq_len': + return [128]; + case 'get_max_context_len': + return [512]; + case 'get_vocab_size': + return [100]; + case 'use_kv_cache': + case 'enable_dynamic_shape': + return [true]; + case 'get_eos_ids': + return [1]; + default: + return undefined; + } + }, + }); + + const runner = tracked( + await createLLMRunner(modelBuffer, TOKENIZER_PATH, undefined, undefined, { + dataMapPath: weightsBuffer, + }) + ); + + expect(runner.modelPath).toBe(''); + expect(fakeJsi.modelLoads()).toEqual([ + { source: modelBuffer, options: { dataMapPath: weightsBuffer } }, + ]); + }); +}); diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 40c01d474b..7a0abe7af9 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -19,6 +21,8 @@ #include +#include +#include #include #include #include @@ -28,6 +32,174 @@ namespace { using rnexecutorch::core::error::RnExecuTorchErrorCode; using rnexecutorch::core::error::RnExecuTorchException; +void freeAlignedBuffer(void * /*context*/, void *data, size_t /*size*/) { + ::operator delete[](static_cast(data), std::align_val_t(64)); +} + +class OwnedBufferDataLoader final : public executorch::runtime::DataLoader { +public: + OwnedBufferDataLoader(const uint8_t *src, size_t size) + : storage_(static_cast(size == 0 ? nullptr : ::operator new[](size, std::align_val_t(64))), + [](uint8_t *ptr) { ::operator delete[](ptr, std::align_val_t(64)); }), + loader_(storage_.get(), size) { + if (size > 0) { + std::memcpy(storage_.get(), src, size); + } + } + + ET_NODISCARD executorch::runtime::Result + load(size_t offset, size_t size, const SegmentInfo &segmentInfo) const override { + return loader_.load(offset, size, segmentInfo); + } + + ET_NODISCARD executorch::runtime::Error + load_into(size_t offset, size_t size, const SegmentInfo &segmentInfo, void *buffer) const override { + return loader_.load_into(offset, size, segmentInfo, buffer); + } + + ET_NODISCARD executorch::runtime::Result size() const override { + return loader_.size(); + } + +private: + std::unique_ptr storage_; + executorch::extension::BufferDataLoader loader_; +}; + +class JsiDataLoader final : public executorch::runtime::DataLoader { +public: + JsiDataLoader(facebook::jsi::Runtime &rt, std::string ctx, facebook::jsi::Object loader, + facebook::jsi::Function sizeFn, facebook::jsi::Function loadFn, + std::shared_ptr lastError) + : rt_(&rt), ctx_(std::move(ctx)), loader_(std::move(loader)), + sizeFn_(std::move(sizeFn)), loadFn_(std::move(loadFn)), + lastError_(std::move(lastError)) {} + + ET_NODISCARD executorch::runtime::Result + load(size_t offset, size_t size, const SegmentInfo & /*segmentInfo*/) const override { + if (size == 0) { + return executorch::runtime::FreeableBuffer(nullptr, 0, nullptr); + } + if (rt_ == nullptr || !loader_.has_value() || !loadFn_.has_value()) { + return executorch::runtime::Error::InvalidState; + } + try { + auto result = loadFn_->callWithThis(*rt_, *loader_, + facebook::jsi::Value(static_cast(offset)), + facebook::jsi::Value(static_cast(size))); + auto buffer = rnexecutorch::core::conversions::asType( + *rt_, ctx_ + ".load()", result); + if (buffer.size(*rt_) != size) { + throw rnexecutorch::core::error::InvalidArgument( + std::format("{}.load() returned {} bytes, expected {}", ctx_, buffer.size(*rt_), size)); + } + auto *copy = static_cast(::operator new[](size, std::align_val_t(64))); + std::memcpy(copy, buffer.data(*rt_), size); + return executorch::runtime::FreeableBuffer(copy, size, freeAlignedBuffer); + } catch (...) { + if (lastError_ && !*lastError_) { + *lastError_ = std::current_exception(); + } + return executorch::runtime::Error::AccessFailed; + } + } + + ET_NODISCARD executorch::runtime::Error + load_into(size_t offset, size_t size, const SegmentInfo &segmentInfo, void *buffer) const override { + if (buffer == nullptr && size != 0) { + return executorch::runtime::Error::InvalidArgument; + } + if (size == 0) { + return executorch::runtime::Error::Ok; + } + auto loaded = load(offset, size, segmentInfo); + if (!loaded.ok()) { + return loaded.error(); + } + std::memcpy(buffer, loaded->data(), size); + return executorch::runtime::Error::Ok; + } + + ET_NODISCARD executorch::runtime::Result size() const override { + if (rt_ == nullptr || !loader_.has_value() || !sizeFn_.has_value()) { + return executorch::runtime::Error::InvalidState; + } + try { + auto result = sizeFn_->callWithThis(*rt_, *loader_); + return static_cast( + rnexecutorch::core::conversions::asType(*rt_, ctx_ + ".size()", result)); + } catch (...) { + if (lastError_ && !*lastError_) { + *lastError_ = std::current_exception(); + } + return executorch::runtime::Error::AccessFailed; + } + } + + void detachRuntime() { + rt_ = nullptr; + loader_.reset(); + sizeFn_.reset(); + loadFn_.reset(); + } + +private: + facebook::jsi::Runtime *rt_; + std::string ctx_; + std::optional loader_; + std::optional sizeFn_; + std::optional loadFn_; + std::shared_ptr lastError_; +}; + +std::unique_ptr +createDataLoader(facebook::jsi::Runtime &rt, const std::string &ctx, const facebook::jsi::Value &val, + const std::shared_ptr &lastError, + JsiDataLoader **outJsiLoader, std::string *outPath) { + if (val.isString()) { + auto path = rnexecutorch::core::conversions::asType(rt, ctx, val); + if (outPath != nullptr) { + *outPath = path; + } + auto loader = executorch::extension::MmapDataLoader::from( + path.c_str(), executorch::extension::MmapDataLoader::MlockConfig::NoMlock); + if (!loader.ok()) { + throw rnexecutorch::core::error::LoadFailed( + std::format("Failed to load model from '{}': {}", path, + executorch::runtime::to_string(loader.error())), + loader.error()); + } + return std::make_unique(std::move(loader.get())); + } + if (val.isObject()) { + auto obj = val.asObject(rt); + if (obj.isArrayBuffer(rt)) { + auto buffer = obj.getArrayBuffer(rt); + if (outPath != nullptr) { + *outPath = ""; + } + return std::make_unique(buffer.data(rt), buffer.size(rt)); + } + if (!obj.isArray(rt) && !obj.isFunction(rt)) { + auto sizeFn = rnexecutorch::core::conversions::getRequiredProperty( + rt, ctx, obj, "size"); + auto loadFn = rnexecutorch::core::conversions::getRequiredProperty( + rt, ctx, obj, "load"); + if (outPath != nullptr) { + *outPath = ""; + } + auto loader = std::make_unique( + rt, ctx, std::move(obj), std::move(sizeFn), std::move(loadFn), lastError); + if (outJsiLoader != nullptr) { + *outJsiLoader = loader.get(); + } + return loader; + } + } + throw rnexecutorch::core::error::InvalidArgument( + ctx + " must be a string, an ArrayBuffer, or a ModelDataLoader"); +} + template T unwrap(RnExecuTorchErrorCode code, const std::string &ctx, executorch::runtime::Result result) { if (!result.ok()) { @@ -72,9 +244,15 @@ namespace conversions = rnexecutorch::core::conversions; using executorch::extension::Module; using rnexecutorch::core::tensor::TensorHostObject; -ModelHostObject::ModelHostObject(const std::string &modelPath, const bool eagerLoadMethods) - : modelPath_(modelPath), - etModule_(std::make_unique(modelPath, Module::LoadMode::Mmap)) { +ModelHostObject::ModelHostObject(std::string modelPath, + std::unique_ptr modelLoader, + std::unique_ptr dataMapLoader, + const bool eagerLoadMethods, + const bool usesJsiDataLoader) + : modelPath_(std::move(modelPath)), + etModule_(std::make_unique( + std::move(modelLoader), nullptr, nullptr, nullptr, std::move(dataMapLoader))), + usesJsiDataLoader_(usesJsiDataLoader) { auto loadError = etModule_->load(); if (!etModule_->is_loaded()) { @@ -212,6 +390,11 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { schema::validateRuntimeConstraints(rt, methodSpec.runtimeConstraints, inputShapes, std::format("execute: method '{}'", methodName)); if (!self->etModule_->is_method_loaded(methodName)) { + if (self->usesJsiDataLoader_) { + throw error::InvalidState( + "execute: Methods of a model loaded from a ModelDataLoader must be loaded eagerly " + "(eagerLoadMethods: true)"); + } requireRegisteredBackends(self->modelPath_, methodName, self->backends_.at(methodName)); } @@ -353,18 +536,44 @@ void install_loadModel(jsi::Runtime &rt, jsi::Object &module) { const auto *name = "loadModel"; auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { if (count < 1 || count > 2) { - throw error::InvalidArgument("loadModel: Usage: loadModel(path, options?)"); + throw error::InvalidArgument("loadModel: Usage: loadModel(source, options?)"); } - auto modelPath = conversions::asType(rt, "loadModel: path", args[0]); + std::string modelPath; + auto lastLoaderError = std::make_shared(); + JsiDataLoader *jsiModelLoader = nullptr; + JsiDataLoader *jsiDataMapLoader = nullptr; + auto modelLoader = createDataLoader( + rt, "loadModel: source", args[0], lastLoaderError, &jsiModelLoader, &modelPath); + std::unique_ptr dataMapLoader; bool eagerLoadMethods = true; if (count >= 2 && !args[1].isUndefined() && !args[1].isNull()) { auto options = conversions::asType(rt, "loadModel: options", args[1]); eagerLoadMethods = conversions::getOptionalProperty(rt, "loadModel: options", options, "eagerLoadMethods").value_or(true); + if (auto dataMap = conversions::getOptionalProperty(rt, "loadModel: options", options, "dataMapPath")) { + dataMapLoader = createDataLoader( + rt, "loadModel: options: option 'dataMapPath'", *dataMap, lastLoaderError, &jsiDataMapLoader, nullptr); + } } - return jsi::Object::createFromHostObject(rt, std::make_shared(modelPath, eagerLoadMethods)); + try { + const bool usesJsiDataLoader = jsiModelLoader != nullptr || jsiDataMapLoader != nullptr; + auto hostObject = std::make_shared( + modelPath, std::move(modelLoader), std::move(dataMapLoader), eagerLoadMethods, usesJsiDataLoader); + if (jsiModelLoader != nullptr) { + jsiModelLoader->detachRuntime(); + } + if (jsiDataMapLoader != nullptr) { + jsiDataMapLoader->detachRuntime(); + } + return jsi::Object::createFromHostObject(rt, hostObject); + } catch (...) { + if (*lastLoaderError) { + std::rethrow_exception(*lastLoaderError); + } + throw; + } }; auto fn = jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 2, error::guarded(fnBody)); diff --git a/packages/react-native-executorch/cpp/core/model.h b/packages/react-native-executorch/cpp/core/model.h index 03eb603bdd..93fddd2ec2 100644 --- a/packages/react-native-executorch/cpp/core/model.h +++ b/packages/react-native-executorch/cpp/core/model.h @@ -12,6 +12,7 @@ #include #include +#include namespace rnexecutorch::core::model { namespace jsi = facebook::jsi; @@ -40,7 +41,11 @@ class ModelHostObject final : public jsi::HostObject, * @param modelPath Absolute file system path to the `.pte` model binary. * @param eagerLoadMethods Whether to eagerly load and compile all model methods. */ - explicit ModelHostObject(const std::string &modelPath, bool eagerLoadMethods); + explicit ModelHostObject(std::string modelPath, + std::unique_ptr modelLoader, + std::unique_ptr dataMapLoader, + bool eagerLoadMethods, + bool usesJsiDataLoader); jsi::Value get(jsi::Runtime &rt, const jsi::PropNameID &name) override; std::vector getPropertyNames(jsi::Runtime &rt) override; @@ -69,6 +74,10 @@ class ModelHostObject final : public jsi::HostObject, /// Unique pointer to the underlying ExecuTorch module instance. std::unique_ptr etModule_; + + /// True when the program or external data map uses a JS-backed DataLoader + /// whose JS callbacks are detached once `loadModel` returns. + bool usesJsiDataLoader_ = false; }; void install_loadModel(jsi::Runtime &rt, jsi::Object &module); diff --git a/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp b/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp index bd163ae0d5..e0863ec279 100644 --- a/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp +++ b/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -146,6 +148,93 @@ TEST_F(ModelTest, LoadsAProgramAndExposesItsPath) { EXPECT_EQ(evalString(std::format("{} return model.path;", loadFixtureJs())), kFixture); } +TEST_F(ModelTest, LoadsAProgramFromAnArrayBuffer) { + std::ifstream file(kFixture, std::ios::binary); + std::vector bytes((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + ASSERT_FALSE(bytes.empty()); + + auto arrayBufferCtor = rt().global().getPropertyAsFunction(rt(), "ArrayBuffer"); + auto bufferObj = arrayBufferCtor.callAsConstructor(rt(), static_cast(bytes.size())).getObject(rt()); + std::memcpy(bufferObj.getArrayBuffer(rt()).data(rt()), bytes.data(), bytes.size()); + rt().global().setProperty(rt(), "__fixtureBuffer", bufferObj); + + EXPECT_EQ(evalString(R"( + const model = __rnexecutorch_jsi__.loadModel(__fixtureBuffer, { eagerLoadMethods: false }); + return `${model.path}:${model.backends.forward.join(',')}`; + )"), + ":XnnpackBackend"); +} + +TEST_F(ModelTest, LoadsAProgramFromAModelDataLoader) { + std::ifstream file(kFixture, std::ios::binary); + std::vector bytes((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + ASSERT_FALSE(bytes.empty()); + + auto arrayBufferCtor = rt().global().getPropertyAsFunction(rt(), "ArrayBuffer"); + auto bufferObj = arrayBufferCtor.callAsConstructor(rt(), static_cast(bytes.size())).getObject(rt()); + std::memcpy(bufferObj.getArrayBuffer(rt()).data(rt()), bytes.data(), bytes.size()); + rt().global().setProperty(rt(), "__fixtureBuffer", bufferObj); + + EXPECT_EQ(evalString(R"( + let calls = 0; + const loader = { + size: () => __fixtureBuffer.byteLength, + load: (offset, size) => { + calls += 1; + return __fixtureBuffer.slice(offset, offset + size); + }, + }; + const model = __rnexecutorch_jsi__.loadModel(loader, { eagerLoadMethods: false }); + return `${model.path}:${model.backends.forward.join(',')}:${calls > 0}`; + )"), + ":XnnpackBackend:true"); +} + +TEST_F(ModelTest, RejectsAModelDataLoaderReturningWrongByteLength) { + auto thrown = evalThrowing(R"( + const loader = { + size: () => 64, + load: () => new ArrayBuffer(4), + }; + __rnexecutorch_jsi__.loadModel(loader, { eagerLoadMethods: false }); + )"); + + EXPECT_TRUE(isCodedError(thrown, "INVALID_ARGUMENT", "loadModel: source.load() returned 4 bytes")); +} + +TEST_F(ModelTest, RejectsAModelDataLoaderMissingLoad) { + auto thrown = evalThrowing(R"( + __rnexecutorch_jsi__.loadModel({ size: () => 64 }, { eagerLoadMethods: false }); + )"); + + EXPECT_TRUE(isCodedError(thrown, "INVALID_ARGUMENT", "option 'load' is required")); +} + +TEST_F(ModelTest, PropagatesExceptionThrownInsideModelDataLoader) { + auto thrown = evalThrowing(R"( + const loader = { + size: () => 64, + load: () => { throw new Error('disk read failed'); }, + }; + __rnexecutorch_jsi__.loadModel(loader, { eagerLoadMethods: false }); + )"); + + EXPECT_THAT(thrown.message, HasSubstr("disk read failed")); +} + +TEST_F(ModelTest, AcceptsUndefinedDataMapPathAndRejectsInvalidDataMapPath) { + EXPECT_EQ(evalString(std::format( + "const m = __rnexecutorch_jsi__.loadModel('{}', {{ eagerLoadMethods: false, dataMapPath: undefined }}); return m.path;", + kFixture)), + kFixture); + + auto thrown = evalThrowing(std::format( + "__rnexecutorch_jsi__.loadModel('{}', {{ eagerLoadMethods: false, dataMapPath: 123 }});", kFixture)); + EXPECT_TRUE(isCodedError( + thrown, "INVALID_ARGUMENT", + "loadModel: options: option 'dataMapPath' must be a string, an ArrayBuffer, or a ModelDataLoader")); +} + TEST_F(ModelTest, ExposesTheSchemaToJs) { EXPECT_TRUE(evalBool(std::format( "{} return typeof model.schema.forward === 'object';", loadFixtureJs()))); @@ -199,12 +288,12 @@ TEST_F(ModelTest, ReportsAMissingFileAsLoadFailed) { TEST_F(ModelTest, RejectsWrongArgumentCount) { EXPECT_THAT(evalThrowingMessage("__rnexecutorch_jsi__.loadModel();"), - HasSubstr("Usage: loadModel(path, options?)")); + HasSubstr("Usage: loadModel(source, options?)")); } TEST_F(ModelTest, RejectsANonStringPath) { EXPECT_THAT(evalThrowingMessage("__rnexecutorch_jsi__.loadModel(42);"), - HasSubstr("must be a string")); + HasSubstr("must be a string, an ArrayBuffer, or a ModelDataLoader")); } TEST_F(ModelTest, ExecuteRejectsWrongArgumentCount) { diff --git a/packages/react-native-executorch/src/core/model.ts b/packages/react-native-executorch/src/core/model.ts index da42f818eb..a24b4d0ffd 100644 --- a/packages/react-native-executorch/src/core/model.ts +++ b/packages/react-native-executorch/src/core/model.ts @@ -12,6 +12,35 @@ import type { ModelSpec, ConcreteDim } from './schema'; declare const modelBrand: unique symbol; +/** + * Custom random-access byte loader for an ExecuTorch `.pte` program or `.ptd` + * external weight archive. + * + * The runtime requests only the byte ranges it needs and releases each segment + * buffer once the backend has unpacked it, enabling chunked or on-demand + * loading without materializing the entire file in memory at once. + * @category Core / Types + */ +export type ModelDataLoader = { + /** Total byte length of the underlying data source. */ + size(): number; + /** + * Reads `size` bytes starting at `offset`. + * @param offset Byte offset from the start of the data source. + * @param size Number of bytes to read. + * @returns An `ArrayBuffer` of `size` bytes. + */ + load(offset: number, size: number): ArrayBuffer; +}; + +/** + * A local filesystem path, an in-memory `ArrayBuffer`, or a custom + * {@link ModelDataLoader} supplying an ExecuTorch `.pte` program or `.ptd` + * external weight archive. + * @category Core / Types + */ +export type ModelSource = string | ArrayBuffer | ModelDataLoader; + /** * A value that can be passed as an input to a model's `execute` method. * @category Core / Types @@ -36,7 +65,11 @@ export type ModelOutput = Tensor | number | boolean | string | null; * @category Core / Types */ export type Model = { - /** The local filesystem path of the `.pte` model file. */ + /** + * The local filesystem path of the `.pte` model file, `''` when the + * model was loaded from an `ArrayBuffer`, or `''` when loaded from a + * {@link ModelDataLoader}. + */ readonly path: string; /** The exported schema of this model. */ readonly schema: ModelSpec; @@ -90,23 +123,33 @@ export type LoadModelOptions = { * guaranteeing instantaneous first-inference latency. * * Set to `false` to lazily load and compile methods on their first execution. + * Models loaded from a {@link ModelDataLoader} must keep `eagerLoadMethods` + * set to `true` when executing methods. * @default true */ readonly eagerLoadMethods?: boolean; + /** + * Optional external `.ptd` weight archive path, `ArrayBuffer`, or + * {@link ModelDataLoader} for models exported with separated weights. + */ + readonly dataMapPath?: ModelSource; }; /** - * Loads and compiles an ExecuTorch `.pte` model from the local filesystem. + * Loads and compiles an ExecuTorch `.pte` model from a local filesystem path, + * an in-memory `ArrayBuffer`, or a custom {@link ModelDataLoader}. * * The model is loaded synchronously into native memory. Prefer calling this * inside a worklet runtime thread (via {@link wrapAsync}) to avoid blocking the * JS thread during compilation. * @category Core / Functions - * @param modelPath The absolute local path to the `.pte` model file. + * @param modelSource The absolute local path, `ArrayBuffer`, or + * {@link ModelDataLoader} supplying the `.pte` model program. * @param options Optional loading configuration. See {@link LoadModelOptions}. * @returns The compiled {@link Model} instance, ready for execution. - * @throws {RnExecuTorchError} Thrown with code `LOAD_FAILED` if the model file - * cannot be opened, has an invalid format, or fails native initialization. + * @throws {RnExecuTorchError} Thrown with code `INVALID_ARGUMENT` if the source + * or options are invalid, or `LOAD_FAILED` if the model cannot be opened, has + * an invalid format, or fails native initialization. * @see {@link wrapAsync} * @example * ```typescript @@ -123,7 +166,7 @@ export type LoadModelOptions = { * } * ``` */ -export function loadModel(modelPath: string, options?: LoadModelOptions): Model { +export function loadModel(modelSource: ModelSource, options?: LoadModelOptions): Model { 'worklet'; - return rnexecutorchJsi.loadModel(modelPath, options) as Model; + return rnexecutorchJsi.loadModel(modelSource, options) as Model; } diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index c430e54bbb..d82540f758 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -10,7 +10,7 @@ import { type WorkletRuntime } from 'react-native-worklets'; import type { Tensor } from '../../core/tensor'; import { wrapAsync } from '../../core/runtime'; -import { loadModel } from '../../core/model'; +import { loadModel, type LoadModelOptions, type ModelSource } from '../../core/model'; import { RnExecuTorchError } from '../../core/error'; import { createResourceScope } from '../../core/lifetime'; import { validateSpec } from '../../core/schema'; @@ -170,23 +170,27 @@ export type LLMRunner = { * Loads the model and its tokenizer, validates the model's exported methods * against the supported runner specs, and returns the matching runner. * @category LLM / Functions - * @param modelPath Path to the local `.pte` model file. + * @param modelPath Local `.pte` model file path, `ArrayBuffer`, or + * {@link ModelDataLoader}. * @param tokenizerPath Path to the local tokenizer file (e.g. `tokenizer.json`). * @param modalities List of supported input non-text modalities (e.g. * `['image']`). When omitted, defaults to text-only. * @param runtime Optional worklet runtime thread on which to load the model and * tokenizer. + * @param options Optional model-loading configuration, including an external + * `.ptd` weight archive. See {@link LoadModelOptions}. * @returns A promise resolving to the {@link LLMRunner} instance. */ export async function createLLMRunner( - modelPath: string, + modelPath: ModelSource, tokenizerPath: string, modalities?: readonly Modality[], - runtime?: WorkletRuntime + runtime?: WorkletRuntime, + options?: LoadModelOptions ): Promise { const scope = createResourceScope(); try { - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath, options)); const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); const { variant } = validateSpec(model.schema, { diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index ebd7293a40..e7831f2f99 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -7,6 +7,7 @@ import { scheduleOnRN, type WorkletRuntime } from 'react-native-worklets'; import RNBlobUtil from 'react-native-blob-util'; import { wrapAsync } from '../../../core/runtime'; +import type { ModelSource } from '../../../core/model'; import { RnExecuTorchError } from '../../../core/error'; import { createResourceScope } from '../../../core/lifetime'; import { @@ -30,10 +31,14 @@ import type { ToolDefinition, ToolParser } from '../utils/toolCalling'; /** * Model configuration required to instantiate an LLM chat session. * @category LLM / Types + * @typeParam TSource Model source type (`string` path/URL by default, or + * {@link ModelSource} when loading from an `ArrayBuffer`). */ -export type LLMModel = { - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; +export type LLMModel = { + /** Local path, remote URL, `ArrayBuffer`, or {@link ModelDataLoader} of the `.pte` model. */ + readonly modelPath: TSource; + /** Optional local path, remote URL, `ArrayBuffer`, or {@link ModelDataLoader} of an external `.ptd` weight archive. */ + readonly dataMapPath?: ModelSource; /** Local path or remote URL of the `tokenizer.json` file. */ readonly tokenizerPath: string; /** Local path or remote URL of the `tokenizer_config.json` file. */ @@ -176,7 +181,7 @@ const DEFAULT_MAX_TURNS = 5; * @returns A promise resolving to the instantiated {@link LLMChatSession} session. */ export async function createLLMChatSession( - config: LLMModel, + config: LLMModel, options: LLMChatSessionOptions = {}, runtime?: WorkletRuntime ): Promise { @@ -192,8 +197,14 @@ export async function createLLMChatSession( resetOnTurn = false, } = options; - const { modelPath, tokenizerPath, tokenizerConfigPath, modalities, preprocessorConfig } = - config; + const { + modelPath, + dataMapPath, + tokenizerPath, + tokenizerConfigPath, + modalities, + preprocessorConfig, + } = config; const { tools, parseToolCalls, maxToolTurns = DEFAULT_MAX_TURNS } = toolOpts ?? {}; // Read and parse tokenizer_config.json @@ -214,7 +225,15 @@ export async function createLLMChatSession( // Prepare runner // prettier-ignore - const runner = scope.track(await createLLMRunner(modelPath, tokenizerPath, modalities, runtime)); + const runner = scope.track( + await createLLMRunner( + modelPath, + tokenizerPath, + modalities, + runtime, + dataMapPath ? { dataMapPath } : undefined + ) + ); const prefill = wrapAsync(runner.prefill, runtime); const history: ChatMessage[] = []; diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmTranslation.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmTranslation.ts index 294a8cdeda..4d98ee2cb4 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmTranslation.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmTranslation.ts @@ -6,6 +6,7 @@ import { scheduleOnRN, type WorkletRuntime } from 'react-native-worklets'; import RNBlobUtil from 'react-native-blob-util'; import { wrapAsync } from '../../../core/runtime'; +import type { ModelSource } from '../../../core/model'; import { RnExecuTorchError } from '../../../core/error'; import { createResourceScope } from '../../../core/lifetime'; import type { TranslationLanguages, Translator } from '../../nlp/utils/translationUtils'; @@ -38,7 +39,7 @@ export type LLMTranslationPrompt = { * translates between any two of its supported languages. * @category NLP / Types */ -export type LLMTranslatorModel = LLMModel & { +export type LLMTranslatorModel = LLMModel & { /** Identifies the pipeline this config belongs to. */ readonly name: 'llm'; /** @@ -98,14 +99,14 @@ const GENERATION_CONFIG: LLMGenerationConfig = { temperature: 0 }; * match a text LLM. */ export async function createLLMTranslator( - config: LLMTranslatorModel, + config: LLMTranslatorModel, runtime?: WorkletRuntime ): Promise { const scope = createResourceScope(); const dispose = scope.dispose; try { - const { modelPath, tokenizerPath, tokenizerConfigPath } = config; + const { modelPath, dataMapPath, tokenizerPath, tokenizerConfigPath } = config; const tokenizerConfigStr = await RNBlobUtil.fs.readFile(tokenizerConfigPath, 'utf8'); const { chatTemplate, bosToken, eosToken, stopTokens } = parseTokenizerConfig( JSON.parse(tokenizerConfigStr) @@ -114,7 +115,15 @@ export async function createLLMTranslator( createChatPreprocessor({ chatTemplate, bosToken, eosToken }) ); // prettier-ignore - const runner = scope.track(await createLLMRunner(modelPath, tokenizerPath, undefined, runtime)); + const runner = scope.track( + await createLLMRunner( + modelPath, + tokenizerPath, + undefined, + runtime, + dataMapPath ? { dataMapPath } : undefined + ) + ); const generate = wrapAsync(generateWorklet, runtime); let busy = false; diff --git a/packages/react-native-executorch/src/fetcher/fetcher.ts b/packages/react-native-executorch/src/fetcher/fetcher.ts index 13908bd879..52674aa6cf 100644 --- a/packages/react-native-executorch/src/fetcher/fetcher.ts +++ b/packages/react-native-executorch/src/fetcher/fetcher.ts @@ -690,12 +690,46 @@ async function downloadUrlViaStream( return dest; } +function isModelDataLoader(value: unknown): value is object { + return ( + typeof value === 'object' && + value !== null && + typeof (value as { size?: unknown }).size === 'function' && + typeof (value as { load?: unknown }).load === 'function' + ); +} + +const objectIds = new WeakMap(); +let nextObjectId = 1; + +/** + * Serializes a hook config while preserving identity for non-JSON model + * sources (`ArrayBuffer` and `ModelDataLoader` instances). + * @internal + * @param config Config value to serialize. + * @returns Stable string key for `useMemo`. + */ +export function serializeHookConfig(config: unknown): string | undefined { + return JSON.stringify(config, (_key, value) => { + if (value instanceof ArrayBuffer || isModelDataLoader(value)) { + let id = objectIds.get(value); + if (id === undefined) { + id = nextObjectId++; + objectIds.set(value, id); + } + return `__ModelSource_${id}__`; + } + return value; + }); +} + // Plain data containers we recurse into. Anything else (numbers, functions, -// typed arrays, class instances) is left alone. +// typed arrays, class instances, ModelDataLoader objects) is left alone. function isPlainObject(value: unknown): value is Record { if (typeof value !== 'object' || value === null) return false; const proto = Object.getPrototypeOf(value); - return proto === Object.prototype || proto === null; + if (proto !== Object.prototype && proto !== null) return false; + return !isModelDataLoader(value); } /** diff --git a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts index 1d7b8500e3..68b522d6d5 100644 --- a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts +++ b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts @@ -1,5 +1,6 @@ import { useModel } from './useModel'; import { useResourceDownload, type ResourceOptions } from './useResourceDownload'; +import type { ModelSource } from '../core/model'; import { createLLMChatSession, type LLMModel, @@ -23,8 +24,8 @@ import { * combined with loading state and download progress. * @see {@link LLMChatSession} */ -export function useLLMChatSession( - config: LLMModel, +export function useLLMChatSession( + config: LLMModel, options?: LLMChatSessionOptions & ResourceOptions ) { const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); diff --git a/packages/react-native-executorch/src/hooks/useModel.ts b/packages/react-native-executorch/src/hooks/useModel.ts index 98748f7807..1819537cb4 100644 --- a/packages/react-native-executorch/src/hooks/useModel.ts +++ b/packages/react-native-executorch/src/hooks/useModel.ts @@ -1,4 +1,5 @@ import { useEffect, useMemo, useState } from 'react'; +import { serializeHookConfig } from '../fetcher/fetcher'; /** * React hook to instantiate and manage a model or task instance with automatic @@ -29,7 +30,7 @@ export function useModel void }>( // Configs are plain JSON data, so serializing is a sound structural identity // and keeps an inline `config` object from rebuilding the model every render. - const configKey = useMemo(() => JSON.stringify(config), [config]); + const configKey = useMemo(() => serializeHookConfig(config), [config]); useEffect(() => { if (config === undefined) { diff --git a/packages/react-native-executorch/src/hooks/useResourceDownload.ts b/packages/react-native-executorch/src/hooks/useResourceDownload.ts index 33b584b0bc..7207c08ae8 100644 --- a/packages/react-native-executorch/src/hooks/useResourceDownload.ts +++ b/packages/react-native-executorch/src/hooks/useResourceDownload.ts @@ -1,5 +1,5 @@ import { useState, useEffect, useMemo } from 'react'; -import { download } from '../fetcher/fetcher'; +import { download, serializeHookConfig } from '../fetcher/fetcher'; import { isRnExecuTorchError } from '../core/error'; import type { ResourceOptions } from '../utils'; @@ -43,7 +43,7 @@ export function useResourceDownload(config: T | undefined, options?: Resource // Configs are plain JSON data, so serializing is a sound structural identity // and keeps an inline `config` object from re-running the effect every render. - const configKey = useMemo(() => JSON.stringify(config), [config]); + const configKey = useMemo(() => serializeHookConfig(config), [config]); useEffect(() => { setResource(undefined);