diff --git a/packages/fs/README.md b/packages/fs/README.md index 5fc5d28..5021d4f 100644 --- a/packages/fs/README.md +++ b/packages/fs/README.md @@ -139,7 +139,7 @@ publisher.writeFile('/main.js', 'export const x = 1') // text publisher.writeFile('/image.png', arrayBuffer) // binary (each target gets its own copy) publisher.unlink('/old-file.js') publisher.mkdir('/new-dir') -publisher.initFiles({ '/main.js': 'code' }) // bulk init +publisher.initFiles({ '/main.js': 'code' }, { '/logo.png': arrayBuffer }) // bulk init (each target gets its own copy of the binary files) ``` `writeFile()` and `unlink()` accept an operation ID, which is set on the message. A Worker that applies such a message can acknowledge it with a `V_FS_ACK` message (`{ type: 'V_FS_ACK', id, error? }`), so that the main thread knows when the change is applied. `@vrowzer/fs` does not send acknowledgements by itself: the Worker that handles the messages decides when a change counts as applied. diff --git a/packages/fs/docs/watcher/interfaces/FSInitMessage.md b/packages/fs/docs/watcher/interfaces/FSInitMessage.md index 6efdd2e..fad73e0 100644 --- a/packages/fs/docs/watcher/interfaces/FSInitMessage.md +++ b/packages/fs/docs/watcher/interfaces/FSInitMessage.md @@ -4,7 +4,8 @@ Main Thread -> Worker: Initialize files in bulk. Used during setup to populate the virtual filesystem. Text files are in `files`, binary files are in `binaryFiles`. -Binary ArrayBuffers are transferred via postMessage's transfer list. +The publisher transfers a copy of each binary ArrayBuffer to each target via postMessage's +transfer list, so the caller's ArrayBuffers stay usable. ## Signature @@ -16,6 +17,6 @@ export interface FSInitMessage | Name | Type | Description | | --- | --- | --- | -| `binaryFiles` _(optional)_ | `Record` | Binary files: path -> ArrayBuffer content (transferred) | +| `binaryFiles` _(optional)_ | `Record` | Binary files: path -> ArrayBuffer content (a copy transferred via postMessage's transfer list) | | `files` _(optional)_ | `Record` | Text files: path -> UTF-8 string content | | `type` | `"V_FS_INIT"` | | diff --git a/packages/fs/docs/watcher/interfaces/FileSystemPublisher.md b/packages/fs/docs/watcher/interfaces/FileSystemPublisher.md index fc2784f..9b5f0f8 100644 --- a/packages/fs/docs/watcher/interfaces/FileSystemPublisher.md +++ b/packages/fs/docs/watcher/interfaces/FileSystemPublisher.md @@ -48,7 +48,7 @@ Initialize files in bulk. | Name | Type | Description | | --- | --- | --- | | `files` | `Record` | Text files: path -> UTF-8 string content _(optional)_ | -| `binaryFiles` | `Record` | Binary files: path -> ArrayBuffer content (transferred) _(optional)_ | +| `binaryFiles` | `Record` | Binary files: path -> ArrayBuffer content. Each target receives its own copy via postMessage's transfer list, so the caller's ArrayBuffers stay usable. _(optional)_ | #### Returns diff --git a/packages/fs/src/watcher/protocol.ts b/packages/fs/src/watcher/protocol.ts index c2792c6..1977005 100644 --- a/packages/fs/src/watcher/protocol.ts +++ b/packages/fs/src/watcher/protocol.ts @@ -87,7 +87,8 @@ export interface FSMkdirMessage { * Used during setup to populate the virtual filesystem. * * Text files are in `files`, binary files are in `binaryFiles`. - * Binary ArrayBuffers are transferred via postMessage's transfer list. + * The publisher transfers a copy of each binary ArrayBuffer to each target via postMessage's + * transfer list, so the caller's ArrayBuffers stay usable. */ export interface FSInitMessage { type: 'V_FS_INIT' @@ -96,7 +97,7 @@ export interface FSInitMessage { */ files?: Record /** - * Binary files: path -> ArrayBuffer content (transferred) + * Binary files: path -> ArrayBuffer content (a copy transferred via postMessage's transfer list) */ binaryFiles?: Record } diff --git a/packages/fs/src/watcher/publisher.test.ts b/packages/fs/src/watcher/publisher.test.ts index 994d4d1..d49b65d 100644 --- a/packages/fs/src/watcher/publisher.test.ts +++ b/packages/fs/src/watcher/publisher.test.ts @@ -254,17 +254,71 @@ describe('FileSystemPublisher', () => { expect(target.calls[0]!.message.binaryFiles).toEqual(binaryFiles) }) - test('binaryFiles ArrayBuffers are included in transfer list', () => { + test("transfers a copy of each binary file, not the caller's buffer", () => { const target = createMockTarget() const publisher = createFileSystemPublisher([target]) - const buf1 = new ArrayBuffer(4) - const buf2 = new ArrayBuffer(8) - const binaryFiles = { '/a.wasm': buf1, '/b.wasm': buf2 } + const buf1 = new Uint8Array([1, 2]).buffer + const buf2 = new Uint8Array([3, 4, 5]).buffer - publisher.initFiles(undefined, binaryFiles) + publisher.initFiles(undefined, { '/a.wasm': buf1, '/b.wasm': buf2 }) + + const { message, transfer } = target.calls[0]! + expect(transfer).toHaveLength(2) + expect(transfer[0]).toBe(message.binaryFiles['/a.wasm']) + expect(transfer[1]).toBe(message.binaryFiles['/b.wasm']) + expect(transfer).not.toContain(buf1) + expect(transfer).not.toContain(buf2) + expect([...new Uint8Array(message.binaryFiles['/a.wasm'])]).toEqual([1, 2]) + expect([...new Uint8Array(message.binaryFiles['/b.wasm'])]).toEqual([3, 4, 5]) + }) + + test('binaryFiles with multiple targets: every target gets its own copies', () => { + const target1 = createMockTarget() + const target2 = createMockTarget() + const publisher = createFileSystemPublisher([target1, target2]) + const buffer = new Uint8Array([1, 2, 3, 4]).buffer + + publisher.initFiles(undefined, { '/app.wasm': buffer }) + + // Neither target gets the caller's buffer, and the copies are not shared + const copy1 = target1.calls[0]!.message.binaryFiles['/app.wasm'] + const copy2 = target2.calls[0]!.message.binaryFiles['/app.wasm'] + expect(copy1).not.toBe(buffer) + expect(copy2).not.toBe(buffer) + expect(copy1).not.toBe(copy2) + expect(target1.calls[0]!.transfer[0]).toBe(copy1) + expect(target2.calls[0]!.transfer[0]).toBe(copy2) + expect([...new Uint8Array(copy1)]).toEqual([1, 2, 3, 4]) + expect([...new Uint8Array(copy2)]).toEqual([1, 2, 3, 4]) + }) + + test('keeps the caller ArrayBuffers usable and delivers the files to every target', async () => { + const first = createPortTarget() + const second = createPortTarget() + const publisher = createFileSystemPublisher([first.target, second.target]) + const buffer = new Uint8Array([1, 2, 3, 4]).buffer + + publisher.initFiles({ '/main.js': 'code' }, { '/app.wasm': buffer }) + + expect(buffer.byteLength).toBe(4) + expect([...new Uint8Array(buffer)]).toEqual([1, 2, 3, 4]) + for (const message of await Promise.all([first.received, second.received])) { + expect(message).toMatchObject({ type: 'V_FS_INIT', files: { '/main.js': 'code' } }) + expect([...new Uint8Array(message.binaryFiles['/app.wasm'])]).toEqual([1, 2, 3, 4]) + } + }) + + test('accepts the same ArrayBuffer for several paths', async () => { + const { target, received } = createPortTarget() + const publisher = createFileSystemPublisher([target]) + const buffer = new Uint8Array([1, 2, 3]).buffer + + publisher.initFiles(undefined, { '/a.bin': buffer, '/b.bin': buffer }) - expect(target.calls[0]!.transfer).toContain(buf1) - expect(target.calls[0]!.transfer).toContain(buf2) + expect([...new Uint8Array(buffer)]).toEqual([1, 2, 3]) + const message = await received + expect([...new Uint8Array(message.binaryFiles['/a.bin'])]).toEqual([1, 2, 3]) + expect([...new Uint8Array(message.binaryFiles['/b.bin'])]).toEqual([1, 2, 3]) }) }) }) diff --git a/packages/fs/src/watcher/publisher.ts b/packages/fs/src/watcher/publisher.ts index 0e417ad..e9b0683 100644 --- a/packages/fs/src/watcher/publisher.ts +++ b/packages/fs/src/watcher/publisher.ts @@ -74,7 +74,8 @@ export interface FileSystemPublisher { * Initialize files in bulk. * * @param files - Text files: path -> UTF-8 string content - * @param binaryFiles - Binary files: path -> ArrayBuffer content (transferred) + * @param binaryFiles - Binary files: path -> ArrayBuffer content. Each target receives its own copy + * via postMessage's transfer list, so the caller's ArrayBuffers stay usable. */ initFiles(files?: Record, binaryFiles?: Record): void /** @@ -102,9 +103,9 @@ export function createFileSystemPublisher( ): Readonly { const _targets = new Set(targets) - function broadcast(message: FileSystemSyncMessage, transfer: Transferable[] = []) { + function broadcast(message: FileSystemSyncMessage) { for (const target of _targets) { - target.postMessage(message, transfer) + target.postMessage(message, []) } } @@ -143,15 +144,24 @@ export function createFileSystemPublisher( }, initFiles(files, binaryFiles) { - const transfer: Transferable[] = binaryFiles ? Object.values(binaryFiles) : [] - const msg: FSInitMessage = { type: 'V_FS_INIT' } - if (files) { - msg.files = files - } - if (binaryFiles) { - msg.binaryFiles = binaryFiles + // Each target gets its own copies of the binary files in the transfer list. + // The caller's buffers are never transferred, so they stay usable. + for (const target of _targets) { + const msg: FSInitMessage = { type: 'V_FS_INIT' } + if (files) { + msg.files = files + } + const transfer: ArrayBuffer[] = [] + if (binaryFiles) { + msg.binaryFiles = {} + for (const [path, content] of Object.entries(binaryFiles)) { + const buf = content.slice(0) + msg.binaryFiles[path] = buf + transfer.push(buf) + } + } + target.postMessage(msg, transfer) } - broadcast(msg, transfer) }, addTarget(target) {