From 5fdf296552dea5bc5989bf48f0d7f52b1471261b Mon Sep 17 00:00:00 2001 From: Rasaboun <40967731+Rasaboun@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:57:38 +0100 Subject: [PATCH] fix(cv): frame the SDXS prompt with CLIP's start and end tokens Tokenizer.encode adds no special tokens and the exported encode method feeds the ids straight into CLIP's token embedding, so the text encoder never saw <|startoftext|>, and prompts longer than 77 tokens lost <|endoftext|>. Build the ids like CLIPTokenizer: <|startoftext|>, at most 75 prompt ids, then <|endoftext|> padding up to 77. --- .../__tests__/tasks/remainingTasks.test.ts | 29 +++++++++++++++++++ .../extensions/cv/tasks/sdxsTextToImage.ts | 14 ++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts b/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts index 96f8008a2c..c4b4eca1fb 100644 --- a/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts @@ -508,6 +508,35 @@ describe('createSdxsTextToImage', () => { expect(fakeJsi.liveTokenizers()).toEqual([]); }); + /** Generates once and returns the token ids passed to `encode`. */ + const encodedIds = async (vocabulary: string[], prompt: string) => { + const encoded: number[][] = []; + fakeJsi.registerModel(MODEL_PATH, { + schema: SDXS_SCHEMA, + execute: (methodName, inputs) => { + if (methodName !== 'encode') return; + const ids = inputs[0] as FakeTensor; + encoded.push(Array.from({ length: ids.numel }, (_, i) => ids.getElement(i))); + }, + }); + fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: vocabulary }); + const tti = tracked(await createSdxsTextToImage(config)); + await tti.generate(prompt, 0); + return encoded; + }; + + it('frames the prompt with <|startoftext|> and pads with <|endoftext|>', async () => { + const encoded = await encodedIds(['a', 'cat'], 'a cat'); + + expect(encoded).toEqual([[49406, 0, 1, ...new Array(74).fill(49407)]]); + }); + + it('keeps <|endoftext|> after a prompt longer than the context', async () => { + const encoded = await encodedIds(['word'], new Array(100).fill('word').join(' ')); + + expect(encoded).toEqual([[49406, ...new Array(75).fill(0), 49407]]); + }); + it('rejects a model that is missing the denoise method', async () => { fakeJsi.registerModel(MODEL_PATH, { schema: exported({ diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts index 5dcc136d5a..d2a035a0ac 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts @@ -23,7 +23,8 @@ import { toChannelsLast, normalize, cvtColor } from '../ops/image'; // is no unconditional branch and no denoising loop. const CLIP_MAX_TOKENS = 77; const CLIP_HIDDEN_SIZE = 768; -const CLIP_PAD_TOKEN_ID = 49407; +const CLIP_BOS_TOKEN_ID = 49406; // <|startoftext|> +const CLIP_EOS_TOKEN_ID = 49407; // <|endoftext|>, also the padding token const IMAGE_SIZE = 512; const LATENT_CHANNELS = 4; @@ -147,10 +148,15 @@ export async function createSdxsTextToImage( const generateWorklet = (prompt: string, seed?: number): ImageBuffer => { 'worklet'; + // `encode` adds no special tokens and the exported `encode` method reads the + // ids as given, so the prompt is framed here like CLIPTokenizer does: + // <|startoftext|>, at most 75 ids, then <|endoftext|> up to 77 tokens. const ids = tokenizer.encode(prompt); - const tokens = new BigInt64Array(CLIP_MAX_TOKENS); - for (let i = 0; i < CLIP_MAX_TOKENS; i++) { - tokens[i] = BigInt(i < ids.length ? ids[i]! : CLIP_PAD_TOKEN_ID); + const numText = Math.min(ids.length, CLIP_MAX_TOKENS - 2); + const tokens = new BigInt64Array(CLIP_MAX_TOKENS).fill(BigInt(CLIP_EOS_TOKEN_ID)); + tokens[0] = BigInt(CLIP_BOS_TOKEN_ID); + for (let i = 0; i < numText; i++) { + tokens[i + 1] = BigInt(ids[i]!); } tTokens.setData(tokens); model.execute('encode', [tTokens], [tEmbeddings]);