diff --git a/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts b/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts index 96f8008a2c..c4b4eca1fb 100644 --- a/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts @@ -508,6 +508,35 @@ describe('createSdxsTextToImage', () => { expect(fakeJsi.liveTokenizers()).toEqual([]); }); + /** Generates once and returns the token ids passed to `encode`. */ + const encodedIds = async (vocabulary: string[], prompt: string) => { + const encoded: number[][] = []; + fakeJsi.registerModel(MODEL_PATH, { + schema: SDXS_SCHEMA, + execute: (methodName, inputs) => { + if (methodName !== 'encode') return; + const ids = inputs[0] as FakeTensor; + encoded.push(Array.from({ length: ids.numel }, (_, i) => ids.getElement(i))); + }, + }); + fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: vocabulary }); + const tti = tracked(await createSdxsTextToImage(config)); + await tti.generate(prompt, 0); + return encoded; + }; + + it('frames the prompt with <|startoftext|> and pads with <|endoftext|>', async () => { + const encoded = await encodedIds(['a', 'cat'], 'a cat'); + + expect(encoded).toEqual([[49406, 0, 1, ...new Array(74).fill(49407)]]); + }); + + it('keeps <|endoftext|> after a prompt longer than the context', async () => { + const encoded = await encodedIds(['word'], new Array(100).fill('word').join(' ')); + + expect(encoded).toEqual([[49406, ...new Array(75).fill(0), 49407]]); + }); + it('rejects a model that is missing the denoise method', async () => { fakeJsi.registerModel(MODEL_PATH, { schema: exported({ diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts index 5dcc136d5a..d2a035a0ac 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts @@ -23,7 +23,8 @@ import { toChannelsLast, normalize, cvtColor } from '../ops/image'; // is no unconditional branch and no denoising loop. const CLIP_MAX_TOKENS = 77; const CLIP_HIDDEN_SIZE = 768; -const CLIP_PAD_TOKEN_ID = 49407; +const CLIP_BOS_TOKEN_ID = 49406; // <|startoftext|> +const CLIP_EOS_TOKEN_ID = 49407; // <|endoftext|>, also the padding token const IMAGE_SIZE = 512; const LATENT_CHANNELS = 4; @@ -147,10 +148,15 @@ export async function createSdxsTextToImage( const generateWorklet = (prompt: string, seed?: number): ImageBuffer => { 'worklet'; + // `encode` adds no special tokens and the exported `encode` method reads the + // ids as given, so the prompt is framed here like CLIPTokenizer does: + // <|startoftext|>, at most 75 ids, then <|endoftext|> up to 77 tokens. const ids = tokenizer.encode(prompt); - const tokens = new BigInt64Array(CLIP_MAX_TOKENS); - for (let i = 0; i < CLIP_MAX_TOKENS; i++) { - tokens[i] = BigInt(i < ids.length ? ids[i]! : CLIP_PAD_TOKEN_ID); + const numText = Math.min(ids.length, CLIP_MAX_TOKENS - 2); + const tokens = new BigInt64Array(CLIP_MAX_TOKENS).fill(BigInt(CLIP_EOS_TOKEN_ID)); + tokens[0] = BigInt(CLIP_BOS_TOKEN_ID); + for (let i = 0; i < numText; i++) { + tokens[i + 1] = BigInt(ids[i]!); } tTokens.setData(tokens); model.execute('encode', [tTokens], [tEmbeddings]);