Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,35 @@ describe('createSdxsTextToImage', () => {
expect(fakeJsi.liveTokenizers()).toEqual([]);
});

/** Generates once and returns the token ids passed to `encode`. */
const encodedIds = async (vocabulary: string[], prompt: string) => {
const encoded: number[][] = [];
fakeJsi.registerModel(MODEL_PATH, {
schema: SDXS_SCHEMA,
execute: (methodName, inputs) => {
if (methodName !== 'encode') return;
const ids = inputs[0] as FakeTensor;
encoded.push(Array.from({ length: ids.numel }, (_, i) => ids.getElement(i)));
},
});
fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: vocabulary });
const tti = tracked(await createSdxsTextToImage(config));
await tti.generate(prompt, 0);
return encoded;
};

it('frames the prompt with <|startoftext|> and pads with <|endoftext|>', async () => {
const encoded = await encodedIds(['a', 'cat'], 'a cat');

expect(encoded).toEqual([[49406, 0, 1, ...new Array<number>(74).fill(49407)]]);
});

it('keeps <|endoftext|> after a prompt longer than the context', async () => {
const encoded = await encodedIds(['word'], new Array(100).fill('word').join(' '));

expect(encoded).toEqual([[49406, ...new Array<number>(75).fill(0), 49407]]);
});

it('rejects a model that is missing the denoise method', async () => {
fakeJsi.registerModel(MODEL_PATH, {
schema: exported({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ import { toChannelsLast, normalize, cvtColor } from '../ops/image';
// is no unconditional branch and no denoising loop.
const CLIP_MAX_TOKENS = 77;
const CLIP_HIDDEN_SIZE = 768;
const CLIP_PAD_TOKEN_ID = 49407;
const CLIP_BOS_TOKEN_ID = 49406; // <|startoftext|>
const CLIP_EOS_TOKEN_ID = 49407; // <|endoftext|>, also the padding token

const IMAGE_SIZE = 512;
const LATENT_CHANNELS = 4;
Expand Down Expand Up @@ -147,10 +148,15 @@ export async function createSdxsTextToImage(
const generateWorklet = (prompt: string, seed?: number): ImageBuffer => {
'worklet';

// `encode` adds no special tokens and the exported `encode` method reads the
// ids as given, so the prompt is framed here like CLIPTokenizer does:
// <|startoftext|>, at most 75 ids, then <|endoftext|> up to 77 tokens.
const ids = tokenizer.encode(prompt);
const tokens = new BigInt64Array(CLIP_MAX_TOKENS);
for (let i = 0; i < CLIP_MAX_TOKENS; i++) {
tokens[i] = BigInt(i < ids.length ? ids[i]! : CLIP_PAD_TOKEN_ID);
const numText = Math.min(ids.length, CLIP_MAX_TOKENS - 2);
const tokens = new BigInt64Array(CLIP_MAX_TOKENS).fill(BigInt(CLIP_EOS_TOKEN_ID));
tokens[0] = BigInt(CLIP_BOS_TOKEN_ID);
for (let i = 0; i < numText; i++) {
tokens[i + 1] = BigInt(ids[i]!);
}
tTokens.setData(tokens);
model.execute('encode', [tTokens], [tEmbeddings]);
Expand Down
Loading