diff --git a/.changeset/lucky-poems-drive.md b/.changeset/lucky-poems-drive.md new file mode 100644 index 000000000..9dac5319c --- /dev/null +++ b/.changeset/lucky-poems-drive.md @@ -0,0 +1,5 @@ +--- +'@blinkk/root-cms': minor +--- + +feat: v2 translations manager with fallback chains diff --git a/docs/root.config.ts b/docs/root.config.ts index 48b3c5357..8fae135cf 100644 --- a/docs/root.config.ts +++ b/docs/root.config.ts @@ -44,14 +44,14 @@ export default defineConfig({ id: 'www', name: 'Root.js', firebaseConfig: { - apiKey: process.env.GAPI_API_KEY, + apiKey: process.env.GAPI_API_KEY!, authDomain: 'rootjs-dev.firebaseapp.com', projectId: 'rootjs-dev', storageBucket: 'rootjs-dev.appspot.com', }, gapi: { - apiKey: process.env.GAPI_API_KEY, - clientId: process.env.GAPI_CLIENT_ID, + apiKey: process.env.GAPI_API_KEY!, + clientId: process.env.GAPI_CLIENT_ID!, }, gci: true, sidebar: { @@ -125,6 +125,7 @@ export default defineConfig({ ], experiments: { taskManager: true, + v2TranslationsManager: true, }, preview: { channel: true, diff --git a/packages/root-cms/core/batch-request.test.ts b/packages/root-cms/core/batch-request.test.ts new file mode 100644 index 000000000..eea5b9b84 --- /dev/null +++ b/packages/root-cms/core/batch-request.test.ts @@ -0,0 +1,190 @@ +/** + * Tests for BatchRequest/BatchResponse translation resolution against the + * Firestore emulator (see translations-manager.test.ts for details on the + * emulator setup). + */ + +import {getApps, initializeApp} from 'firebase-admin/app'; +import {Timestamp, getFirestore} from 'firebase-admin/firestore'; +import {beforeEach, describe, expect, it} from 'vitest'; +import {RootCMSClient} from './client.js'; + +const FIREBASE_PROJECT_ID = 'rootjs-cms-admin-tests'; + +function getTestApp() { + const existing = getApps().find((app) => app.name === 'batch-test'); + if (existing) { + return existing; + } + return initializeApp({projectId: FIREBASE_PROJECT_ID}, 'batch-test'); +} + +let projectCounter = 0; + +function createTestCmsClient(options?: {i18n?: any}): RootCMSClient { + const app = getTestApp(); + const db = getFirestore(app); + const projectId = `batch-test-${Date.now()}-${projectCounter++}`; + const plugin = { + name: 'root-cms', + getConfig: () => ({ + id: projectId, + firebaseConfig: { + apiKey: 'test', + authDomain: 'test', + projectId: FIREBASE_PROJECT_ID, + storageBucket: 'test', + }, + experiments: {v2TranslationsManager: true}, + }), + getFirebaseApp: () => app, + getFirestore: () => db, + }; + const rootConfig: any = { + rootDir: '/test', + i18n: options?.i18n ?? { + locales: ['en', 'en-GB', 'en-CA', 'es'], + fallbacks: {'en-CA': ['en-GB']}, + }, + plugins: [plugin], + }; + return new RootCMSClient(rootConfig); +} + +async function seedDraftDoc(cmsClient: RootCMSClient, docId: string) { + const [collection, slug] = docId.split('/'); + await cmsClient.db + .doc( + `Projects/${cmsClient.projectId}/Collections/${collection}/Drafts/${slug}` + ) + .set({ + id: docId, + collection, + slug, + sys: { + createdAt: Timestamp.now(), + createdBy: 'test', + modifiedAt: Timestamp.now(), + modifiedBy: 'test', + locales: ['en'], + }, + fields: {title: `Title for ${docId}`}, + }); +} + +describe.skipIf(!process.env.FIRESTORE_EMULATOR_HOST)('BatchRequest', () => { + let cmsClient: RootCMSClient; + + beforeEach(() => { + cmsClient = createTestCmsClient(); + }); + + it('fetches explicitly-added translations without translate', async () => { + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('common', { + hello: {es: 'hola', 'en-GB': 'hello (GB)'}, + }); + + const req = cmsClient.createBatchRequest({mode: 'draft'}); + req.addTranslations('common'); + const res = await req.fetch(); + + expect(Object.keys(res.translations)).toEqual(['common']); + expect(res.getTranslations('es')).toEqual({hello: 'hola'}); + }); + + it('resolves locale fallbacks in getTranslations()', async () => { + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('common', { + color: {'en-GB': 'colour'}, + untranslated: {es: 'es only'}, + }); + + const req = cmsClient.createBatchRequest({mode: 'draft'}); + req.addTranslations('common'); + const res = await req.fetch(); + + // en-CA falls back to en-GB (per i18n.fallbacks). + expect(res.getTranslations('en-CA')).toEqual({ + color: 'colour', + untranslated: 'untranslated', + }); + // An explicit fallback chain can also be provided. + expect(res.getTranslations(['en-CA', 'en-GB'])).toEqual({ + color: 'colour', + untranslated: 'untranslated', + }); + expect(res.getTranslations('es')).toEqual({ + color: 'color', + untranslated: 'es only', + }); + }); + + it('auto-collects translations for docs when translate is enabled', async () => { + await seedDraftDoc(cmsClient, 'Pages/index'); + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('Pages/index', {hello: {es: 'hola (doc)'}}); + await tm.saveTranslations('common', { + hello: {es: 'hola (common)'}, + bye: {es: 'adios'}, + }); + + const req = cmsClient.createBatchRequest({mode: 'draft', translate: true}); + req.addDoc('Pages/index'); + req.addTranslations('common'); + const res = await req.fetch(); + + expect(res.docs['Pages/index']).toBeDefined(); + // Generic translations come first, doc-specific translations last. + expect(Object.keys(res.translations)).toEqual(['common', 'Pages/index']); + // Doc-specific translations take precedence. + expect(res.getTranslations('es')).toEqual({ + hello: 'hola (doc)', + bye: 'adios', + }); + }); + + it('auto-collects translations for query results when translate is enabled', async () => { + await seedDraftDoc(cmsClient, 'BlogPosts/foo'); + await seedDraftDoc(cmsClient, 'BlogPosts/bar'); + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('BlogPosts/foo', {'foo title': {es: 'foo es'}}); + await tm.saveTranslations('BlogPosts/bar', {'bar title': {es: 'bar es'}}); + + const req = cmsClient.createBatchRequest({mode: 'draft', translate: true}); + req.addQuery('blogPosts', 'BlogPosts'); + const res = await req.fetch(); + + expect(res.queries.blogPosts).toHaveLength(2); + expect(res.getTranslations('es')).toEqual({ + 'foo title': 'foo es', + 'bar title': 'bar es', + }); + }); + + it('falls back to id queries when no locales are known', async () => { + const client = createTestCmsClient({i18n: {}}); + const tm = client.getTranslationsManager(); + await tm.saveTranslations('common', {hello: {es: 'hola', fr: 'bonjour'}}); + + const req = client.createBatchRequest({mode: 'draft'}); + req.addTranslations('common'); + const res = await req.fetch(); + + expect(Object.keys(res.translations.common).sort()).toEqual(['es', 'fr']); + expect(res.getTranslations(['fr'])).toEqual({hello: 'bonjour'}); + }); + + it('limits fetched locales via options.locales', async () => { + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('common', {hello: {es: 'hola', 'en-GB': 'hi'}}); + + const req = cmsClient.createBatchRequest({mode: 'draft', locales: ['es']}); + req.addTranslations('common'); + const res = await req.fetch(); + + // Only the `es` fallback chain (es -> en) is fetched. + expect(Object.keys(res.translations.common)).toEqual(['es']); + expect(res.getTranslations('es')).toEqual({hello: 'hola'}); + }); +}); diff --git a/packages/root-cms/core/checks-translations.ts b/packages/root-cms/core/checks-translations.ts index 69beb1eb1..330b5925a 100644 --- a/packages/root-cms/core/checks-translations.ts +++ b/packages/root-cms/core/checks-translations.ts @@ -225,11 +225,18 @@ export function translationsCheck( }; } - // Load all translations tagged to this document. + // Load all translations tagged to this document. When the v2 + // translations manager is enabled, read the doc's draft translations + // from the v2 per-locale docs instead (both shapes are maps of + // `{source: ..., [locale]: ...}` values, keyed by hash in v1 and by + // source in v2). const docId = `${collectionId}/${slug}`; - const translationsMap = await cmsClient.loadTranslations({ - tags: [docId], - }); + const translationsMap = cmsClient.isV2TranslationsEnabled() + ? await cmsClient.getTranslationsManager().loadTranslations({ + ids: [docId], + mode: 'draft', + }) + : await cmsClient.loadTranslations({tags: [docId]}); // Build a map from source string to available translations. const sourceToTranslations: Map< diff --git a/packages/root-cms/core/client.ts b/packages/root-cms/core/client.ts index 41ef5d27c..ac0d8f3e2 100644 --- a/packages/root-cms/core/client.ts +++ b/packages/root-cms/core/client.ts @@ -8,11 +8,19 @@ import { Timestamp, WriteBatch, } from 'firebase-admin/firestore'; +import {resolveLocaleFallbacks} from '../shared/locale-fallbacks.js'; import {normalizeSlug} from '../shared/slug.js'; import {isCronDue} from './cron-schedule.js'; import {CMSPlugin} from './plugin.js'; import {Collection} from './schema.js'; -import {TranslationsManager} from './translations-manager.js'; +import { + TranslationsLocaleDoc, + TranslationsLocaleDocWithRef, + TranslationsManager, + buildTranslationsDbPath, + buildTranslationsLocaleDocDbPath, + chunkArray, +} from './translations-manager.js'; import {validateFields} from './validation.js'; import {setValueAtPath} from './values.js'; @@ -684,10 +692,24 @@ export class RootCMSClient { } } + // If the v2 translations manager is enabled, prefetch the draft + // translations locale docs for each doc so that the doc's translations + // are published in the same batch as the doc itself. + let tm: TranslationsManager | null = null; + let translationsByDocId: Record = + {}; + if (this.isV2TranslationsEnabled()) { + tm = this.getTranslationsManager(); + translationsByDocId = await tm.getTranslationsLocaleDocs( + docs.map((doc) => doc.id), + 'draft' + ); + } + // // Each transaction or batch can write a max of 500 ops. // // https://firebase.google.com/docs/firestore/manage-data/transactions let batchCount = 0; - const batch = options?.batch || this.db.batch(); + let batch = options?.batch || this.db.batch(); const versionTags = ['published']; if (options?.releaseId) { versionTags.push(`release:${options.releaseId}`); @@ -765,10 +787,20 @@ export class RootCMSClient { }); batchCount += 1; + // Publish the doc's translations within the same batch so the doc and + // its translations go live atomically. + if (tm) { + const localeDocs = translationsByDocId[id] || []; + batchCount += tm.addPublishTranslationsOps(localeDocs, batch, { + publishedBy, + }); + } + publishedDocs.push(doc); if (batchCount >= 400) { await batch.commit(); + batch = this.db.batch(); batchCount = 0; } } @@ -813,8 +845,29 @@ export class RootCMSClient { return []; } + // If the v2 translations manager is enabled, unpublish each doc's + // translations along with the doc. + let draftTranslationsByDocId: Record< + string, + TranslationsLocaleDocWithRef[] + > = {}; + let publishedTranslationsByDocId: Record< + string, + TranslationsLocaleDocWithRef[] + > = {}; + const v2TranslationsEnabled = this.isV2TranslationsEnabled(); + if (v2TranslationsEnabled) { + const tm = this.getTranslationsManager(); + const docIdsToUnpublish = docs.map((doc) => doc.id); + [draftTranslationsByDocId, publishedTranslationsByDocId] = + await Promise.all([ + tm.getTranslationsLocaleDocs(docIdsToUnpublish, 'draft'), + tm.getTranslationsLocaleDocs(docIdsToUnpublish, 'published'), + ]); + } + let batchCount = 0; - const batch = options?.batch || this.db.batch(); + let batch = options?.batch || this.db.batch(); const unpublishedDocs: any[] = []; for (const doc of docs) { @@ -848,10 +901,27 @@ export class RootCMSClient { batch.delete(publishedRef); batchCount += 1; + // Unpublish the doc's translations: delete the published locale docs + // and clear the published metadata from the draft locale docs. + if (v2TranslationsEnabled) { + for (const localeDoc of publishedTranslationsByDocId[doc.id] || []) { + batch.delete(localeDoc.ref); + batchCount += 1; + } + for (const localeDoc of draftTranslationsByDocId[doc.id] || []) { + batch.update(localeDoc.ref, { + 'sys.publishedAt': FieldValue.delete(), + 'sys.publishedBy': FieldValue.delete(), + }); + batchCount += 1; + } + } + unpublishedDocs.push(doc); if (batchCount >= 400) { await batch.commit(); + batch = this.db.batch(); batchCount = 0; } } @@ -905,10 +975,24 @@ export class RootCMSClient { return []; } + // If the v2 translations manager is enabled, prefetch the draft + // translations locale docs for each doc so that the doc's translations + // are published in the same batch as the doc itself. + let tm: TranslationsManager | null = null; + let translationsByDocId: Record = + {}; + if (this.isV2TranslationsEnabled()) { + tm = this.getTranslationsManager(); + translationsByDocId = await tm.getTranslationsLocaleDocs( + docs.map((doc) => doc.id), + 'draft' + ); + } + // Each transaction or batch can write a max of 500 ops. // https://firebase.google.com/docs/firestore/manage-data/transactions let batchCount = 0; - const batch = this.db.batch(); + let batch = this.db.batch(); const versionTags = ['published']; const publishedDocs: any[] = []; for (const doc of docs) { @@ -985,10 +1069,20 @@ export class RootCMSClient { }); batchCount += 1; + // Publish the doc's translations within the same batch so the doc and + // its translations go live atomically. + if (tm) { + const localeDocs = translationsByDocId[id] || []; + batchCount += tm.addPublishTranslationsOps(localeDocs, batch, { + publishedBy: scheduledBy || 'root-cms-client', + }); + } + publishedDocs.push(doc); if (batchCount >= 400) { await batch.commit(); + batch = this.db.batch(); batchCount = 0; continue; } @@ -1169,8 +1263,7 @@ export class RootCMSClient { * replace the v1 translations system. */ getTranslationsManager(): TranslationsManager { - const cmsPluginOptions = this.cmsPlugin.getConfig(); - if (cmsPluginOptions.experiments?.v2TranslationsManager) { + if (!this.isV2TranslationsEnabled()) { throw new Error( '`v2TranslationsManager` is not enabled. update root.config.ts and add: `{experiments: {v2TranslationsManager: true}}`' ); @@ -1178,6 +1271,15 @@ export class RootCMSClient { return new TranslationsManager(this); } + /** + * Returns true if the v2 `TranslationsManager` is enabled via the + * `experiments.v2TranslationsManager` plugin config flag. + */ + isV2TranslationsEnabled(): boolean { + const cmsPluginOptions = this.cmsPlugin.getConfig(); + return Boolean(cmsPluginOptions.experiments?.v2TranslationsManager); + } + /** * Loads translations saved in the translations collection, optionally * filtered by tag. @@ -1287,33 +1389,6 @@ export class RootCMSClient { return translationsForLocale(translationsMap, locale); } - /** - * Firestore path for a translations file. - */ - dbTranslationsPath( - translationsId: string, - options: {mode: 'draft' | 'published'} - ) { - const mode = options.mode; - if (!(mode === 'draft' || mode === 'published')) { - throw new Error(`invalid mode: ${mode}`); - } - const slug = normalizeSlug(translationsId); - const dbPath = `Projects/${this.projectId}/TranslationsManager/${mode}/Translations/${slug}`; - return dbPath; - } - - /** - * Firestore doc ref for a translations file. - */ - dbTranslationsRef( - translationsId: string, - options: {mode: 'draft' | 'published'} - ) { - const dbPath = this.dbTranslationsPath(translationsId, options); - return this.db.doc(dbPath); - } - /** * Returns a data source configuration object. */ @@ -2327,9 +2402,15 @@ export interface BatchRequestOptions { mode: 'draft' | 'published'; /** * Whether to automatically fetch translations for the docs retrieved in the - * request. + * request (including docs returned by queries). */ translate?: boolean; + /** + * Locales to fetch translations for. Each locale is expanded through its + * fallback chain (per the `i18n.fallbacks` config). Defaults to the locales + * configured in `i18n.locales`. + */ + locales?: string[]; } export interface BatchRequestQuery { @@ -2346,24 +2427,6 @@ export interface BatchRequestQueryOptions { query?: (query: Query) => Query; } -export interface TranslationsDoc { - id: string; - sys: { - modifiedAt: Timestamp; - modifiedBy: string; - publishedAt?: Timestamp; - publishedBy?: string; - linkedSheet?: { - spreadsheetId: string; - gid: number; - linkedAt: Timestamp; - linkedBy: string; - }; - tags?: string[]; - }; - strings: TranslationsMap; -} - export class BatchRequest { cmsClient: RootCMSClient; private options: BatchRequestOptions; @@ -2372,6 +2435,15 @@ export class BatchRequest { private dataSourceIds: string[] = []; private queries: BatchRequestQuery[] = []; private translationsIds: string[] = []; + /** + * Translations ids auto-collected from query results when + * `options.translate` is enabled. Kept separate from the explicitly-added + * ids so that the merge order is deterministic (generic translations first, + * doc-specific translations last). + */ + private queryTranslationsIds: string[] = []; + /** Translations ids auto-collected from `addDoc()` docs. */ + private docTranslationsIds: string[] = []; constructor(cmsClient: RootCMSClient, options: BatchRequestOptions) { this.cmsClient = cmsClient; @@ -2409,7 +2481,7 @@ export class BatchRequest { } /** - * Adds a translation file to the request. + * Adds a translations doc to the request. */ addTranslations(translationsId: string) { this.translationsIds.push(translationsId); @@ -2419,25 +2491,25 @@ export class BatchRequest { * Fetches data from the DB. */ async fetch(): Promise { - const res = new BatchResponse(); + const res = new BatchResponse(this.cmsClient.rootConfig.i18n || {}); const promises = [ this.fetchDocs(res), this.fetchQueries(res), this.fetchDataSources(res), ]; - // If `options.translate` is disabled and translations are requested, - // fetch the translations in parallel with the other docs. + // If `options.translate` is disabled, any explicitly-added translations + // can be fetched in parallel with the other docs. if (!this.options.translate && this.translationsIds.length > 0) { promises.push(this.fetchTranslations(res)); } await Promise.all(promises); - // If `options.translate` is enabled, the fetchX() methods will - // automatically add each doc's translations id to the request, so - // translations should be fetched after all the other docs are fetched. - if (this.translationsIds.length > 0) { + // If `options.translate` is enabled, `fetchDocs()` and `fetchQueries()` + // auto-collect each doc's translations id, so translations are fetched + // after all the other docs are fetched. + if (this.options.translate) { await this.fetchTranslations(res); } @@ -2465,7 +2537,7 @@ export class BatchRequest { res.docs[docId] = docData; if (this.options.translate) { - this.addTranslations(docId); + this.docTranslationsIds.push(docId); } }); } @@ -2503,6 +2575,11 @@ export class BatchRequest { results.forEach((result) => { const doc = unmarshalData(result.data()) as Doc; docs.push(doc); + // Based on the results of the query, fetch the corresponding + // translations for each doc. + if (this.options.translate && doc.id) { + this.queryTranslationsIds.push(doc.id); + } }); res.queries[queryItem.queryId] = docs; }; @@ -2531,24 +2608,95 @@ export class BatchRequest { } private async fetchTranslations(res: BatchResponse) { - if (this.translationsIds.length === 0) { + // Order the translations ids so that precedence is deterministic: + // generic translations (e.g. "common") first, docs returned from queries + // next, and specific docs (e.g. "Pages/index") last. + const translationsIds = Array.from( + new Set([ + ...this.translationsIds, + ...this.queryTranslationsIds, + ...this.docTranslationsIds, + ]) + ); + if (translationsIds.length === 0) { return; } - - const docRefs = this.translationsIds.map((translationsId) => { - return this.cmsClient.dbTranslationsRef(translationsId, { - mode: this.options.mode, + const mode = this.options.mode; + const project = this.cmsClient.projectId; + const i18nConfig = this.cmsClient.rootConfig.i18n || {}; + + const locales = this.options.locales || i18nConfig.locales; + if (locales && locales.length > 0) { + // When the locales are known, expand each locale through its fallback + // chain and fetch the exact locale doc refs with `getAll()`. + const localeSet = new Set(); + for (const locale of locales) { + for (const fallback of resolveLocaleFallbacks(i18nConfig, locale)) { + localeSet.add(fallback); + } + } + const localeDocRefs: Array<{ + translationsId: string; + locale: string; + ref: FirebaseFirestore.DocumentReference; + }> = []; + for (const translationsId of translationsIds) { + for (const locale of localeSet) { + const dbPath = buildTranslationsLocaleDocDbPath({ + project, + mode, + id: translationsId, + locale, + }); + localeDocRefs.push({ + translationsId, + locale, + ref: this.db.doc(dbPath), + }); + } + } + // Fetch the refs in chunks. Missing locale docs are fine (e.g. a + // translations doc may not have every locale). + const chunks = chunkArray(localeDocRefs, 300); + const snapshotChunks = await Promise.all( + chunks.map((chunk) => this.db.getAll(...chunk.map((item) => item.ref))) + ); + const snapshots = snapshotChunks.flat(); + localeDocRefs.forEach((item, i) => { + const snapshot = snapshots[i]; + if (!snapshot.exists) { + return; + } + res.translations[item.translationsId] ??= {}; + res.translations[item.translationsId][item.locale] = + snapshot.data() as TranslationsLocaleDoc; }); - }); - const docs = await this.db.getAll(...docRefs); - this.translationsIds.forEach((translationsId, i) => { - const doc = docs[i]; - if (!doc.exists) { - // console.warn(`translations "${translationsId}" does not exist`); - return; + } else { + // When the locales are unknown, query the translations locale docs by + // translations id (chunked due to Firestore's `in` query limits). + const dbPath = buildTranslationsDbPath({project, mode}); + const collectionRef = this.db.collection(dbPath); + const snapshots = await Promise.all( + chunkArray(translationsIds, 10).map((chunk) => + collectionRef.where('id', 'in', chunk).get() + ) + ); + const localeDocsById: Record = {}; + snapshots.forEach((snapshot) => { + snapshot.docs.forEach((doc) => { + const localeDoc = doc.data() as TranslationsLocaleDoc; + localeDocsById[localeDoc.id] ??= []; + localeDocsById[localeDoc.id].push(localeDoc); + }); + }); + // Store the results in insertion (precedence) order. + for (const translationsId of translationsIds) { + for (const localeDoc of localeDocsById[translationsId] || []) { + res.translations[translationsId] ??= {}; + res.translations[translationsId][localeDoc.locale] = localeDoc; + } } - res.translations[translationsId] = doc.data() as TranslationsDoc; - }); + } } } @@ -2556,55 +2704,69 @@ export class BatchResponse { docs: Record = {}; queries: Record = {}; dataSources: Record = {}; - translations: Record = {}; + /** + * Translations locale docs retrieved in the request, keyed by translations + * id then by locale. The ids are in precedence order: generic translations + * (e.g. "common") first, doc-specific translations last. + */ + translations: Record> = {}; + + private i18nConfig: RootConfig['i18n']; + + constructor(i18nConfig?: RootConfig['i18n']) { + this.i18nConfig = i18nConfig || {}; + } /** * Returns a map of translations for a given locale or locale fallbacks. * - * The input is either a single locale (e.g. "de") or an array of locales - * representing the fallback tree, e.g. ["en-CA", "en-GB", "en"]. - * - * TODO(stevenle): support the locale fallback tree. + * The input is either a single locale (e.g. "de"), which is expanded + * through the fallback chain configured in `i18n.fallbacks`, or an array of + * locales representing an explicit fallback chain, e.g. + * `["en-CA", "en-GB", "en"]`. * * The returned value is a flat map of source string to translated string, * e.g.: * {"": ""} */ - getTranslations(locale: string): LocaleTranslations { - const translationsMap = this.getTranslationsMap(); - const translations = translationsForLocale(translationsMap, locale); - return translations; - } - - /** - * Merges the strings from all translations files retrieved in the request. - * The returned value is a map of string to translations, e.g.: - * - * {"": {"source": "", "": ""}} - */ - private getTranslationsMap(): TranslationsMap { - // Load translations in the following order: - // - generic translations (e.g. "global") - // - docs returned from queries (e.g. "list all blog posts") - // - specific docs (e.g. "Pages/index") - const translationsDocs = Object.values(this.translations).reverse(); - - // Consolidate the strings from all of the translations files. - // {"": {"source": "", "": ""}} - const translationsMap: TranslationsMap = {}; - for (const translationsDoc of translationsDocs) { - const strings = translationsDoc.strings || {}; - for (const hash in strings) { - const translations = strings[hash]; - translationsMap[hash] ??= {source: translations.source}; - for (const locale in translations) { - if (locale !== 'source' && translations[locale]) { - translationsMap[hash][locale] = translations[locale]; + getTranslations(locale: string | string[]): LocaleTranslations { + const fallbackLocales = Array.isArray(locale) + ? locale + : resolveLocaleFallbacks(this.i18nConfig, locale); + + // Merge the strings from all of the translations docs. The docs are + // merged in precedence order (generic translations first, doc-specific + // translations last) so that later docs override earlier ones. + const merged: Record< + string, + {source: string; translations: Record} + > = {}; + for (const localeDocs of Object.values(this.translations)) { + for (const localeDoc of Object.values(localeDocs)) { + const strings = localeDoc.strings || {}; + for (const hash in strings) { + const entry = strings[hash]; + merged[hash] ??= {source: entry.source, translations: {}}; + if (entry.translation) { + merged[hash].translations[localeDoc.locale] = entry.translation; } } } } - return translationsMap; + // For each string, pick the first locale in the fallback chain with a + // non-empty translation, falling back to the source string. + const localeTranslations: LocaleTranslations = {}; + for (const item of Object.values(merged)) { + let translation = item.source; + for (const fallbackLocale of fallbackLocales) { + if (item.translations[fallbackLocale]) { + translation = item.translations[fallbackLocale]; + break; + } + } + localeTranslations[item.source] = translation; + } + return localeTranslations; } } diff --git a/packages/root-cms/core/core.ts b/packages/root-cms/core/core.ts index 70994ee9a..a451abbb5 100644 --- a/packages/root-cms/core/core.ts +++ b/packages/root-cms/core/core.ts @@ -3,3 +3,4 @@ export * from './route.js'; export * from './runtime.js'; export * as schema from './schema.js'; export * from './translations-manager.js'; +export * from './translations-migration.js'; diff --git a/packages/root-cms/core/plugin.ts b/packages/root-cms/core/plugin.ts index bb6f9c9a7..31a27e960 100644 --- a/packages/root-cms/core/plugin.ts +++ b/packages/root-cms/core/plugin.ts @@ -781,6 +781,29 @@ export function cmsPlugin(options: CMSPluginOptions): CMSPlugin { */ preBuild: async (rootConfig: RootConfig) => { await writeCollectionSchemasToJson(rootConfig); + + // When the v2 translations manager is enabled, migrate v1 + // translations before the build (SSG reads translations at build + // time). A failure fails the build. + if (options.experiments?.v2TranslationsManager) { + const {migrateV1TranslationsIfNeeded} = + await import('./translations-migration.js'); + const cmsClient = new RootCMSClient(rootConfig); + try { + await migrateV1TranslationsIfNeeded(cmsClient, { + trigger: 'build', + }); + } catch (err) { + console.error( + '[root cms] v1 -> v2 translations migration failed. the build ' + + 'requires Firestore access when `v2TranslationsManager` is ' + + 'enabled — ensure application default credentials (ADC) are ' + + 'available, e.g. run `gcloud auth application-default login` ' + + 'or set GOOGLE_APPLICATION_CREDENTIALS.' + ); + throw err; + } + } }, }, @@ -800,6 +823,25 @@ export function cmsPlugin(options: CMSPluginOptions): CMSPlugin { server: Server, serverOptions: ConfigureServerOptions ) => { + // When the v2 translations manager is enabled, migrate v1 translations + // on dev server startup. Fire-and-forget so dev startup isn't blocked; + // prod servers never migrate (the migration runs at build time via the + // preBuild hook). + if ( + options.experiments?.v2TranslationsManager && + serverOptions.type === 'dev' + ) { + import('./translations-migration.js') + .then(({migrateV1TranslationsIfNeeded}) => { + const cmsClient = new RootCMSClient(serverOptions.rootConfig); + return migrateV1TranslationsIfNeeded(cmsClient, {trigger: 'dev'}); + }) + .catch((err) => { + console.error('[root cms] v1 -> v2 translations migration failed:'); + console.error(err); + }); + } + // The AI "prepare" routes can carry a moderately large body (the edit // flow sends the JSON object being edited). Use a larger limit for those // routes only; body-parser short-circuits when `req._body` is already diff --git a/packages/root-cms/core/publish-translations.test.ts b/packages/root-cms/core/publish-translations.test.ts new file mode 100644 index 000000000..83a44934d --- /dev/null +++ b/packages/root-cms/core/publish-translations.test.ts @@ -0,0 +1,196 @@ +/** + * Tests for the publish/unpublish integration between content docs and the + * v2 TranslationsManager, against the Firestore emulator (see + * translations-manager.test.ts for details on the emulator setup). + */ + +import {getApps, initializeApp} from 'firebase-admin/app'; +import {Timestamp, getFirestore} from 'firebase-admin/firestore'; +import {beforeEach, describe, expect, it} from 'vitest'; +import {RootCMSClient} from './client.js'; +import {TranslationsLocaleDoc} from './translations-manager.js'; + +const FIREBASE_PROJECT_ID = 'rootjs-cms-admin-tests'; + +function getTestApp() { + const existing = getApps().find((app) => app.name === 'publish-test'); + if (existing) { + return existing; + } + return initializeApp({projectId: FIREBASE_PROJECT_ID}, 'publish-test'); +} + +let projectCounter = 0; + +function createTestCmsClient(options?: { + experiments?: Record; +}): RootCMSClient { + const app = getTestApp(); + const db = getFirestore(app); + const projectId = `publish-test-${Date.now()}-${projectCounter++}`; + const plugin = { + name: 'root-cms', + getConfig: () => ({ + id: projectId, + firebaseConfig: { + apiKey: 'test', + authDomain: 'test', + projectId: FIREBASE_PROJECT_ID, + storageBucket: 'test', + }, + experiments: options?.experiments ?? {v2TranslationsManager: true}, + }), + getFirebaseApp: () => app, + getFirestore: () => db, + }; + const rootConfig: any = { + rootDir: '/test', + i18n: {locales: ['en', 'es']}, + plugins: [plugin], + }; + return new RootCMSClient(rootConfig); +} + +async function seedDraftDoc(cmsClient: RootCMSClient, docId: string) { + const [collection, slug] = docId.split('/'); + await cmsClient.db + .doc( + `Projects/${cmsClient.projectId}/Collections/${collection}/Drafts/${slug}` + ) + .set({ + id: docId, + collection, + slug, + sys: { + createdAt: Timestamp.now(), + createdBy: 'test', + modifiedAt: Timestamp.now(), + modifiedBy: 'test', + locales: ['en', 'es'], + }, + fields: {title: `Title for ${docId}`}, + }); +} + +describe.skipIf(!process.env.FIRESTORE_EMULATOR_HOST)( + 'publishDocs() translations integration', + () => { + let cmsClient: RootCMSClient; + + beforeEach(() => { + cmsClient = createTestCmsClient(); + }); + + it('publishes a doc together with its translations', async () => { + await seedDraftDoc(cmsClient, 'Pages/index'); + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('Pages/index', {hello: {es: 'hola'}}); + + await cmsClient.publishDocs(['Pages/index'], { + publishedBy: 'test@example.com', + }); + + // The published doc exists. + const publishedDoc = await cmsClient.db + .doc( + `Projects/${cmsClient.projectId}/Collections/Pages/Published/index` + ) + .get(); + expect(publishedDoc.exists).toBe(true); + + // The published translations exist. + const publishedTranslations = await tm.loadTranslations({ + ids: ['Pages/index'], + mode: 'published', + }); + expect(publishedTranslations.hello.es).toBe('hola'); + + // The draft locale doc's sys records the publish. + const draftLocaleDoc = await cmsClient.db + .doc( + `Projects/${cmsClient.projectId}/TranslationsManager/draft/Translations/Pages--index:es` + ) + .get(); + const draftData = draftLocaleDoc.data() as TranslationsLocaleDoc; + expect(draftData.sys.publishedBy).toBe('test@example.com'); + }); + + it('does not publish unrelated translations docs', async () => { + await seedDraftDoc(cmsClient, 'Pages/index'); + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('Pages/index', {hello: {es: 'hola'}}); + await tm.saveTranslations('common', {bye: {es: 'adios'}}); + + await cmsClient.publishDocs(['Pages/index'], { + publishedBy: 'test@example.com', + }); + + const published = await tm.loadTranslations({ + ids: ['common'], + mode: 'published', + }); + expect(published).toEqual({}); + }); + + it('does not touch translations when the flag is disabled', async () => { + const v1Client = createTestCmsClient({experiments: {}}); + await seedDraftDoc(v1Client, 'Pages/index'); + // Seed a draft translations locale doc directly (the manager API is + // gated behind the flag). + await v1Client.db + .doc( + `Projects/${v1Client.projectId}/TranslationsManager/draft/Translations/Pages--index:es` + ) + .set({ + id: 'Pages/index', + locale: 'es', + tags: [], + strings: {}, + sys: {modifiedAt: Timestamp.now(), modifiedBy: 'test'}, + }); + + await v1Client.publishDocs(['Pages/index'], { + publishedBy: 'test@example.com', + }); + + const publishedLocaleDoc = await v1Client.db + .doc( + `Projects/${v1Client.projectId}/TranslationsManager/published/Translations/Pages--index:es` + ) + .get(); + expect(publishedLocaleDoc.exists).toBe(false); + }); + + it('unpublishDocs() removes published translations', async () => { + await seedDraftDoc(cmsClient, 'Pages/index'); + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('Pages/index', {hello: {es: 'hola'}}); + await cmsClient.publishDocs(['Pages/index'], { + publishedBy: 'test@example.com', + }); + + await cmsClient.unpublishDocs(['Pages/index'], { + unpublishedBy: 'test@example.com', + }); + + // The published locale docs are deleted. + const published = await tm.loadTranslations({ + ids: ['Pages/index'], + mode: 'published', + }); + expect(published).toEqual({}); + + // The draft locale doc's published metadata is cleared. + const draftLocaleDoc = await cmsClient.db + .doc( + `Projects/${cmsClient.projectId}/TranslationsManager/draft/Translations/Pages--index:es` + ) + .get(); + const draftData = draftLocaleDoc.data() as TranslationsLocaleDoc; + expect(draftData.sys.publishedAt).toBeUndefined(); + expect(draftData.sys.publishedBy).toBeUndefined(); + // The draft translations themselves remain. + expect(draftData.id).toBe('Pages/index'); + }); + } +); diff --git a/packages/root-cms/core/route.ts b/packages/root-cms/core/route.ts index f9b6f2ef4..35e519917 100644 --- a/packages/root-cms/core/route.ts +++ b/packages/root-cms/core/route.ts @@ -33,8 +33,10 @@ import { RootConfig, RouteParams, } from '@blinkk/root'; +import {resolveLocaleFallbacks} from '../shared/locale-fallbacks.js'; import {normalizeSlug} from '../shared/slug.js'; import {DocMode, RootCMSClient, translationsForLocale} from './client.js'; +import {translationsForLocaleV2} from './translations-manager.js'; export interface RootCMSDoc { /** The id of the doc, e.g. "Pages/foo-bar". */ @@ -224,8 +226,46 @@ export function createRoute(options: CreateRouteOptions): Route { return resolvePromisesMap(promisesMap); } + /** + * Loads the translations used by the route. When the v2 translations + * manager is enabled, translations are read from the v2 per-locale docs + * (using the same ids as the v1 tags); otherwise the v1 translations + * collection is used. + */ + function loadRouteTranslations( + cmsClient: RootCMSClient, + translationsTags: string[], + mode: DocMode + ): Promise> { + if (cmsClient.isV2TranslationsEnabled()) { + const tm = cmsClient.getTranslationsManager(); + return tm.loadTranslations({ids: translationsTags, mode}); + } + return cmsClient.loadTranslations({tags: translationsTags}); + } + + /** + * Flattens a loaded translations map to a source -> translation map for a + * locale, resolving the locale's fallback chain (`i18n.fallbacks`) when the + * v2 translations manager is enabled. + */ + function translationsMapForLocale( + cmsClient: RootCMSClient, + translationsMap: Record, + locale: string + ): Record { + if (cmsClient.isV2TranslationsEnabled()) { + const fallbackLocales = resolveLocaleFallbacks( + cmsClient.rootConfig.i18n, + locale + ); + return translationsForLocaleV2(translationsMap, fallbackLocales); + } + return translationsForLocale(translationsMap, locale); + } + async function generateProps(routeContext: RouteContext, locale: string) { - const {slug, mode} = routeContext; + const {slug, mode, cmsClient} = routeContext; const translationsTags = [ 'common', `${options.collection}/${normalizeSlug(slug)}`, @@ -234,12 +274,16 @@ export function createRoute(options: CreateRouteOptions): Route { const tags = options.translations(routeContext)?.tags || []; translationsTags.push(...tags); } + const siteLocales = cmsClient.rootConfig.i18n?.locales || ['en']; const [doc, translationsMap, data] = await Promise.all([ cmsClient.getDoc(options.collection, slug, { mode, }), - cmsClient.loadTranslations({tags: translationsTags}), + // Only load translations for sites that have >1 locale configured. + siteLocales.length > 1 + ? loadRouteTranslations(cmsClient, translationsTags, mode) + : Promise.resolve({}), fetchData(routeContext), ]); if (!doc) { @@ -250,7 +294,11 @@ export function createRoute(options: CreateRouteOptions): Route { return {notFound: true}; } - const translations = translationsForLocale(translationsMap, locale); + const translations = translationsMapForLocale( + cmsClient, + translationsMap, + locale + ); let props: any = {...data, locale, mode, slug, doc}; if (options.preRenderHook) { props = await options.preRenderHook(props, routeContext); @@ -305,7 +353,7 @@ export function createRoute(options: CreateRouteOptions): Route { }), // Only load translations for sites that have >1 locale configured. siteLocales.length > 1 - ? cmsClient.loadTranslations({tags: translationsTags}) + ? loadRouteTranslations(cmsClient, translationsTags, mode) : Promise.resolve({}), fetchData(routeContext), ]); @@ -408,7 +456,11 @@ export function createRoute(options: CreateRouteOptions): Route { } } - const translations = translationsForLocale(translationsMap, locale); + const translations = translationsMapForLocale( + cmsClient, + translationsMap, + locale + ); let props: any = {...data, req, locale, mode, slug, doc, country}; if (options.preRenderHook) { props = await options.preRenderHook(props, routeContext); diff --git a/packages/root-cms/core/translations-manager.test.ts b/packages/root-cms/core/translations-manager.test.ts new file mode 100644 index 000000000..b3f7fcc58 --- /dev/null +++ b/packages/root-cms/core/translations-manager.test.ts @@ -0,0 +1,379 @@ +/** + * Tests for the v2 TranslationsManager. These tests run against the Firestore + * emulator (the package's test script runs vitest under + * `firebase emulators:exec`, which sets FIRESTORE_EMULATOR_HOST so the admin + * SDK auto-connects to the emulator). + */ + +import crypto from 'node:crypto'; +import {getApps, initializeApp} from 'firebase-admin/app'; +import {Timestamp, getFirestore} from 'firebase-admin/firestore'; +import {beforeEach, describe, expect, it} from 'vitest'; +import {hashStr} from '../shared/strings.js'; +import {RootCMSClient} from './client.js'; +import { + TranslationsLocaleDoc, + translationsForLocaleV2, +} from './translations-manager.js'; + +const FIREBASE_PROJECT_ID = 'rootjs-cms-admin-tests'; + +function getTestApp() { + const existing = getApps().find((app) => app.name === 'tm-test'); + if (existing) { + return existing; + } + return initializeApp({projectId: FIREBASE_PROJECT_ID}, 'tm-test'); +} + +let projectCounter = 0; + +/** + * Creates a RootCMSClient backed by the Firestore emulator. Each call uses a + * unique CMS project id so tests are isolated from each other. + */ +function createTestCmsClient(options?: { + i18n?: any; + experiments?: Record; +}): RootCMSClient { + const app = getTestApp(); + const db = getFirestore(app); + const projectId = `test-project-${Date.now()}-${projectCounter++}`; + const plugin = { + name: 'root-cms', + getConfig: () => ({ + id: projectId, + firebaseConfig: { + apiKey: 'test', + authDomain: 'test', + projectId: FIREBASE_PROJECT_ID, + storageBucket: 'test', + }, + experiments: options?.experiments ?? {v2TranslationsManager: true}, + }), + getFirebaseApp: () => app, + getFirestore: () => db, + }; + const rootConfig: any = { + rootDir: '/test', + i18n: options?.i18n ?? {locales: ['en', 'es', 'fr']}, + plugins: [plugin], + }; + return new RootCMSClient(rootConfig); +} + +function sha1(str: string) { + return crypto.createHash('sha1').update(str).digest('hex'); +} + +describe.skipIf(!process.env.FIRESTORE_EMULATOR_HOST)( + 'TranslationsManager', + () => { + let cmsClient: RootCMSClient; + + beforeEach(() => { + cmsClient = createTestCmsClient(); + }); + + it('getTranslationsManager() throws when the flag is disabled', () => { + const disabledClient = createTestCmsClient({experiments: {}}); + expect(() => disabledClient.getTranslationsManager()).toThrowError( + /v2TranslationsManager/ + ); + expect(disabledClient.isV2TranslationsEnabled()).toBe(false); + expect(cmsClient.isV2TranslationsEnabled()).toBe(true); + }); + + it('saves draft translations as per-locale docs', async () => { + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations( + 'Pages/index', + { + one: {es: 'uno', fr: 'un'}, + two: {es: 'dos'}, + }, + {tags: ['Pages/index']} + ); + + const dbPath = `Projects/${cmsClient.projectId}/TranslationsManager/draft/Translations`; + // Slashes in the translations id are normalized to `--` in doc keys. + const esDoc = await cmsClient.db.doc(`${dbPath}/Pages--index:es`).get(); + expect(esDoc.exists).toBe(true); + const esData = esDoc.data() as TranslationsLocaleDoc; + expect(esData.id).toBe('Pages/index'); + expect(esData.locale).toBe('es'); + expect(esData.tags).toEqual(['Pages/index']); + expect(esData.strings[hashStr('one')]).toEqual({ + source: 'one', + translation: 'uno', + }); + expect(esData.strings[hashStr('two')]).toEqual({ + source: 'two', + translation: 'dos', + }); + expect(esData.sys.modifiedAt).toBeDefined(); + + const frDoc = await cmsClient.db.doc(`${dbPath}/Pages--index:fr`).get(); + expect(frDoc.exists).toBe(true); + expect((frDoc.data() as TranslationsLocaleDoc).strings).toEqual({ + [hashStr('one')]: {source: 'one', translation: 'un'}, + }); + }); + + it('loads draft translations by id', async () => { + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('common', {hello: {es: 'hola'}}); + const strings = await tm.loadTranslations({ + ids: ['common'], + mode: 'draft', + }); + expect(strings).toEqual({hello: {source: 'hello', es: 'hola'}}); + }); + + it('chunks `in` queries when loading more than 10 ids', async () => { + const tm = cmsClient.getTranslationsManager(); + const ids: string[] = []; + for (let i = 0; i < 12; i++) { + const id = `Pages/page-${i}`; + ids.push(id); + await tm.saveTranslations(id, {[`string ${i}`]: {es: `cadena ${i}`}}); + } + const strings = await tm.loadTranslations({ids, mode: 'draft'}); + expect(Object.keys(strings)).toHaveLength(12); + expect(strings['string 11'].es).toBe('cadena 11'); + }); + + it('merges translations in ids order (doc-specific wins)', async () => { + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('common', {hello: {es: 'hola (common)'}}); + await tm.saveTranslations('Pages/index', {hello: {es: 'hola (doc)'}}); + const strings = await tm.loadTranslations({ + ids: ['common', 'Pages/index'], + mode: 'draft', + }); + expect(strings.hello.es).toBe('hola (doc)'); + }); + + it('loads translations by tags using array-contains-any', async () => { + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations( + 'common', + {hello: {es: 'hola'}}, + { + tags: ['common'], + } + ); + await tm.saveTranslations( + 'Pages/foo', + {bye: {es: 'adios'}}, + { + tags: ['Pages/foo'], + } + ); + const strings = await tm.loadTranslations({ + tags: ['common', 'other'], + mode: 'draft', + }); + expect(strings).toEqual({hello: {source: 'hello', es: 'hola'}}); + }); + + it('loadTranslationsForLocale() applies config fallbacks', async () => { + const client = createTestCmsClient({ + i18n: { + locales: ['en', 'en-GB', 'en-CA'], + fallbacks: {'en-CA': ['en-GB']}, + }, + }); + const tm = client.getTranslationsManager(); + await tm.saveTranslations('common', { + color: {'en-GB': 'colour'}, + localOnly: {'en-CA': 'local (en-CA)', 'en-GB': 'local (en-GB)'}, + untranslated: {fr: 'french only'}, + }); + const translations = await tm.loadTranslationsForLocale('en-CA', { + mode: 'draft', + }); + expect(translations.color).toBe('colour'); + expect(translations.localOnly).toBe('local (en-CA)'); + // Strings translated only into locales outside the fallback chain are + // not loaded (only the chain's locale docs are fetched). + expect(translations.untranslated).toBeUndefined(); + }); + + it('publishes draft translations to the published collection', async () => { + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('Pages/index', {one: {es: 'uno'}}); + await tm.publishTranslations('Pages/index', { + publishedBy: 'test@example.com', + }); + + const publishedPath = `Projects/${cmsClient.projectId}/TranslationsManager/published/Translations/Pages--index:es`; + const publishedDoc = await cmsClient.db.doc(publishedPath).get(); + expect(publishedDoc.exists).toBe(true); + const publishedData = publishedDoc.data() as TranslationsLocaleDoc; + expect(publishedData.strings[hashStr('one')].translation).toBe('uno'); + expect(publishedData.sys.publishedBy).toBe('test@example.com'); + expect(publishedData.sys.publishedAt).toBeDefined(); + + // The draft doc's sys should be updated with the published metadata. + const draftPath = `Projects/${cmsClient.projectId}/TranslationsManager/draft/Translations/Pages--index:es`; + const draftDoc = await cmsClient.db.doc(draftPath).get(); + expect((draftDoc.data() as TranslationsLocaleDoc).sys.publishedBy).toBe( + 'test@example.com' + ); + + // Published translations are returned by loadTranslations(). + const strings = await tm.loadTranslations({ids: ['Pages/index']}); + expect(strings.one.es).toBe('uno'); + }); + + it('publishTranslationsBulk() publishes multiple ids', async () => { + const tm = cmsClient.getTranslationsManager(); + await tm.saveTranslations('common', {hello: {es: 'hola'}}); + await tm.saveTranslations('Pages/index', {one: {es: 'uno', fr: 'un'}}); + const res = await tm.publishTranslationsBulk( + ['common', 'Pages/index', 'does-not-exist'], + {publishedBy: 'test@example.com'} + ); + expect(res.publishedIds.sort()).toEqual(['Pages/index', 'common']); + const strings = await tm.loadTranslations({ + ids: ['common', 'Pages/index'], + mode: 'published', + }); + expect(strings.hello.es).toBe('hola'); + expect(strings.one.fr).toBe('un'); + }); + + describe('importTranslationsFromV1', () => { + async function seedV1Translation( + source: string, + translations: Record, + tags?: string[] + ) { + const data: Record = {...translations, source}; + if (tags) { + data.tags = tags; + } + await cmsClient.db + .doc(`Projects/${cmsClient.projectId}/Translations/${sha1(source)}`) + .set(data); + } + + it('groups strings into a translations doc per tag', async () => { + await seedV1Translation('hello', {es: 'hola'}, [ + 'Pages', + 'Pages/index', + 'common', + ]); + await seedV1Translation('bye', {es: 'adios'}, ['Pages', 'Pages/foo']); + await seedV1Translation('untagged string', {es: 'sin etiqueta'}); + // Non-string locale values are skipped. + await seedV1Translation('bad values', {es: ['not', 'a', 'string']}, [ + 'common', + ]); + + const tm = cmsClient.getTranslationsManager(); + const res = await tm.importTranslationsFromV1(); + expect(res.ids.sort()).toEqual([ + 'Pages', + 'Pages/foo', + 'Pages/index', + 'common', + 'v1-untagged', + ]); + expect(res.stats.numStrings).toBe(4); + expect(res.stats.numDocs).toBe(5); + + const commonStrings = await tm.loadTranslations({ + ids: ['common'], + mode: 'draft', + }); + expect(commonStrings.hello.es).toBe('hola'); + // The bad-values string has no valid translations but its source is + // grouped without locale values (nothing to store per locale). + expect(commonStrings['bad values']).toBeUndefined(); + + const untagged = await tm.loadTranslations({ + ids: ['v1-untagged'], + mode: 'draft', + }); + expect(untagged['untagged string'].es).toBe('sin etiqueta'); + + // The import saves drafts only; nothing is published yet. + const published = await tm.loadTranslations({ids: ['common']}); + expect(published).toEqual({}); + }); + + it('migrates the linked l10n sheet from the doc', async () => { + await seedV1Translation('hello', {es: 'hola'}, ['Pages/index']); + await cmsClient.db + .doc(`Projects/${cmsClient.projectId}/Collections/Pages/Drafts/index`) + .set({ + id: 'Pages/index', + collection: 'Pages', + slug: 'index', + sys: { + createdAt: Timestamp.now(), + createdBy: 'test', + modifiedAt: Timestamp.now(), + modifiedBy: 'test', + l10nSheet: { + spreadsheetId: 'sheet123', + gid: 0, + linkedAt: Timestamp.now(), + linkedBy: 'test@example.com', + }, + }, + fields: {}, + }); + + const tm = cmsClient.getTranslationsManager(); + await tm.importTranslationsFromV1(); + + const localeDoc = await cmsClient.db + .doc( + `Projects/${cmsClient.projectId}/TranslationsManager/draft/Translations/Pages--index:es` + ) + .get(); + const data = localeDoc.data() as TranslationsLocaleDoc; + expect(data.sys.linkedSheet?.spreadsheetId).toBe('sheet123'); + }); + + it('returns empty results when there are no v1 translations', async () => { + const tm = cmsClient.getTranslationsManager(); + const res = await tm.importTranslationsFromV1(); + expect(res.ids).toEqual([]); + expect(res.stats).toEqual({numStrings: 0, numDocs: 0}); + }); + }); + } +); + +describe('translationsForLocaleV2', () => { + it('resolves translations through the fallback chain', () => { + const strings = { + one: {'en-GB': 'one (GB)', en: 'one (en)'}, + two: {en: 'two (en)'}, + three: {fr: 'trois'}, + }; + const translations = translationsForLocaleV2(strings, [ + 'en-CA', + 'en-GB', + 'en', + ]); + expect(translations).toEqual({ + one: 'one (GB)', + two: 'two (en)', + three: 'three', + }); + }); + + it('skips empty translations in the chain', () => { + const strings = { + one: {'en-CA': '', 'en-GB': 'one (GB)'}, + }; + expect(translationsForLocaleV2(strings, ['en-CA', 'en-GB'])).toEqual({ + one: 'one (GB)', + }); + }); +}); diff --git a/packages/root-cms/core/translations-manager.ts b/packages/root-cms/core/translations-manager.ts index 4b5ac2e2c..1ca78a77e 100644 --- a/packages/root-cms/core/translations-manager.ts +++ b/packages/root-cms/core/translations-manager.ts @@ -1,9 +1,11 @@ import { + DocumentReference, FieldValue, Query, Timestamp, WriteBatch, } from 'firebase-admin/firestore'; +import {resolveLocaleFallbacks} from '../shared/locale-fallbacks.js'; import {normalizeSlug} from '../shared/slug.js'; import {hashStr} from '../shared/strings.js'; import type {RootCMSClient} from './client.js'; @@ -13,6 +15,18 @@ const TRANSLATIONS_DB_PATH_FORMAT = const TRANSLATIONS_LOCALE_DOC_DB_PATH_FORMAT = `${TRANSLATIONS_DB_PATH_FORMAT}/{id}:{locale}`; +/** + * Firestore limits `in` queries to 10 values, so larger queries are broken up + * into chunks. + */ +const IN_QUERY_CHUNK_SIZE = 10; + +/** + * Firestore batches allow a max of 500 write ops. Use a slightly lower limit + * to leave headroom for callers that add their own ops to a shared batch. + */ +const MAX_BATCH_OPS = 400; + export type Locale = string; export type SourceString = string; export type TranslatedString = string; @@ -26,7 +40,7 @@ export type TranslationsDocMode = 'draft' | 'published'; * This type is not meant to be used by external callers since this is primarily * an internal implementation detail. */ -interface TranslationsLocaleDoc { +export interface TranslationsLocaleDoc { /** * Translations id. In most cases, this is the same as the doc id, e.g. * `Pages/foo--bar`. @@ -40,15 +54,25 @@ interface TranslationsLocaleDoc { modifiedBy: string; publishedAt?: Timestamp; publishedBy?: string; - linkedSheet?: { - spreadsheetId: string; - gid: number; - linkedAt: Timestamp; - linkedBy: string; - }; + linkedSheet?: TranslationsLinkedSheet; }; } +export interface TranslationsLinkedSheet { + spreadsheetId: string; + gid: number; + linkedAt: Timestamp; + linkedBy: string; +} + +/** + * A translations locale doc paired with its Firestore doc ref. + */ +export interface TranslationsLocaleDocWithRef { + ref: DocumentReference; + data: TranslationsLocaleDoc; +} + export interface TranslationsLocaleDocHashMap { /** * A hash map of a source string's hash fingerprint to the source string and @@ -107,6 +131,20 @@ export interface SingleLocaleTranslationsMap { [source: SourceString]: TranslatedString; } +/** + * Stats returned by `importTranslationsFromV1()`. + */ +export interface ImportTranslationsFromV1Result { + /** Translations doc ids that were created or updated. */ + ids: string[]; + stats: { + /** Number of v1 source strings imported. */ + numStrings: number; + /** Number of v2 translations docs saved. */ + numDocs: number; + }; +} + export class TranslationsManager { cmsClient: RootCMSClient; @@ -129,7 +167,11 @@ export class TranslationsManager { async saveTranslations( id: string, strings: MultiLocaleTranslationsMap, - options?: {tags?: string[]; modifiedBy?: string} + options?: { + tags?: string[]; + modifiedBy?: string; + linkedSheet?: TranslationsLinkedSheet; + } ) { const mode = 'draft'; const localesSet: Set = new Set(); @@ -141,9 +183,15 @@ export class TranslationsManager { }); }); - const batch = this.cmsClient.db.batch(); + const db = this.cmsClient.db; + let batch = db.batch(); + let numOps = 0; const locales = Array.from(localesSet); - locales.forEach((locale) => { + for (const locale of locales) { + const hashMap = this.toLocaleDocHashMap(strings, locale); + if (Object.keys(hashMap).length === 0) { + continue; + } const updates: Record = { id: id, locale: locale, @@ -151,34 +199,32 @@ export class TranslationsManager { modifiedAt: Timestamp.now(), modifiedBy: options?.modifiedBy || 'root-cms-client', }, - strings: {}, + strings: hashMap, }; if (options?.tags && options.tags.length > 0) { updates.tags = FieldValue.arrayUnion(...options.tags); } - let numUpdates = 0; - const hashMap = this.toLocaleDocHashMap(strings, locale); - Object.entries(hashMap).forEach(([hash, translations]) => { - Object.entries(translations).forEach(([locale, translation]) => { - if (translation) { - updates.strings[hash] ??= {}; - updates.strings[hash][locale] = translation; - numUpdates += 1; - } - }); + if (options?.linkedSheet) { + updates.sys.linkedSheet = options.linkedSheet; + } + const localeDocPath = buildTranslationsLocaleDocDbPath({ + project: this.cmsClient.projectId, + mode: mode, + id: id, + locale: locale, }); - if (numUpdates > 0) { - const localeDocPath = buildTranslationsLocaleDocDbPath({ - project: this.cmsClient.projectId, - mode: mode, - id: id, - locale: locale, - }); - const localeDocRef = this.cmsClient.db.doc(localeDocPath); - batch.set(localeDocRef, updates, {merge: true}); + const localeDocRef = db.doc(localeDocPath); + batch.set(localeDocRef, updates, {merge: true}); + numOps += 1; + if (numOps >= MAX_BATCH_OPS) { + await batch.commit(); + batch = db.batch(); + numOps = 0; } - }); - await batch.commit(); + } + if (numOps > 0) { + await batch.commit(); + } } /** @@ -189,40 +235,128 @@ export class TranslationsManager { options?: {batch?: WriteBatch; publishedBy?: string} ) { const db = this.cmsClient.db; - const project = this.cmsClient.projectId; - const draftPath = buildTranslationsDbPath({project, mode: 'draft'}); - const query = db.collection(draftPath).where('id', '==', id); - const res = await query.get(); - if (res.size === 0) { + const localeDocsById = await this.getTranslationsLocaleDocs([id], 'draft'); + const localeDocs = localeDocsById[id] || []; + if (localeDocs.length === 0) { console.warn(`no translations to publish for ${id}`); return; } const batch = options?.batch || db.batch(); - res.docs.forEach((doc) => { - const translationsLocaleDoc = doc.data() as TranslationsLocaleDoc; + this.addPublishTranslationsOps(localeDocs, batch, { + publishedBy: options?.publishedBy, + }); + + // If a batch was provided, assume that the caller is responsible for + // calling `batch.commit()`. + const shouldCommitBatch = !options?.batch; + if (shouldCommitBatch) { + await batch.commit(); + } + } + + /** + * Publishes multiple translations docs by id, e.g.: + * ``` + * await tm.publishTranslationsBulk(['Pages/index', 'common']); + * ``` + */ + async publishTranslationsBulk( + ids: string[], + options?: {publishedBy?: string} + ): Promise<{publishedIds: string[]}> { + const db = this.cmsClient.db; + const localeDocsById = await this.getTranslationsLocaleDocs(ids, 'draft'); + const publishedIds: string[] = []; + let batch = db.batch(); + let numOps = 0; + for (const id of Object.keys(localeDocsById)) { + const localeDocs = localeDocsById[id]; + if (localeDocs.length === 0) { + continue; + } + // Keep all of a translations doc's ops within a single commit. + if (numOps > 0 && numOps + 2 * localeDocs.length > MAX_BATCH_OPS) { + await batch.commit(); + batch = db.batch(); + numOps = 0; + } + numOps += this.addPublishTranslationsOps(localeDocs, batch, options); + publishedIds.push(id); + } + if (numOps > 0) { + await batch.commit(); + } + return {publishedIds}; + } + + /** + * Adds the write ops for publishing a set of draft translations locale docs + * to a batch. For each locale doc, the draft doc's `sys` is updated with + * `publishedAt/By` and a copy is saved to the published collection. Returns + * the number of ops added to the batch (2 per locale doc). + */ + addPublishTranslationsOps( + localeDocs: TranslationsLocaleDocWithRef[], + batch: WriteBatch, + options?: {publishedBy?: string} + ): number { + const db = this.cmsClient.db; + const project = this.cmsClient.projectId; + const publishedBy = options?.publishedBy || 'root-cms-client'; + let numOps = 0; + for (const localeDoc of localeDocs) { + const data = localeDoc.data; const sys = { - ...translationsLocaleDoc.sys, + ...data.sys, publishedAt: Timestamp.now(), - publishedBy: options?.publishedBy || 'root-cms-client', + publishedBy: publishedBy, }; - batch.update(doc.ref, {sys}); + batch.update(localeDoc.ref, {sys}); const publishedDocPath = buildTranslationsLocaleDocDbPath({ - project, + project: project, mode: 'published', - id: translationsLocaleDoc.id, - locale: translationsLocaleDoc.locale, + id: data.id, + locale: data.locale, }); const publishedDocRef = db.doc(publishedDocPath); - batch.set(publishedDocRef, {...translationsLocaleDoc, sys}); - }); + batch.set(publishedDocRef, {...data, sys}); + numOps += 2; + } + return numOps; + } - // If a batch was provided, assume that the caller is responsible for - // calling `batch.commit()`. - const shouldCommitBatch = !options?.batch; - if (shouldCommitBatch) { - await batch.commit(); + /** + * Fetches the translations locale docs (with their doc refs) for a set of + * translations doc ids, grouped by id. + */ + async getTranslationsLocaleDocs( + ids: string[], + mode: TranslationsDocMode + ): Promise> { + const localeDocsById: Record = {}; + const uniqueIds = Array.from(new Set(ids)); + if (uniqueIds.length === 0) { + return localeDocsById; } + const dbPath = buildTranslationsDbPath({ + project: this.cmsClient.projectId, + mode: mode, + }); + const collectionRef = this.cmsClient.db.collection(dbPath); + const snapshots = await Promise.all( + chunkArray(uniqueIds, IN_QUERY_CHUNK_SIZE).map((chunk) => + collectionRef.where('id', 'in', chunk).get() + ) + ); + snapshots.forEach((snapshot) => { + snapshot.docs.forEach((doc) => { + const data = doc.data() as TranslationsLocaleDoc; + localeDocsById[data.id] ??= []; + localeDocsById[data.id].push({ref: doc.ref, data}); + }); + }); + return localeDocsById; } /** @@ -265,31 +399,69 @@ export class TranslationsManager { tags?: string[]; locales?: Locale[]; mode?: TranslationsDocMode; - }) { + }): Promise { const mode = options?.mode || 'published'; const dbPath = buildTranslationsDbPath({ project: this.cmsClient.projectId, mode: mode, }); + const collectionRef = this.cmsClient.db.collection(dbPath); - let query = this.cmsClient.db.collection(dbPath) as Query; - if (options?.ids && options.ids.length > 0) { - query = query.where('id', 'in', options.ids); - } - if (options?.tags && options.tags.length > 0) { - query = query.where('tags', 'array-contains', options.tags); - } - if (options?.locales && options.locales.length > 0) { - query = query.where('locale', 'in', options.locales); + const ids = options?.ids || []; + const tags = options?.tags || []; + const locales = options?.locales || []; + + let localeDocs: TranslationsLocaleDoc[]; + if (ids.length > 0) { + // Chunk the `in` query (Firestore limits `in` filters to 10 values) and + // run the chunks in parallel. Any tags/locales filters are applied in + // memory to avoid combining disjunctive filters in a single query. + const snapshots = await Promise.all( + chunkArray(ids, IN_QUERY_CHUNK_SIZE).map((chunk) => + collectionRef.where('id', 'in', chunk).get() + ) + ); + localeDocs = snapshots.flatMap((snapshot) => + snapshot.docs.map((doc) => doc.data() as TranslationsLocaleDoc) + ); + if (tags.length > 0) { + localeDocs = localeDocs.filter((localeDoc) => + (localeDoc.tags || []).some((tag) => tags.includes(tag)) + ); + } + if (locales.length > 0) { + localeDocs = localeDocs.filter((localeDoc) => + locales.includes(localeDoc.locale) + ); + } + // Merge the results in `ids` order so that precedence is deterministic, + // e.g. for `{ids: ['common', 'Pages/index']}` the doc-specific + // translations take precedence over the generic ones. + const idOrder = new Map(ids.map((id, i) => [id, i])); + localeDocs.sort( + (a, b) => (idOrder.get(a.id) ?? 0) - (idOrder.get(b.id) ?? 0) + ); + } else { + let query = collectionRef as Query; + if (tags.length > 0) { + query = query.where('tags', 'array-contains-any', tags); + } + if (locales.length > 0) { + query = query.where('locale', 'in', locales); + } + const results = await query.get(); + localeDocs = results.docs.map( + (doc) => doc.data() as TranslationsLocaleDoc + ); } - const results = await query.get(); const strings: MultiLocaleTranslationsMap = {}; - results.forEach((result) => { - const localeDoc = result.data() as TranslationsLocaleDoc; + localeDocs.forEach((localeDoc) => { Object.values(localeDoc.strings || {}).forEach((item) => { strings[item.source] ??= {source: item.source}; - strings[item.source][localeDoc.locale] = item.translation; + if (item.translation) { + strings[item.source][localeDoc.locale] = item.translation; + } }); }); return strings; @@ -299,6 +471,9 @@ export class TranslationsManager { * Fetches translations for a given locale, with optional fallbacks. * The return value is a map of source string to translated string. * + * If no `fallbackLocales` are provided, the fallback chain is resolved from + * the project's `i18n.fallbacks` config. + * * Example: * ``` * await translationsDoc.loadTranslationsForLocale('es'); @@ -315,49 +490,15 @@ export class TranslationsManager { ): Promise { const localeSet: Set = new Set([ locale, - ...(options?.fallbackLocales || []), + ...(options?.fallbackLocales || + resolveLocaleFallbacks(this.cmsClient.rootConfig.i18n, locale)), ]); const fallbackLocales = Array.from(localeSet); const multiLocaleStrings = await this.loadTranslations({ mode: options?.mode, locales: fallbackLocales, }); - return this.toSingleLocaleMap(multiLocaleStrings, fallbackLocales); - } - - /** - * Converts a multi-locale translations map to a flat single-locale map, - * with optional support for fallback locales. - * - * ``` - * const multiLocaleStrings = { - * 'one': {es: 'uno', fr: 'un'}, - * 'two': {es: 'dos', fr: 'deux'} - * }; - * translationsDoc.toSingleLocaleMap(multiLocaleStrings, ['es']); - * // => - * // { - * // "one": "uno", - * // "two": "dos", - * // } - * ``` - */ - private toSingleLocaleMap( - multiLocaleStrings: MultiLocaleTranslationsMap, - fallbackLocales: Locale[] - ): SingleLocaleTranslationsMap { - const singleLocaleStrings: SingleLocaleTranslationsMap = {}; - Object.entries(multiLocaleStrings).forEach(([source, translations]) => { - let translation = source; - for (const locale of fallbackLocales) { - if (translations[locale]) { - translation = translations[locale]; - break; - } - } - singleLocaleStrings[source] = translation; - }); - return singleLocaleStrings; + return translationsForLocaleV2(multiLocaleStrings, fallbackLocales); } /** @@ -396,71 +537,148 @@ export class TranslationsManager { } /** - * Import translations from the v1 system to the TranslationsManager. + * Imports translations from the v1 system to the TranslationsManager. + * + * Each v1 string is grouped into a v2 translations doc per tag (e.g. a + * string tagged `Pages/index` is saved to the `Pages/index` translations + * doc). Untagged strings are grouped into a `v1-untagged` doc so that + * nothing is dropped. The imported translations are saved as drafts; use + * `publishTranslationsBulk()` to publish them. */ - async importTranslationsFromV1() { + async importTranslationsFromV1(): Promise { const projectId = this.cmsClient.projectId; const db = this.cmsClient.db; const dbPath = `Projects/${projectId}/Translations`; const query = db.collection(dbPath); const querySnapshot = await query.get(); + const stats = {numStrings: 0, numDocs: 0}; if (querySnapshot.size === 0) { - return; + return {ids: [], stats}; } console.log( '[root cms] importing v1 Translations to v2 TranslationsManager' ); - const translationsDocs: Record = {}; + const translationsDocs: Record< + string, + {id: string; strings: MultiLocaleTranslationsMap} + > = {}; querySnapshot.forEach((doc) => { const translation = doc.data(); - const source = this.cmsClient.normalizeString(translation.source); - delete translation.source; - const tags = (translation.tags || []) as string[]; - delete translation.tags; - for (const tag of tags) { - if (tag.includes('/')) { - const translationsId = tag; - translationsDocs[translationsId] ??= { - id: translationsId, - tags: tags, - strings: {}, - }; - translationsDocs[translationsId].strings[source] = translation; + const source = this.cmsClient.normalizeString(translation.source || ''); + if (!source) { + return; + } + // Collect the locale values, ignoring metadata keys and any non-string + // values. + const localeValues: Record = {}; + for (const [key, value] of Object.entries(translation)) { + if (key === 'source' || key === 'tags') { + continue; } + if (typeof value !== 'string' || !value) { + continue; + } + localeValues[key] = value; + } + // Group the string into a translations doc per tag. Untagged strings + // are grouped into a `v1-untagged` doc so that nothing is dropped. + const tags = (translation.tags || []) as string[]; + const translationsIds = tags.length > 0 ? tags : ['v1-untagged']; + for (const translationsId of translationsIds) { + translationsDocs[translationsId] ??= { + id: translationsId, + strings: {}, + }; + translationsDocs[translationsId].strings[source] = localeValues; } + stats.numStrings += 1; }); - if (Object.keys(translationsDocs).length === 0) { + const ids = Object.keys(translationsDocs); + if (ids.length === 0) { console.log('[root cms] no v1 translations to save'); - return; + return {ids: [], stats}; } - // Move the doc's "l10nSheet" to the translations doc's "linkedSheet". - for (const docId in translationsDocs) { - const [collection, slug] = docId.split('/'); - if (collection && slug) { - const doc: any = await this.cmsClient.getDoc(collection, slug, { - mode: 'draft', - }); - const linkedSheet = doc?.sys?.l10nSheet; - if (linkedSheet) { - translationsDocs[docId].sys.linkedSheet = linkedSheet; + for (const translationsId of ids) { + const data = translationsDocs[translationsId]; + + // For doc-backed translations ids (e.g. `Pages/index`), move the doc's + // "l10nSheet" to the translations doc's "linkedSheet". + let linkedSheet: TranslationsLinkedSheet | undefined; + const sepIndex = translationsId.indexOf('/'); + if (sepIndex > 0) { + const collection = translationsId.slice(0, sepIndex); + const slug = translationsId.slice(sepIndex + 1); + try { + const rawDoc = await this.cmsClient.getRawDoc(collection, slug, { + mode: 'draft', + }); + linkedSheet = rawDoc?.sys?.l10nSheet; + } catch (err) { + // Tags are user-defined and may look like a doc id without matching + // an actual collection. Ignore lookup errors. + console.warn( + `[root cms] failed to look up doc for tag "${translationsId}":`, + String(err) + ); } } - } - Object.entries(translationsDocs).forEach(([translationsId, data]) => { - const len = Object.keys(data.strings).length; - console.log(`[root cms] saving ${len} string(s) to ${translationsId}...`); - this.saveTranslations(translationsId, data.strings, { - tags: data.tags || [translationsId], + const numStrings = Object.keys(data.strings).length; + console.log( + `[root cms] saving ${numStrings} string(s) to ${translationsId}...` + ); + await this.saveTranslations(translationsId, data.strings, { + tags: [translationsId], + linkedSheet: linkedSheet, + modifiedBy: 'root-cms v1 migration', }); - }); + stats.numDocs += 1; + } + return {ids, stats}; } } +/** + * Converts a multi-locale translations map to a flat single-locale map using + * a locale fallback chain. For each source string, the first locale in the + * chain with a non-empty translation wins; if no locale matches, the source + * string is returned. + * + * ``` + * const multiLocaleStrings = { + * 'one': {'en-GB': 'one!', es: 'uno'}, + * 'two': {es: 'dos'} + * }; + * translationsForLocaleV2(multiLocaleStrings, ['en-CA', 'en-GB', 'en']); + * // => + * // { + * // "one": "one!", + * // "two": "two", + * // } + * ``` + */ +export function translationsForLocaleV2( + multiLocaleStrings: MultiLocaleTranslationsMap, + fallbackLocales: Locale[] +): SingleLocaleTranslationsMap { + const singleLocaleStrings: SingleLocaleTranslationsMap = {}; + Object.entries(multiLocaleStrings).forEach(([source, translations]) => { + let translation = source; + for (const locale of fallbackLocales) { + if (translations[locale]) { + translation = translations[locale]; + break; + } + } + singleLocaleStrings[source] = translation; + }); + return singleLocaleStrings; +} + export function buildTranslationsDbPath(options: TranslationsDbPathOptions) { return TRANSLATIONS_DB_PATH_FORMAT.replace( '{project}', @@ -479,3 +697,14 @@ export function buildTranslationsLocaleDocDbPath( .replace('{id}', normalizeSlug(options.id)) .replace('{locale}', options.locale); } + +/** + * Splits an array into chunks of (up to) a given size. + */ +export function chunkArray(items: T[], chunkSize: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; +} diff --git a/packages/root-cms/core/translations-migration.test.ts b/packages/root-cms/core/translations-migration.test.ts new file mode 100644 index 000000000..a748e67ee --- /dev/null +++ b/packages/root-cms/core/translations-migration.test.ts @@ -0,0 +1,210 @@ +/** + * Tests for the v1 -> v2 translations auto-migration, against the Firestore + * emulator (see translations-manager.test.ts for details on the emulator + * setup). + */ + +import crypto from 'node:crypto'; +import {getApps, initializeApp} from 'firebase-admin/app'; +import {Timestamp, getFirestore} from 'firebase-admin/firestore'; +import {beforeEach, describe, expect, it} from 'vitest'; +import {RootCMSClient} from './client.js'; +import { + TranslationsMigrationState, + migrateV1TranslationsIfNeeded, +} from './translations-migration.js'; + +const FIREBASE_PROJECT_ID = 'rootjs-cms-admin-tests'; + +function getTestApp() { + const existing = getApps().find((app) => app.name === 'migration-test'); + if (existing) { + return existing; + } + return initializeApp({projectId: FIREBASE_PROJECT_ID}, 'migration-test'); +} + +let projectCounter = 0; + +function createTestCmsClient(): RootCMSClient { + const app = getTestApp(); + const db = getFirestore(app); + const projectId = `migration-test-${Date.now()}-${projectCounter++}`; + const plugin = { + name: 'root-cms', + getConfig: () => ({ + id: projectId, + firebaseConfig: { + apiKey: 'test', + authDomain: 'test', + projectId: FIREBASE_PROJECT_ID, + storageBucket: 'test', + }, + experiments: {v2TranslationsManager: true}, + }), + getFirebaseApp: () => app, + getFirestore: () => db, + }; + const rootConfig: any = { + rootDir: '/test', + i18n: {locales: ['en', 'es']}, + plugins: [plugin], + }; + return new RootCMSClient(rootConfig); +} + +function sha1(str: string) { + return crypto.createHash('sha1').update(str).digest('hex'); +} + +async function seedV1Translation( + cmsClient: RootCMSClient, + source: string, + translations: Record, + tags?: string[] +) { + const data: Record = {...translations, source}; + if (tags) { + data.tags = tags; + } + await cmsClient.db + .doc(`Projects/${cmsClient.projectId}/Translations/${sha1(source)}`) + .set(data); +} + +function migrationStateRef(cmsClient: RootCMSClient) { + return cmsClient.db.doc( + `Projects/${cmsClient.projectId}/TranslationsManager/migration` + ); +} + +describe.skipIf(!process.env.FIRESTORE_EMULATOR_HOST)( + 'migrateV1TranslationsIfNeeded', + () => { + let cmsClient: RootCMSClient; + + beforeEach(() => { + cmsClient = createTestCmsClient(); + }); + + it('migrates and publishes v1 translations', async () => { + await seedV1Translation(cmsClient, 'hello', {es: 'hola'}, [ + 'common', + 'Pages/index', + ]); + + const res = await migrateV1TranslationsIfNeeded(cmsClient, { + trigger: 'dev', + }); + expect(res).toEqual({status: 'complete', skipped: false}); + + // v1 translations were live, so the migrated data is published. + const tm = cmsClient.getTranslationsManager(); + const published = await tm.loadTranslations({ + ids: ['common', 'Pages/index'], + mode: 'published', + }); + expect(published.hello.es).toBe('hola'); + + // The migration state doc records completion. + const state = ( + await migrationStateRef(cmsClient).get() + ).data() as TranslationsMigrationState; + expect(state.status).toBe('complete'); + expect(state.version).toBe(1); + expect(state.stats).toEqual({numStrings: 1, numDocs: 2}); + + // The v1 data is left untouched as a backup. + const v1Doc = await cmsClient.db + .doc(`Projects/${cmsClient.projectId}/Translations/${sha1('hello')}`) + .get(); + expect(v1Doc.exists).toBe(true); + }); + + it('skips when the migration already completed', async () => { + await seedV1Translation(cmsClient, 'hello', {es: 'hola'}, ['common']); + const first = await migrateV1TranslationsIfNeeded(cmsClient); + expect(first.skipped).toBe(false); + + // Add a new v1 string; a second run should not import it. + await seedV1Translation(cmsClient, 'bye', {es: 'adios'}, ['common']); + const second = await migrateV1TranslationsIfNeeded(cmsClient); + expect(second).toEqual({status: 'complete', skipped: true}); + + const tm = cmsClient.getTranslationsManager(); + const published = await tm.loadTranslations({ + ids: ['common'], + mode: 'published', + }); + expect(published.bye).toBeUndefined(); + }); + + it('completes with empty stats when there is nothing to migrate', async () => { + const res = await migrateV1TranslationsIfNeeded(cmsClient); + expect(res).toEqual({status: 'complete', skipped: false}); + const state = ( + await migrationStateRef(cmsClient).get() + ).data() as TranslationsMigrationState; + expect(state.stats).toEqual({numStrings: 0, numDocs: 0}); + }); + + it('skips when another process holds a fresh running lock', async () => { + await seedV1Translation(cmsClient, 'hello', {es: 'hola'}, ['common']); + await migrationStateRef(cmsClient).set({ + version: 1, + status: 'running', + startedAt: Timestamp.now(), + startedBy: 'other-process', + }); + + const res = await migrateV1TranslationsIfNeeded(cmsClient); + expect(res).toEqual({status: 'running', skipped: true}); + + const tm = cmsClient.getTranslationsManager(); + const published = await tm.loadTranslations({ + ids: ['common'], + mode: 'published', + }); + expect(published).toEqual({}); + }); + + it('takes over a stale running lock', async () => { + await seedV1Translation(cmsClient, 'hello', {es: 'hola'}, ['common']); + const staleMillis = Date.now() - 11 * 60 * 1000; + await migrationStateRef(cmsClient).set({ + version: 1, + status: 'running', + startedAt: Timestamp.fromMillis(staleMillis), + startedBy: 'crashed-process', + }); + + const res = await migrateV1TranslationsIfNeeded(cmsClient); + expect(res).toEqual({status: 'complete', skipped: false}); + + const tm = cmsClient.getTranslationsManager(); + const published = await tm.loadTranslations({ + ids: ['common'], + mode: 'published', + }); + expect(published.hello.es).toBe('hola'); + }); + + it('re-runs after a failed migration', async () => { + await seedV1Translation(cmsClient, 'hello', {es: 'hola'}, ['common']); + await migrationStateRef(cmsClient).set({ + version: 1, + status: 'error', + error: 'something went wrong', + startedAt: Timestamp.now(), + }); + + const res = await migrateV1TranslationsIfNeeded(cmsClient); + expect(res).toEqual({status: 'complete', skipped: false}); + const state = ( + await migrationStateRef(cmsClient).get() + ).data() as TranslationsMigrationState; + expect(state.status).toBe('complete'); + expect(state.error).toBeUndefined(); + }); + } +); diff --git a/packages/root-cms/core/translations-migration.ts b/packages/root-cms/core/translations-migration.ts new file mode 100644 index 000000000..793969667 --- /dev/null +++ b/packages/root-cms/core/translations-migration.ts @@ -0,0 +1,159 @@ +/** + * Auto-migration of v1 translations to the v2 TranslationsManager. + * + * When the `experiments.v2TranslationsManager` flag is enabled, the CMS + * plugin runs `migrateV1TranslationsIfNeeded()` on dev server startup and + * before prod builds. The migration copies the v1 translations + * (`Projects/{p}/Translations`) into per-locale v2 docs and publishes them + * (v1 translations were live as soon as they were saved, so the migrated + * data must be live too). The v1 data is left untouched as a backup. + * + * Migration state is tracked in a + * `Projects/{p}/TranslationsManager/migration` doc (a sibling of the + * `draft`/`published` container docs) so the migration only runs once. A + * Firestore transaction guards against concurrent runs; a stale `running` + * lock expires after 10 minutes. + */ + +import {Timestamp} from 'firebase-admin/firestore'; +import type {RootCMSClient} from './client.js'; + +/** + * Bump this version to force the migration to re-run on projects that have + * already completed an older version of the migration. + */ +const MIGRATION_VERSION = 1; + +/** A `running` migration lock older than this is considered stale. */ +const STALE_LOCK_TIMEOUT_MS = 10 * 60 * 1000; + +export type TranslationsMigrationStatus = 'running' | 'complete' | 'error'; + +export interface TranslationsMigrationState { + version: number; + status: TranslationsMigrationStatus; + startedAt?: Timestamp; + startedBy?: string; + finishedAt?: Timestamp; + error?: string; + stats?: { + numStrings: number; + numDocs: number; + }; +} + +export interface MigrateV1TranslationsOptions { + /** What triggered the migration (used for logging/state). */ + trigger?: 'build' | 'dev'; +} + +export interface MigrateV1TranslationsResult { + status: TranslationsMigrationStatus; + /** + * True if the migration was skipped, either because it already completed + * or because another process holds the `running` lock. + */ + skipped: boolean; +} + +function migrationStateDbPath(projectId: string) { + return `Projects/${projectId}/TranslationsManager/migration`; +} + +/** + * Migrates v1 translations to the v2 TranslationsManager if the migration + * hasn't already run for this project. Safe to call on every dev server boot + * and build: after the first successful run, this costs a single Firestore + * read. + */ +export async function migrateV1TranslationsIfNeeded( + cmsClient: RootCMSClient, + options?: MigrateV1TranslationsOptions +): Promise { + const trigger = options?.trigger || 'build'; + const db = cmsClient.db; + const stateRef = db.doc(migrationStateDbPath(cmsClient.projectId)); + + // Fast path: a single read per boot. `complete` is the "migrated" tag. + const snapshot = await stateRef.get(); + const state = snapshot.data() as TranslationsMigrationState | undefined; + if (state?.status === 'complete' && state.version >= MIGRATION_VERSION) { + return {status: 'complete', skipped: true}; + } + + const startedAt = Timestamp.now(); + const startedBy = `root-cms (${trigger})`; + + // Claim the migration via a transaction so that only one process runs it + // (e.g. multiple devs booting dev servers at the same time). + const claimed = await db.runTransaction(async (tx) => { + const snapshot = await tx.get(stateRef); + const state = snapshot.data() as TranslationsMigrationState | undefined; + if (state?.status === 'complete' && state.version >= MIGRATION_VERSION) { + return false; + } + if (state?.status === 'running') { + const startedAt = state.startedAt?.toMillis?.() || 0; + if (Date.now() - startedAt < STALE_LOCK_TIMEOUT_MS) { + return false; + } + console.warn( + '[root cms] taking over stale translations migration lock ' + + `(started ${new Date(startedAt).toISOString()})` + ); + } + const runningState: TranslationsMigrationState = { + version: MIGRATION_VERSION, + status: 'running', + startedAt: startedAt, + startedBy: startedBy, + }; + tx.set(stateRef, runningState); + return true; + }); + if (!claimed) { + return {status: state?.status || 'running', skipped: true}; + } + + try { + const tm = cmsClient.getTranslationsManager(); + const res = await tm.importTranslationsFromV1(); + // v1 translations were live as soon as they were saved, so the migrated + // translations must be published for the site to keep serving them. + if (res.ids.length > 0) { + await tm.publishTranslationsBulk(res.ids, { + publishedBy: `root-cms migration (${trigger})`, + }); + } + const completeState: TranslationsMigrationState = { + version: MIGRATION_VERSION, + status: 'complete', + startedAt: startedAt, + startedBy: startedBy, + finishedAt: Timestamp.now(), + stats: res.stats, + }; + await stateRef.set(completeState); + if (res.ids.length > 0) { + console.log( + `[root cms] migrated ${res.stats.numStrings} v1 translation(s) into ` + + `${res.stats.numDocs} translations doc(s)` + ); + } + return {status: 'complete', skipped: false}; + } catch (err) { + const errorState: Partial = { + status: 'error', + finishedAt: Timestamp.now(), + error: String(err), + }; + // Best effort: release the lock so a subsequent boot can retry. + await stateRef.set(errorState, {merge: true}).catch((stateErr) => { + console.error( + '[root cms] failed to save translations migration error state:', + stateErr + ); + }); + throw err; + } +} diff --git a/packages/root-cms/shared/locale-fallbacks.test.ts b/packages/root-cms/shared/locale-fallbacks.test.ts new file mode 100644 index 000000000..76882cca0 --- /dev/null +++ b/packages/root-cms/shared/locale-fallbacks.test.ts @@ -0,0 +1,101 @@ +import {describe, expect, it} from 'vitest'; +import {resolveLocaleFallbacks} from './locale-fallbacks.js'; + +describe('resolveLocaleFallbacks', () => { + it('returns the locale followed by the default locale', () => { + expect(resolveLocaleFallbacks({}, 'fr')).toEqual(['fr', 'en']); + expect(resolveLocaleFallbacks(undefined, 'fr')).toEqual(['fr', 'en']); + }); + + it('uses the configured defaultLocale', () => { + expect(resolveLocaleFallbacks({defaultLocale: 'de'}, 'fr')).toEqual([ + 'fr', + 'de', + ]); + }); + + it('does not duplicate the default locale', () => { + expect(resolveLocaleFallbacks({}, 'en')).toEqual(['en']); + expect(resolveLocaleFallbacks({defaultLocale: 'de'}, 'de')).toEqual(['de']); + }); + + it('follows configured fallback chains', () => { + const i18n = {fallbacks: {'en-CA': ['en-GB']}}; + expect(resolveLocaleFallbacks(i18n, 'en-CA')).toEqual([ + 'en-CA', + 'en-GB', + 'en', + ]); + }); + + it('resolves fallbacks recursively (breadth first)', () => { + const i18n = { + fallbacks: { + 'fr-ca': ['fr-fr', 'en-CA'], + 'fr-fr': ['fr'], + 'en-CA': ['en-GB'], + }, + }; + expect(resolveLocaleFallbacks(i18n, 'fr-ca')).toEqual([ + 'fr-ca', + 'fr-fr', + 'en-CA', + 'fr', + 'en-GB', + 'en', + ]); + }); + + it('matches fallback keys case-insensitively', () => { + const i18n = {fallbacks: {'EN-ca': ['en-GB']}}; + expect(resolveLocaleFallbacks(i18n, 'en-CA')).toEqual([ + 'en-CA', + 'en-GB', + 'en', + ]); + }); + + it('preserves the configured casing of fallback values', () => { + const i18n = {fallbacks: {'en-ca': ['EN-gb']}}; + expect(resolveLocaleFallbacks(i18n, 'en-CA')).toEqual([ + 'en-CA', + 'EN-gb', + 'en', + ]); + }); + + it('protects against cycles', () => { + const i18n = { + fallbacks: { + a: ['b'], + b: ['c'], + c: ['a'], + }, + }; + expect(resolveLocaleFallbacks(i18n, 'a')).toEqual(['a', 'b', 'c', 'en']); + }); + + it('protects against self-referencing fallbacks', () => { + const i18n = {fallbacks: {'en-ca': ['en-CA', 'en-GB']}}; + expect(resolveLocaleFallbacks(i18n, 'en-CA')).toEqual([ + 'en-CA', + 'en-GB', + 'en', + ]); + }); + + it('dedupes shared fallbacks across chains', () => { + const i18n = { + fallbacks: { + 'es-mx': ['es-419', 'es'], + 'es-419': ['es'], + }, + }; + expect(resolveLocaleFallbacks(i18n, 'es-mx')).toEqual([ + 'es-mx', + 'es-419', + 'es', + 'en', + ]); + }); +}); diff --git a/packages/root-cms/shared/locale-fallbacks.ts b/packages/root-cms/shared/locale-fallbacks.ts new file mode 100644 index 000000000..c37672839 --- /dev/null +++ b/packages/root-cms/shared/locale-fallbacks.ts @@ -0,0 +1,71 @@ +/** + * Utilities for resolving the locale fallback chain used by translations. + * + * The `i18n.fallbacks` config in root.config.ts maps a locale to an ordered + * list of fallback locales. When a translation is missing for a locale, each + * fallback locale is checked (in order) before falling back to + * `i18n.defaultLocale` and finally the source string. For example: + * + * ```ts + * i18n: { + * locales: ['en', 'en-GB', 'en-CA'], + * fallbacks: { + * 'en-CA': ['en-GB'], + * }, + * } + * ``` + * + * With the config above, `resolveLocaleFallbacks(i18n, 'en-CA')` returns + * `['en-CA', 'en-GB', 'en']`. Fallbacks are resolved recursively (breadth + * first), and all matching is case-insensitive. + */ + +export interface LocaleFallbacksI18nConfig { + locales?: string[]; + defaultLocale?: string; + fallbacks?: Record; +} + +/** + * Resolves the ordered locale fallback chain for a locale, starting with the + * locale itself and ending with the default locale. Fallback chains are + * followed recursively (breadth first) with cycle protection, and locale keys + * are matched case-insensitively while preserving the configured casing of + * each fallback value. + */ +export function resolveLocaleFallbacks( + i18nConfig: LocaleFallbacksI18nConfig | undefined, + locale: string +): string[] { + const fallbacks = i18nConfig?.fallbacks || {}; + // Normalize the fallback keys for case-insensitive lookups. + const fallbacksByLowerKey: Record = {}; + for (const [key, values] of Object.entries(fallbacks)) { + fallbacksByLowerKey[key.toLowerCase()] = values || []; + } + + const chain: string[] = []; + const visited = new Set(); + const queue: string[] = [locale]; + while (queue.length > 0) { + const current = queue.shift()!; + const lower = String(current).toLowerCase(); + if (visited.has(lower)) { + continue; + } + visited.add(lower); + chain.push(current); + for (const fallback of fallbacksByLowerKey[lower] || []) { + if (!visited.has(String(fallback).toLowerCase())) { + queue.push(fallback); + } + } + } + + // Always fall back to the default locale last. + const defaultLocale = i18nConfig?.defaultLocale || 'en'; + if (!visited.has(defaultLocale.toLowerCase())) { + chain.push(defaultLocale); + } + return chain; +} diff --git a/packages/root-cms/ui/components/DocEditor/DocEditor.tsx b/packages/root-cms/ui/components/DocEditor/DocEditor.tsx index 9a2e1a19a..33dd43342 100644 --- a/packages/root-cms/ui/components/DocEditor/DocEditor.tsx +++ b/packages/root-cms/ui/components/DocEditor/DocEditor.tsx @@ -92,6 +92,7 @@ import {autokey} from '../../utils/rand.js'; import {buildPreviewValue} from '../../utils/schema-previews.js'; import {testFieldEmpty} from '../../utils/test-field-empty.js'; import {formatDateTime} from '../../utils/time.js'; +import {testV2TranslationsEnabled} from '../../utils/translations-manager.js'; import {useAiEditModal} from '../AiEditModal/AiEditModal.js'; import { ComponentPickerOption, @@ -423,6 +424,12 @@ DocEditor.StatusBar = (props: StatusBarProps) => { size="xs" leftIcon={} onClick={() => { + // When the v2 translations manager is enabled, the doc's + // translations are edited in the translations manager. + if (testV2TranslationsEnabled()) { + route(`/cms/translations/${props.docId}`); + return; + } localizationModal.open({ docId: props.docId, collection: props.collection, diff --git a/packages/root-cms/ui/components/PublishDocModal/PublishDocModal.tsx b/packages/root-cms/ui/components/PublishDocModal/PublishDocModal.tsx index 1325439c4..95cd46418 100644 --- a/packages/root-cms/ui/components/PublishDocModal/PublishDocModal.tsx +++ b/packages/root-cms/ui/components/PublishDocModal/PublishDocModal.tsx @@ -15,6 +15,7 @@ import {errorMessage} from '../../utils/notifications.js'; import {testCanPublish} from '../../utils/permissions.js'; import {extractReferenceDocIds} from '../../utils/references.js'; import {getLocalISOString} from '../../utils/time.js'; +import {testV2TranslationsEnabled} from '../../utils/translations-manager.js'; import {useAddToReleaseModal} from '../AddToReleaseModal/AddToReleaseModal.js'; import {AiSummary} from '../AiSummary/AiSummary.js'; import {DocDiffViewer} from '../DocDiffViewer/DocDiffViewer.js'; @@ -272,6 +273,14 @@ export function PublishDocModal( + {testV2TranslationsEnabled() && ( +
+ + The doc's translations will be published together with the doc. + +
+ )} +
+ ); +}; + +TranslationsManagerEditPage.AddStringInput = (props: { + onAdd: (source: string) => void; +}) => { + const [value, setValue] = useState(''); + + function submit() { + if (!value.trim()) { + return; + } + props.onAdd(value); + setValue(''); + } + + return ( +
+ { + setValue((e.target as HTMLInputElement).value); + }} + onKeyDown={(e: KeyboardEvent) => { + if (e.key === 'Enter') { + e.preventDefault(); + submit(); + } + }} + /> + +
+ ); +}; diff --git a/packages/root-cms/ui/pages/TranslationsManagerPage/TranslationsManagerPage.css b/packages/root-cms/ui/pages/TranslationsManagerPage/TranslationsManagerPage.css new file mode 100644 index 000000000..4a0c052b8 --- /dev/null +++ b/packages/root-cms/ui/pages/TranslationsManagerPage/TranslationsManagerPage.css @@ -0,0 +1,174 @@ +.TranslationsManagerPage { + --side-width: 280px; + background: var(--page-background); +} + +.TranslationsManagerPage__layout { + position: relative; + min-height: calc(100vh - var(--header-height)); +} + +.TranslationsManagerPage__side { + position: absolute; + top: 0; + left: 0; + bottom: 0; + overflow: auto; + width: var(--side-width); + z-index: 10; + padding-top: var(--page-padding); + padding-left: var(--page-padding); + padding-bottom: var(--page-padding); +} + +.TranslationsManagerPage__side__title { + font-size: 14px; + line-height: 1.2; + font-weight: 600; + margin-bottom: 12px; +} + +.TranslationsManagerPage__side__groups { + display: flex; + flex-direction: column; + gap: 2px; +} + +.TranslationsManagerPage__side__group { + display: flex; + align-items: center; + gap: 8px; + padding: 6px 10px; + border: none; + border-radius: 6px; + background: transparent; + font-size: 13px; + line-height: 1.2; + text-align: left; + cursor: pointer; +} + +.TranslationsManagerPage__side__group:hover { + background: rgba(0, 0, 0, 0.05); +} + +.TranslationsManagerPage__side__group.active { + background: rgba(0, 0, 0, 0.08); + font-weight: 600; +} + +.TranslationsManagerPage__side__group__label { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.TranslationsManagerPage__side__group__count { + font-size: 11px; + color: #868e96; +} + +.TranslationsManagerPage__main { + padding: var(--page-padding); + padding-left: calc(var(--side-width) + var(--page-padding)); +} + +.TranslationsManagerPage__main__header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 20px; + margin-bottom: 16px; +} + +.TranslationsManagerPage__main__header__controls { + display: flex; + align-items: center; + gap: 10px; +} + +.TranslationsManagerPage__main__header__search { + width: 240px; +} + +.TranslationsManagerPage__main__content { + padding: 8px 0; +} + +.TranslationsManagerPage__loading, +.TranslationsManagerPage__empty { + display: flex; + align-items: center; + justify-content: center; + padding: 60px 20px; + font-size: 14px; + color: #868e96; +} + +.TranslationsManagerPage__docsList__header, +.TranslationsManagerPage__docsList__doc { + display: grid; + grid-template-columns: minmax(200px, 1fr) 90px 170px 140px 140px 44px; + align-items: center; + gap: 12px; + padding: 8px 20px; +} + +.TranslationsManagerPage__docsList__header { + font-size: 11px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + color: #868e96; + border-bottom: 1px solid #e9ecef; +} + +.TranslationsManagerPage__docsList__doc { + border-bottom: 1px solid #f1f3f5; + font-size: 13px; +} + +.TranslationsManagerPage__docsList__doc:last-child { + border-bottom: none; +} + +.TranslationsManagerPage__docsList__doc__id { + font-weight: 600; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + text-decoration: none; +} + +.TranslationsManagerPage__docsList__doc__id:hover { + text-decoration: underline; + text-underline-offset: 4px; +} + +.TranslationsManagerPage__docsList__doc__locales { + font-size: 12px; + color: #495057; +} + +.TranslationsManagerPage__docsList__doc__timestamp { + font-size: 12px; + color: #495057; +} + +.TranslationsManagerPage__docsList__doc__controls { + display: flex; + justify-content: flex-end; +} + +.TranslationsManagerPage__confirmText { + font-size: 14px; + line-height: 1.5; +} + +.TranslationsManagerPage__confirmText code { + background: #f1f3f5; + border-radius: 4px; + padding: 1px 5px; + font-size: 12px; +} diff --git a/packages/root-cms/ui/pages/TranslationsManagerPage/TranslationsManagerPage.tsx b/packages/root-cms/ui/pages/TranslationsManagerPage/TranslationsManagerPage.tsx new file mode 100644 index 000000000..59e65fbcb --- /dev/null +++ b/packages/root-cms/ui/pages/TranslationsManagerPage/TranslationsManagerPage.tsx @@ -0,0 +1,381 @@ +import './TranslationsManagerPage.css'; + +import { + ActionIcon, + Badge, + Loader, + Menu, + TextInput, + Tooltip, +} from '@mantine/core'; +import {useModals} from '@mantine/modals'; +import {showNotification} from '@mantine/notifications'; +import { + IconDotsVertical, + IconFolder, + IconLanguage, + IconPencil, + IconRocket, + IconSearch, + IconTrash, + IconWorld, +} from '@tabler/icons-preact'; +import {useEffect, useMemo, useState} from 'preact/hooks'; +import {useLocation} from 'preact-iso'; +import {Heading} from '../../components/Heading/Heading.js'; +import {Surface} from '../../components/Surface/Surface.js'; +import {UserActionTooltip} from '../../components/UserActionTooltip/UserActionTooltip.js'; +import {useModalTheme} from '../../hooks/useModalTheme.js'; +import {usePageTitle} from '../../hooks/usePageTitle.js'; +import {Layout} from '../../layout/Layout.js'; +import {joinClassNames} from '../../utils/classes.js'; +import {notifyErrors} from '../../utils/notifications.js'; +import {formatDateTime, getTimeAgo} from '../../utils/time.js'; +import { + TranslationsDocSummary, + deleteTranslationsDoc, + listTranslationsDocs, + publishTranslations, +} from '../../utils/translations-manager.js'; + +/** Group used for translations ids without a `/` prefix, e.g. `common`. */ +const GLOBAL_GROUP = 'Global'; + +function getGroup(id: string): string { + const sepIndex = id.indexOf('/'); + if (sepIndex <= 0) { + return GLOBAL_GROUP; + } + return id.slice(0, sepIndex); +} + +export function TranslationsManagerPage() { + usePageTitle('Translations'); + const [loading, setLoading] = useState(true); + const [docs, setDocs] = useState([]); + const [selectedGroup, setSelectedGroup] = useState(null); + const [searchQuery, setSearchQuery] = useState(''); + + async function reload() { + await notifyErrors(async () => { + const docs = await listTranslationsDocs(); + setDocs(docs); + }); + setLoading(false); + } + + useEffect(() => { + reload(); + }, []); + + const groups = useMemo(() => { + const byGroup: Record = {}; + docs.forEach((doc) => { + const group = getGroup(doc.id); + byGroup[group] = (byGroup[group] || 0) + 1; + }); + return Object.entries(byGroup) + .map(([name, count]) => ({name, count})) + .sort((a, b) => a.name.localeCompare(b.name)); + }, [docs]); + + const filteredDocs = useMemo(() => { + let filtered = docs; + if (selectedGroup) { + filtered = filtered.filter((doc) => getGroup(doc.id) === selectedGroup); + } + const query = searchQuery.trim().toLowerCase(); + if (query) { + filtered = filtered.filter((doc) => doc.id.toLowerCase().includes(query)); + } + return filtered; + }, [docs, selectedGroup, searchQuery]); + + return ( + +
+
+
+
+ Translations +
+
+ + {groups.map((group) => ( + + ))} +
+
+
+
+ + {selectedGroup + ? `Translations: ${selectedGroup}` + : 'Translations'} + +
+ } + size="xs" + value={searchQuery} + onChange={(e: Event) => { + setSearchQuery((e.target as HTMLInputElement).value); + }} + /> +
+
+ + {loading ? ( +
+ +
+ ) : filteredDocs.length === 0 ? ( +
+ {docs.length === 0 + ? 'No translations yet. Translations saved from a doc appear here.' + : 'No translations match your filters.'} +
+ ) : ( + reload()} + /> + )} + +
+
+
+ + ); +} + +TranslationsManagerPage.DocsList = (props: { + docs: TranslationsDocSummary[]; + reload: () => void; +}) => { + return ( +
+
+
+ ID +
+
+ Locales +
+
+ Status +
+
+ Modified +
+
+ Published +
+
+
+ {props.docs.map((doc) => ( + + ))} +
+ ); +}; + +TranslationsManagerPage.DocRow = (props: { + doc: TranslationsDocSummary; + reload: () => void; +}) => { + const doc = props.doc; + const {route} = useLocation(); + const modals = useModals(); + const modalTheme = useModalTheme(); + const editUrl = `/cms/translations/${doc.id}`; + + function onPublish() { + modals.openConfirmModal({ + ...modalTheme, + title: `Publish translations: ${doc.id}`, + children: ( +
+ Are you sure you want to publish the translations for{' '} + {doc.id}? The translations will go live immediately. +
+ ), + labels: {confirm: 'Publish', cancel: 'Cancel'}, + cancelProps: {size: 'xs'}, + confirmProps: {color: 'dark', size: 'xs'}, + onConfirm: async () => { + await notifyErrors(async () => { + await publishTranslations(doc.id); + showNotification({ + title: 'Published translations', + message: `Published translations for ${doc.id}.`, + autoClose: 10000, + }); + props.reload(); + }); + }, + }); + } + + function onDelete() { + modals.openConfirmModal({ + ...modalTheme, + title: `Delete translations: ${doc.id}`, + children: ( +
+ Are you sure you want to delete the translations for{' '} + {doc.id}? This deletes both the draft and published + translations and cannot be undone. +
+ ), + labels: {confirm: 'Delete', cancel: 'Cancel'}, + cancelProps: {size: 'xs'}, + confirmProps: {color: 'red', size: 'xs'}, + onConfirm: async () => { + await notifyErrors(async () => { + await deleteTranslationsDoc(doc.id); + showNotification({ + title: 'Deleted translations', + message: `Deleted translations for ${doc.id}.`, + autoClose: 10000, + }); + props.reload(); + }); + }, + }); + } + + return ( +
+ + {doc.id} + +
+ + {doc.locales.length} locale{doc.locales.length === 1 ? '' : 's'} + +
+
+ {doc.publishedAt ? ( + doc.hasUnpublishedChanges ? ( + + Unpublished changes + + ) : ( + + Published + + ) + ) : ( + + Draft + + )} +
+
+ {doc.modifiedAt ? ( + + {getTimeAgo(doc.modifiedAt.toMillis())} + + ) : ( + '—' + )} +
+
+ {doc.publishedAt ? ( + + {getTimeAgo(doc.publishedAt.toMillis())} + + ) : ( + '—' + )} +
+
+ + + + } + > + } + onClick={() => route(editUrl)} + > + Edit + + } onClick={onPublish}> + Publish… + + } + color="red" + onClick={onDelete} + > + Delete… + + +
+
+ ); +}; diff --git a/packages/root-cms/ui/ui.tsx b/packages/root-cms/ui/ui.tsx index 3e6637d43..6db055e8d 100644 --- a/packages/root-cms/ui/ui.tsx +++ b/packages/root-cms/ui/ui.tsx @@ -143,6 +143,16 @@ const TranslationsArbPage = lazyRoute(() => (m) => m.TranslationsArbPage ) ); +const TranslationsManagerEditPage = lazyRoute(() => + import('./pages/TranslationsManagerEditPage/TranslationsManagerEditPage.js').then( + (m) => m.TranslationsManagerEditPage + ) +); +const TranslationsManagerPage = lazyRoute(() => + import('./pages/TranslationsManagerPage/TranslationsManagerPage.js').then( + (m) => m.TranslationsManagerPage + ) +); const TranslationsEditPage = lazyRoute(() => import('./pages/TranslationsEditPage/TranslationsEditPage.js').then( (m) => m.TranslationsEditPage @@ -168,9 +178,11 @@ declare global { gci: string | boolean; i18n: { locales?: string[]; + defaultLocale?: string; urlFormat?: string; groups?: Record; translationLanguages?: Record; + fallbacks?: Record; }; server: { trailingSlash?: boolean; @@ -198,6 +210,7 @@ declare global { experiments?: { ai?: boolean | {endpoint?: string}; taskManager?: boolean; + v2TranslationsManager?: boolean; }; ai?: { defaultModel?: string; @@ -251,6 +264,46 @@ declare global { } function App() { + const v2TranslationsEnabled = Boolean( + window.__ROOT_CTX.experiments?.v2TranslationsManager + ); + // NOTE: conditional route children must be arrays (not fragments) — + // preact-iso's Router only flattens arrays. + const translationsRoutes = v2TranslationsEnabled + ? [ + , + , + ] + : [ + , + , + , + , + ]; return ( - - - - + {translationsRoutes} diff --git a/packages/root-cms/ui/utils/doc.ts b/packages/root-cms/ui/utils/doc.ts index 4b40c9e0d..9884493d9 100644 --- a/packages/root-cms/ui/utils/doc.ts +++ b/packages/root-cms/ui/utils/doc.ts @@ -35,6 +35,15 @@ import { normalizeString, sourceHash, } from './l10n.js'; +import { + TranslationsLocaleDocWithRef, + addDeleteTranslationsOpsToBatch, + addPublishTranslationsOpsToBatch, + addUnpublishTranslationsOpsToBatch, + getDraftTranslationsLocaleDocs, + getTranslationsLocaleDocs, + testV2TranslationsEnabled, +} from './translations-manager.js'; export interface CMSDoc { id: string; @@ -122,6 +131,17 @@ export async function cmsDeleteDoc(docId: string) { batch.delete(publishedDocRef); // Delete any scheduled doc. batch.delete(scheduledDocRef); + // Delete the doc's translations (draft and published) in the same batch. + if (testV2TranslationsEnabled()) { + const [draftTranslations, publishedTranslations] = await Promise.all([ + getTranslationsLocaleDocs([docId], 'draft'), + getTranslationsLocaleDocs([docId], 'published'), + ]); + addDeleteTranslationsOpsToBatch(batch, [ + ...(draftTranslations[docId] || []), + ...(publishedTranslations[docId] || []), + ]); + } await batch.commit(); console.log(`deleted doc: ${docId}`); logAction('doc.delete', {metadata: {docId}}); @@ -155,24 +175,62 @@ export async function cmsPublishDocs( } const draftDocs = await getDraftDocs(docIds); - const batch = options?.batch || writeBatch(db); - const versionTags = ['published']; - if (options?.releaseId) { - versionTags.push(`release:${options.releaseId}`); - } + // Validate all docs upfront so that a failure can't leave a partially + // committed publish (commits below may be chunked). docIds.forEach((docId) => { const draftData = draftDocs[docId]; if (!draftData) { throw new Error(`doc does not exist: ${docId}`); } + if (testPublishingLocked(draftData)) { + throw new Error(`publishing is locked for doc: ${draftData.id}`); + } + if (testIsArchived(draftData)) { + throw new Error(`cannot publish archived doc: ${draftData.id}`); + } + }); + + // If the v2 translations manager is enabled, prefetch each doc's draft + // translations locale docs so the translations are published in the same + // write batch as the doc. + const v2TranslationsEnabled = testV2TranslationsEnabled(); + const translationsByDocId: Record = + v2TranslationsEnabled ? await getDraftTranslationsLocaleDocs(docIds) : {}; + + const versionTags = ['published']; + if (options?.releaseId) { + versionTags.push(`release:${options.releaseId}`); + } + + // Firestore batches allow a max of 500 write ops. Each doc adds 4 ops plus + // 2 ops per translations locale doc, so commit in op-counted chunks, + // keeping each doc and its translations within a single chunk. + const MAX_OPS_PER_BATCH = 450; + let batch = options?.batch || writeBatch(db); + let opCount = 0; + for (const docId of docIds) { + const translationsLocaleDocs = translationsByDocId[docId] || []; + const docOps = 4 + 2 * translationsLocaleDocs.length; + if (opCount > 0 && opCount + docOps > MAX_OPS_PER_BATCH) { + await batch.commit(); + batch = writeBatch(db); + opCount = 0; + } updatePublishedDocDataInBatch( batch, docId, - draftData, + draftDocs[docId], versionTags, options?.publishMessage ); - }); + opCount += 4; + if (v2TranslationsEnabled) { + opCount += addPublishTranslationsOpsToBatch( + batch, + translationsLocaleDocs + ); + } + } await batch.commit(); if (docIds.length === 1) { @@ -397,6 +455,18 @@ export async function cmsUnpublishDoc(docId: string) { batch.delete(scheduledDocRef); // Delete the "published" doc. batch.delete(publishedDocRef); + // Unpublish the doc's translations in the same batch. + if (testV2TranslationsEnabled()) { + const [draftTranslations, publishedTranslations] = await Promise.all([ + getTranslationsLocaleDocs([docId], 'draft'), + getTranslationsLocaleDocs([docId], 'published'), + ]); + addUnpublishTranslationsOpsToBatch( + batch, + draftTranslations[docId] || [], + publishedTranslations[docId] || [] + ); + } await batch.commit(); console.log(`unpublished ${docId}`); logAction('doc.unpublish', {metadata: {docId}}); diff --git a/packages/root-cms/ui/utils/translations-manager.ts b/packages/root-cms/ui/utils/translations-manager.ts new file mode 100644 index 000000000..8e314eba3 --- /dev/null +++ b/packages/root-cms/ui/utils/translations-manager.ts @@ -0,0 +1,477 @@ +/** + * Browser-side data layer for the v2 TranslationsManager (mirrors the + * Firestore paths used by `core/translations-manager.ts`). + * + * Translations are stored in per-locale docs at: + * ``` + * /Projects/{p}/TranslationsManager/{draft|published}/Translations/{normalizeSlug(id)}:{locale} + * ``` + */ + +import { + DocumentReference, + Timestamp, + WriteBatch, + arrayUnion, + collection, + deleteField, + doc, + getDocs, + query, + where, + writeBatch, +} from 'firebase/firestore'; +import {normalizeSlug} from '../../shared/slug.js'; +import {hashStr, normalizeStr} from '../../shared/strings.js'; +import {logAction} from './actions.js'; +import {getLocalesForTranslationLanguage} from './l10n.js'; + +/** + * Firestore limits `in` queries to 10 values, so larger queries are broken up + * into chunks. + */ +const IN_QUERY_CHUNK_SIZE = 10; + +/** + * Firestore batches allow a max of 500 write ops. Use a lower limit to leave + * headroom for callers that add their own ops to a shared batch. + */ +const MAX_BATCH_OPS = 400; + +export interface TranslationsLocaleDoc { + /** Translations id, e.g. `Pages/foo--bar` or `common`. */ + id: string; + locale: string; + tags?: string[]; + strings: { + [hash: string]: {source: string; translation: string}; + }; + sys: { + modifiedAt: Timestamp; + modifiedBy: string; + publishedAt?: Timestamp; + publishedBy?: string; + linkedSheet?: { + spreadsheetId: string; + gid: number; + linkedAt: Timestamp; + linkedBy: string; + }; + }; +} + +export interface TranslationsLocaleDocWithRef { + ref: DocumentReference; + data: TranslationsLocaleDoc; +} + +/** + * Summary of a translations doc (all of its locale docs grouped by id), used + * by the translations manager list page. + */ +export interface TranslationsDocSummary { + id: string; + /** Locales that have a draft locale doc. */ + locales: string[]; + modifiedAt?: Timestamp; + modifiedBy?: string; + publishedAt?: Timestamp; + publishedBy?: string; + hasUnpublishedChanges: boolean; +} + +export interface TranslationsDocData { + id: string; + draft: Record; + published: Record; +} + +/** + * An edit to a source string's translations, keyed by "translation language" + * (which may be shared by multiple root locales per the + * `i18n.translationLanguages` config). + */ +export interface TranslationsEdit { + source: string; + translations: Record; +} + +export type TranslationsDocMode = 'draft' | 'published'; + +function getTranslationsManagerCollection(mode: TranslationsDocMode) { + const projectId = window.__ROOT_CTX.rootConfig.projectId; + const db = window.firebase.db; + return collection( + db, + 'Projects', + projectId, + 'TranslationsManager', + mode, + 'Translations' + ); +} + +function getTranslationsLocaleDocRef( + mode: TranslationsDocMode, + id: string, + locale: string +) { + return doc( + getTranslationsManagerCollection(mode), + `${normalizeSlug(id)}:${locale}` + ); +} + +function chunkArray(items: T[], chunkSize: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; +} + +function toMillis(ts?: Timestamp) { + return ts?.toMillis ? ts.toMillis() : 0; +} + +/** + * Lists all translations docs (draft), grouped by translations id. + * + * NOTE: the locale docs are grouped by the doc data's `id` field rather than + * by parsing the `{id}:{locale}` doc key, which would be ambiguous with + * slugs containing `--`. + */ +export async function listTranslationsDocs(): Promise< + TranslationsDocSummary[] +> { + const snapshot = await getDocs(getTranslationsManagerCollection('draft')); + const byId: Record = {}; + snapshot.forEach((docSnapshot) => { + const data = docSnapshot.data() as TranslationsLocaleDoc; + if (!data.id) { + return; + } + const summary = (byId[data.id] ??= { + id: data.id, + locales: [], + hasUnpublishedChanges: false, + }); + if (data.locale) { + summary.locales.push(data.locale); + } + // The doc's modified/published timestamps are the max across its locale + // docs. + if (toMillis(data.sys?.modifiedAt) > toMillis(summary.modifiedAt)) { + summary.modifiedAt = data.sys.modifiedAt; + summary.modifiedBy = data.sys.modifiedBy; + } + if (toMillis(data.sys?.publishedAt) > toMillis(summary.publishedAt)) { + summary.publishedAt = data.sys.publishedAt; + summary.publishedBy = data.sys.publishedBy; + } + }); + const summaries = Object.values(byId); + summaries.forEach((summary) => { + summary.locales.sort(); + summary.hasUnpublishedChanges = + !summary.publishedAt || + toMillis(summary.modifiedAt) > toMillis(summary.publishedAt); + }); + summaries.sort((a, b) => a.id.localeCompare(b.id)); + return summaries; +} + +/** + * Loads the draft and published locale docs for a translations doc id, keyed + * by locale. + */ +export async function loadTranslationsDoc( + id: string +): Promise { + const [draft, published] = await Promise.all( + (['draft', 'published'] as TranslationsDocMode[]).map(async (mode) => { + const q = query( + getTranslationsManagerCollection(mode), + where('id', '==', id) + ); + const snapshot = await getDocs(q); + const byLocale: Record = {}; + snapshot.forEach((docSnapshot) => { + const data = docSnapshot.data() as TranslationsLocaleDoc; + if (data.locale) { + byLocale[data.locale] = data; + } + }); + return byLocale; + }) + ); + return {id, draft, published}; +} + +/** + * Fetches the draft translations locale docs (with their doc refs) for a set + * of translations doc ids, grouped by id. Used to publish a doc's + * translations in the same batch as the doc itself. + */ +export async function getDraftTranslationsLocaleDocs( + ids: string[] +): Promise> { + return getTranslationsLocaleDocs(ids, 'draft'); +} + +/** + * Fetches the translations locale docs (with their doc refs) for a set of + * translations doc ids, grouped by id. + */ +export async function getTranslationsLocaleDocs( + ids: string[], + mode: TranslationsDocMode +): Promise> { + const localeDocsById: Record = {}; + const uniqueIds = Array.from(new Set(ids)); + if (uniqueIds.length === 0) { + return localeDocsById; + } + const colRef = getTranslationsManagerCollection(mode); + const snapshots = await Promise.all( + chunkArray(uniqueIds, IN_QUERY_CHUNK_SIZE).map((chunk) => + getDocs(query(colRef, where('id', 'in', chunk))) + ) + ); + snapshots.forEach((snapshot) => { + snapshot.forEach((docSnapshot) => { + const data = docSnapshot.data() as TranslationsLocaleDoc; + localeDocsById[data.id] ??= []; + localeDocsById[data.id].push({ref: docSnapshot.ref, data}); + }); + }); + return localeDocsById; +} + +/** + * Saves draft translations edits for a translations doc id. Each edit's + * translations are keyed by "translation language", which is expanded to the + * root locales that share it (per `i18n.translationLanguages`). + */ +export async function saveDraftTranslations( + id: string, + edits: TranslationsEdit[], + options?: {tags?: string[]} +) { + const db = window.firebase.db; + const modifiedBy = window.firebase.user.email || 'root-cms-ui'; + + // Group the edits by root locale: + // {locale: {hash: {source, translation}}} + const stringsByLocale: Record< + string, + Record + > = {}; + for (const edit of edits) { + const source = normalizeStr(edit.source); + if (!source) { + continue; + } + const hash = hashStr(source); + for (const [lang, translation] of Object.entries(edit.translations)) { + for (const locale of getLocalesForTranslationLanguage(lang)) { + stringsByLocale[locale] ??= {}; + stringsByLocale[locale][hash] = { + source, + translation: normalizeStr(translation || ''), + }; + } + } + } + + const locales = Object.keys(stringsByLocale); + if (locales.length === 0) { + return; + } + + let batch = writeBatch(db); + let numOps = 0; + for (const locale of locales) { + const localeDocRef = getTranslationsLocaleDocRef('draft', id, locale); + const updates: Record = { + id: id, + locale: locale, + strings: stringsByLocale[locale], + sys: { + modifiedAt: Timestamp.now(), + modifiedBy: modifiedBy, + }, + }; + if (options?.tags && options.tags.length > 0) { + updates.tags = arrayUnion(...options.tags); + } + batch.set(localeDocRef, updates, {merge: true}); + numOps += 1; + if (numOps >= MAX_BATCH_OPS) { + await batch.commit(); + batch = writeBatch(db); + numOps = 0; + } + } + if (numOps > 0) { + await batch.commit(); + } + logAction('translations.save_draft', {metadata: {translationsId: id}}); +} + +/** + * Adds the write ops for publishing a set of draft translations locale docs + * to a batch: updates each draft doc's `sys` with `publishedAt/By` and copies + * the doc to the published collection. Returns the number of ops added to + * the batch (2 per locale doc). + */ +export function addPublishTranslationsOpsToBatch( + batch: WriteBatch, + localeDocs: TranslationsLocaleDocWithRef[] +): number { + const publishedBy = window.firebase.user.email || 'root-cms-ui'; + let numOps = 0; + for (const localeDoc of localeDocs) { + const data = localeDoc.data; + const sys = { + ...data.sys, + publishedAt: Timestamp.now(), + publishedBy: publishedBy, + }; + batch.update(localeDoc.ref, {sys}); + const publishedDocRef = getTranslationsLocaleDocRef( + 'published', + data.id, + data.locale + ); + batch.set(publishedDocRef, {...data, sys}); + numOps += 2; + } + return numOps; +} + +/** + * Publishes a translations doc (copies the draft locale docs to the + * published collection). + */ +export async function publishTranslations(id: string) { + const db = window.firebase.db; + const localeDocsById = await getTranslationsLocaleDocs([id], 'draft'); + const localeDocs = localeDocsById[id] || []; + if (localeDocs.length === 0) { + throw new Error(`no draft translations to publish for ${id}`); + } + const batch = writeBatch(db); + addPublishTranslationsOpsToBatch(batch, localeDocs); + await batch.commit(); + logAction('translations.publish', {metadata: {translationsId: id}}); +} + +/** + * Adds the write ops for unpublishing translations to a batch: deletes the + * published locale docs and clears the published metadata from the draft + * locale docs. Returns the number of ops added. + */ +export function addUnpublishTranslationsOpsToBatch( + batch: WriteBatch, + draftLocaleDocs: TranslationsLocaleDocWithRef[], + publishedLocaleDocs: TranslationsLocaleDocWithRef[] +): number { + let numOps = 0; + for (const localeDoc of publishedLocaleDocs) { + batch.delete(localeDoc.ref); + numOps += 1; + } + for (const localeDoc of draftLocaleDocs) { + batch.update(localeDoc.ref, { + 'sys.publishedAt': deleteField(), + 'sys.publishedBy': deleteField(), + }); + numOps += 1; + } + return numOps; +} + +/** + * Unpublishes a translations doc: deletes the published locale docs and + * clears the published metadata from the draft locale docs. + */ +export async function unpublishTranslations(id: string) { + const db = window.firebase.db; + const [draftDocsById, publishedDocsById] = await Promise.all([ + getTranslationsLocaleDocs([id], 'draft'), + getTranslationsLocaleDocs([id], 'published'), + ]); + const batch = writeBatch(db); + addUnpublishTranslationsOpsToBatch( + batch, + draftDocsById[id] || [], + publishedDocsById[id] || [] + ); + await batch.commit(); + logAction('translations.unpublish', {metadata: {translationsId: id}}); +} + +/** + * Adds ops to a batch that remove all of a translations doc's locale docs + * (draft and published). Returns the number of ops added. + */ +export function addDeleteTranslationsOpsToBatch( + batch: WriteBatch, + localeDocs: TranslationsLocaleDocWithRef[] +): number { + for (const localeDoc of localeDocs) { + batch.delete(localeDoc.ref); + } + return localeDocs.length; +} + +/** + * Deletes a translations doc (all of its draft and published locale docs). + */ +export async function deleteTranslationsDoc(id: string) { + const db = window.firebase.db; + const [draftDocsById, publishedDocsById] = await Promise.all([ + getTranslationsLocaleDocs([id], 'draft'), + getTranslationsLocaleDocs([id], 'published'), + ]); + const batch = writeBatch(db); + addDeleteTranslationsOpsToBatch(batch, [ + ...(draftDocsById[id] || []), + ...(publishedDocsById[id] || []), + ]); + await batch.commit(); + logAction('translations.delete', {metadata: {translationsId: id}}); +} + +/** + * Deletes the draft and published translations locale docs for a set of ids + * without logging an action. Used when deleting a content doc. + */ +export async function deleteTranslationsForDocIds(ids: string[]) { + const db = window.firebase.db; + const [draftDocsById, publishedDocsById] = await Promise.all([ + getTranslationsLocaleDocs(ids, 'draft'), + getTranslationsLocaleDocs(ids, 'published'), + ]); + const localeDocs = [ + ...Object.values(draftDocsById).flat(), + ...Object.values(publishedDocsById).flat(), + ]; + if (localeDocs.length === 0) { + return; + } + for (const chunk of chunkArray(localeDocs, MAX_BATCH_OPS)) { + const batch = writeBatch(db); + addDeleteTranslationsOpsToBatch(batch, chunk); + await batch.commit(); + } +} + +/** + * Returns true if the v2 translations manager is enabled via the + * `experiments.v2TranslationsManager` flag. + */ +export function testV2TranslationsEnabled(): boolean { + return Boolean(window.__ROOT_CTX.experiments?.v2TranslationsManager); +} diff --git a/packages/root/src/core/config.ts b/packages/root/src/core/config.ts index 71d9105b5..ee23c8e17 100644 --- a/packages/root/src/core/config.ts +++ b/packages/root/src/core/config.ts @@ -222,6 +222,24 @@ export interface RootI18nConfig { * language. */ translationLanguages?: Record; + + /** + * Locale fallback chains for translations. When a translation is missing + * for a locale, each fallback locale is checked (in order) before falling + * back to `defaultLocale` and finally the source string. Fallbacks are + * resolved recursively, e.g. with the config below, `en-CA` resolves to + * `['en-CA', 'en-GB', 'en']`. For example: + * + * ``` + * i18n: { + * locales: ['en', 'en-GB', 'en-CA'], + * fallbacks: { + * 'en-CA': ['en-GB'], + * }, + * } + * ``` + */ + fallbacks?: Record; } export interface RootBuildConfig {