diff --git a/.eslintrc.json b/.eslintrc.json index a13dc81..26d2cbd 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -13,5 +13,8 @@ "@babel/plugin-syntax-import-attributes" ] } + }, + "rules": { + "indent": ["error", "tab"] } } diff --git a/README.md b/README.md index e9268bc..70637db 100644 --- a/README.md +++ b/README.md @@ -31,15 +31,18 @@ Output the `myMessage` message from the `es-es` file with all messageformat stru mfv -l es-es -p lang/ highlight myMessage ``` +Re-write messages to a standard format, using newlines and tabs for complex arguments, and removing any plural categories that are invalid for each locale (e.g. `zero` for English, `one` for Korean) +```shell +mfv -s en -p lang/ format -nr +``` + ### Options: `-V, --version` - output the version number -`-e, --throw-errors` - Throw an error if error issues are found - `--no-issues` - Don't output issues -`-i, --ignoreIssueTypes ` - Ignore these comma-separated issue types +`-i, --ignore ` - Ignore these comma-separated issue types `-l, --locales ` - Process only these comma-separated locales @@ -51,7 +54,7 @@ mfv -l es-es -p lang/ highlight myMessage `--json-obj` - Indicate that the files to be parsed are JSON files with keys that have objects for values with their own keys: `translation` and `context` -`-h, --help` - display help for command +`-h, --help` - Display help for command ### Subommands: @@ -65,7 +68,7 @@ mfv -l es-es -p lang/ highlight myMessage `highlight ` - Output a message with all non-translatable ICU MessageFormat structure highlighted -`help [command]` - display help for command +`format` - Rewrite messages to a standard format ## Config File @@ -74,39 +77,47 @@ Some options can be configured with default values in `mfv.config.json` { "source": "en" "path": "lang/", - "locales": "ar,de,en,es,es-es,hi,tr", + "locales": ["ar", "de", "en", "es", "es-es", "hi", "tr"], "jsonObj": true } ``` ## Errors -`argument` - There are unrecognized arguments in the target message. +`argument` - Unrecognized argument -`brace` - There are mismatched braces in the target message. +`brace` - Mismatched braces -`case` - There are unrecognized cases in the target message. +`category` - Unsupported category -`extraneous` - There is an extraneous message in the target locale. +`duplicate` - Multiple messages with the same name -`missing` - There is a message missing from the target locale. +`extraneous` - Message does not exist in the source locale -`nbsp` - There are invalid non-breaking spaces in the structure of the target message. +`missing` - Message missing from the target locale -`nest` - The nesting order of the target message does not match the source message. +`nbsp` - Message structure contains non-breaking space -`other` - The target message is missing an `other` case +`nest` - The nesting order of the target message does not match the source message -`parse` - The target message can not be parsed. +`option` - Unrecognized option -`source` - There is an error in the source message. +`option-missing` - Missing option used in the source + +`other` - Missing "other" option + +`parse` - Failed to parse message + +`source` - Failed to parse source message ## Warnings -`categories` - The target message is missing plural categories used in the target locale +`category-missing` - Missing categories used by the target locale + +`nest-ideal` - A `select` is nested inside a `plural` or `selectordinal` -`nest` - There is a `select` nested inside a `plural` or `selectordinal` in the target message. +`nest-order` - Nesting order does not match source -`split` - The target message is split by a non-argument. `plural`, `selectordinal`, and `select` cases should contain complete translations. +`split` - Split by a complex argument -`untranslated` - The target message has not been translated. +`untranslated` - Message has not been translated diff --git a/bin/cli.js b/bin/cli.js index f716fcf..6736a6b 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -2,300 +2,512 @@ /* eslint-disable no-console */ -import { parseLocales, structureRegEx, validateLocales } from '../src/validate.js'; +import { env } from 'node:process'; +import { parseLocales, validateLocales } from '../src/validate.js'; import { readFile, readdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; -import findConfig from 'find-config'; import glob from 'glob'; import pkg from '../package.json' with { type: 'json' }; -import { program } from 'commander'; +import { program, Option } from 'commander'; +import { getConfig, sortFn, structureRegEx } from '../src/utils.js'; -const configPath = findConfig('mfv.config.json'); -const { path, source: globalSource, locales: globalLocales, jsonObj: globalJsonObj } = configPath ? (await import(`file://${configPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; +let formatMessage, commandOpts; +const programArgs = {}; + +const { + path, + source: globalSource, + locales: globalLocales, + jsonObj: globalJsonObj +} = await getConfig(env.PWD); program - .version(pkg.version) - .option('-e, --throw-errors', 'Throw an error if error issues are found') - .option('--no-issues', 'Don\'t output issues') - .option('-i, --ignoreIssueTypes ', 'Ignore these comma-separated issue types') - .option('-l, --locales ', 'Process only these comma-separated locales') - .option('-p, --path ', 'Path to a directory containing locale files') - .option('-t, --translator-output', 'Output JSON of all source strings that are missing or untranslated in the target') - .option('-s, --source-locale ', 'The locale to use as the source') - .option('--json-obj', 'Indicate that the files to be parsed are JSON files with keys that have objects for values') - .command('validate', { isDefault: true, hidden: true }) - .action(() => { - program.validate = true; - }); + .version(pkg.version) + .option('--no-issues', 'Don\'t output issues') + .option('-i, --ignore ', 'Ignore these comma-separated issue types') + .option('-l, --locales ', 'Process only these comma-separated locales') + .option('-p, --path ', 'Glob path to a directory containing locale files') + .option('-s, --source-locale ', 'The locale to use as the source') + .option('--json-obj', 'Indicate that the files to be parsed are JSON files with keys that have objects for values') + .command('validate', { isDefault: true, hidden: true }) + .action(() => { + program.validate = true; + }); program - .command('remove-extraneous') - .description('Remove strings that do not exist in the source locale') - .action(() => { - program.removeExtraneous = true; - }); + .command('build') + .description('Build locale data for configured locales') + .action(() => { + program.build = true; + }); program - .command('add-missing') - .description('Add strings that do not exist in the target locale') - .action(() => { - program.addMissing = true; - }); + .command('print-missing') + .description('Output JSON of all source messages that are missing or untranslated in the target') + .action(() => { + program.printMissing = true; + }); program - .command('sort') - .description('Sort strings alphabetically by key, maintaining any blocks') - .action(() => { - program.sort = true; - }); + .command('remove-extraneous') + .description('Remove messages that do not exist in the source locale') + .action(() => { + program.removeExtraneous = true; + }); program - .command('rename ') - .description('Rename a string') - .action((oldKey, newKey) => { - program.rename = true; - program.oldKey = oldKey; - program.newKey = newKey; - }); + .command('add-missing') + .description('Add messages that do not exist in the target locale') + .action(() => { + program.addMissing = true; + }); program - .command('highlight ') - .description('Output a string with all non-translatable ICU MessageFormat structure highlighted') - .action(key => { - program.highlight = key; - }); + .command('sort') + .description('Sort messages alphabetically by key, maintaining any blocks') + .action(() => { + program.sort = true; + }); -program.parse(process.argv); +program + .command('rename ') + .description('Rename a message') + .action((oldKey, newKey) => { + program.rename = true; + programArgs.oldKey = oldKey; + programArgs.newKey = newKey; + }); -const pathCombined = program.path || path; -if (!pathCombined) throw new Error('Must provide a path to the locale files using either the -p option or a config file.'); +program + .command('format') + .description('Rewrite messages to a standard format') + .option('-n, --newlines', 'When formatting complex arguments, use newlines and indentation for readability') + .option('-a, --add', 'Add cases for missing supported pural and selectordinal categories') + .option('-r, --remove', 'Remove cases for unsupported pural and selectordinal categories') + .option('-d, --dedupe', 'Remove complex argument cases that duplicate the `other` case. Takes precedence over --add.') + .option('-t, --trim', 'Trim whitespace from both ends of messages. Disables selector hoisting.') + .option('-c, --correct', 'Attempt to correct argument names. A source locale must be provided.') + .addOption(new Option('--no-hoist', 'Do not hoist selectors').hideHelp()) + .addOption(new Option('-q, --quotes ', 'Replace quote characters with locale-appropriate characters').choices(['source', 'straight', 'both'])) + .action(async function() { + formatMessage = (await import('../src/format.js')).formatMessage; + program.format = true; + commandOpts = this.opts(); + /* + program.newlines = opts.newlines; + program.add = opts.add; + program.remove = opts.remove; + program.trim = opts.trim; + program.quotes = opts.quotes; + program.dedupe = opts.dedupe + */ + }); +program + .command('highlight ') + .description('Output a message with all non-translatable ICU MessageFormat structure highlighted') + .action(key => { + program.highlight = key; + }); + +await program.parseAsync(process.argv); +const programOpts = program.opts(); + +if (program.build) { + await import('../build-locale-data.js'); + process.exit(); +} + +const pathCombined = programOpts.path || path; +if (!pathCombined) { + console.error('Must provide a path to the locale files using either the -p option or a config file.'); + process.exit(1); +} + +const noSource = () => { + console.error('Must provide a source locale using either the -s option or a config file.'); + process.exit(1); +}; + +const writes = []; const localesPaths = glob.sync(pathCombined); -localesPaths.forEach(async localesPath => { +const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { - const absLocalesPath = `${process.cwd()}/${localesPath}`; + const absLocalesPath = `${process.cwd()}/${localesPath}`; - const subConfigPath = findConfig('mfv.config.json', { cwd: absLocalesPath }); + const { source, locales: configLocales, jsonObj } = await getConfig(absLocalesPath); - const { source, locales: configLocales, jsonObj } = subConfigPath ? (await import(`file://${subConfigPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; /* eslint-disable-line global-require */ + const files = await readdir(absLocalesPath).catch(err => { + console.log(`Failed to read ${absLocalesPath}\n`); + throw err; + }); - const files = await readdir(absLocalesPath).catch(err => { - console.log(`Failed to read ${absLocalesPath}\n`); - throw err; - }); + const sourceLocale = programOpts.sourceLocale || source || globalSource; + const allowedLocales = programOpts.locales?.replace(/\s/g, '').split(',') || configLocales || globalLocales; + const filteredFiles = !allowedLocales ? + files.filter(file => !(/^\..*/g).test(file)) : + files.filter(file => allowedLocales.concat(sourceLocale).includes(file.split('.')[0])); + const targetLocales = filteredFiles.map(file => file.split('.')[0]); + + if (program.removeExtraneous) { + if (!sourceLocale) noSource(); + console.log('Removing extraneous messages from:', targetLocales.join(', ')); + } + + if (program.addMissing) { + if (!sourceLocale) noSource(); + console.log('Adding missing messages to:', targetLocales.join(', ')); + } + + if (program.rename) { + console.log(`Renaming "${programArgs.oldKey}" to "${programArgs.newKey}" in:`, targetLocales.join(', ')); + } + + if (program.format) { + if (!sourceLocale) noSource(); + //console.log(`Formatting:`, targetLocales.join(', ')); + } + + const resources = await Promise.all(filteredFiles.map(file => readFile(absLocalesPath + file, 'utf8'))) + .then(readFiles => readFiles.map((contents, idx) => ({ + file: filteredFiles[idx], + contents + }))) + .catch(err => { + console.error(err); + process.exitCode = 1; + }); + if (!resources) return; + + const useJSONObj = programOpts.jsonObj || jsonObj || globalJsonObj; + + const locales = parseLocales(resources, useJSONObj); + + if (program.highlight) { + + const showWS = str => str + .replace(/ /g, '·') + .replace(/\t/g, '··') + .replace(/\n/g, '␤\n'); - const sourceLocale = program.sourceLocale || source || globalSource; - const allowedLocalesString = program.locales || configLocales || globalLocales; - const allowedLocales = allowedLocalesString && allowedLocalesString.split(',').concat(sourceLocale); - const filteredFiles = !allowedLocales ? - files.filter(file => !(/^\..*/g).test(file)) : - files.filter(file => allowedLocales.includes(file.split('.')[0])); - const targetLocales = filteredFiles.map(file => file.split('.')[0]); + Object.keys(locales).forEach(locale => { + if ((!allowedLocales || allowedLocales.includes(locale)) && locales[locale].parsed[program.highlight]) { + const str = String(locales[locale].parsed[program.highlight].val); - if (program.removeExtraneous) { - console.log('Removing extraneous strings from:', targetLocales.join(', ')); - } + let match; + let prevEnd = 0; + const sections = []; + + while((match = structureRegEx.exec(str)) !== null) { + sections.push(showWS(str.substring(prevEnd, match.index))); + sections.push(chalk.red(showWS(str.substr(match.index, match[0].length)))); + prevEnd = match.index + match[0].length; + } + + sections.push(showWS(str.substring(prevEnd))); + + const highlighted = sections.join(''); + console.log(highlighted); + + } + }); - if (program.addMissing) { - console.log('Adding missing strings to:', targetLocales.join(', ')); - } + return; + } + + if (program.format) { + let count = 0; - if (program.rename) { - console.log(`Renaming "${program.oldKey}" to "${program.newKey}" in:`, targetLocales.join(', ')); - } - - const resources = await Promise.all(filteredFiles.map(file => readFile(absLocalesPath + file, 'utf8'))) - .then(readFiles => readFiles.map((contents, idx) => ({ - file: filteredFiles[idx], - contents - }))) - .catch(err => { - console.error(err); - process.exitCode = 1; - }); - if (!resources) return; - - const useJSONObj = program.jsonObj || jsonObj || globalJsonObj; - - const locales = parseLocales(resources, useJSONObj); - - if (program.highlight) { - - const showWS = str => str - .replace(/ /g, '·') - .replace(/\t/g, '··') - .replace(/\n/g, '␤\n'); - - Object.keys(locales).forEach(locale => { - if ((!allowedLocales || allowedLocales.includes(locale)) && locales[locale].parsed[program.highlight]) { - const str = String(locales[locale].parsed[program.highlight].val); - - let match; - let prevEnd = 0; - const sections = []; - - while((match = structureRegEx.exec(str)) !== null) { - sections.push(showWS(str.substring(prevEnd, match.index))); - sections.push(chalk.red(showWS(str.substr(match.index, match[0].length)))); - prevEnd = match.index + match[0].length; - } - - sections.push(showWS(str.substring(prevEnd))); - - const highlighted = sections.join(''); - console.log(highlighted); - - } - }); - - return; - } - - if (program.rename) { - let count = 0; - await Promise.all(Object.keys(locales).map(async locale => { - if (!allowedLocales || allowedLocales.includes(locale)) { - - const localeContents = locales[locale].contents; - const t = locales[locale].parsed[program.oldKey]; - - if (localeContents.includes(t)) { - const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; - const noo = `${t.keyQuote}${program.newKey}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; - - count += 1; - const newLocaleContents = localeContents.replace(old, noo); - - await writeFile(absLocalesPath + locales[locale].file, newLocaleContents); - console.log(`${chalk.green('\u2714')} ${locales[locale].file} - Renamed`); - } - else { - console.log(`${chalk.red('\u2716')} ${locales[locale].file} - Missing`); - } - } - })); - - const cliReport = `\n ${chalk.green('\u2714')} Renamed ${count} strings`; - console.log(cliReport); - - return; - } - - const output = validateLocales({ locales, sourceLocale }); - const translatorOutput = {}; - - await Promise.all(output.map(async(locale, idx) => { - const localePath = `${absLocalesPath}${locales[locale.locale].file}`; - - if (!allowedLocales || allowedLocales.includes(locale.locale)) { - console.log((idx > 0 ? '\n' : '') + chalk.underline(localePath)); - if (program.issues) { - - locale.report.totals.ignored = 0; - - if (program.sort) { - const sorted = Object.values(locales[locale.locale].parsed) - .reduce((acc, val) => { - const block = !val.startsWith('\n\n') ? acc.pop() || [] : []; - block.push(val.replace('\n\n', '\n')); - acc.push(block); - return acc; - }, []) - .map(block => block.sort().join('')) - .sort() - .join('\n'); - - locales[locale.locale].contents = locales[locale.locale].contents.replace(Object.values(locales[locale.locale].parsed).join(''), sorted); - } - else { - - locale.issues.forEach(issue => { - if (program.removeExtraneous) { - if (issue.type === 'extraneous') { - locales[locale.locale].contents = locales[locale.locale].contents.replace(locales[locale.locale].parsed[issue.key], '') - console.log('Removed:', issue.key); - } - } - else if (program.addMissing) { - if (issue.type === 'missing') { - const keys = Object.keys(locales[sourceLocale].parsed); - const targetKeys = Object.keys(locales[locale.locale].parsed); - const keyIdx = keys.indexOf(issue.key); - const nextKey = keys[keyIdx + 1]; - const previousKey = keys[keyIdx - 1]; - const nextString = locales[locale.locale].parsed[nextKey]; - const siblingString = nextString || locales[locale.locale].parsed[previousKey] || locales[locale.locale].parsed[targetKeys[targetKeys.length - 1]]; - const contents = locales[locale.locale].contents; - const insertAt = contents.indexOf(siblingString) + Number(!nextString ? String(siblingString).length : 0); - const comma = !nextString && !siblingString.comma ? `,${siblingString.comment}` : ''; - const commaOffset = comma ? siblingString.comment.length : 0; - const sourceString = `${comma}${locales[sourceLocale].parsed[issue.key]}`; - locales[locale.locale].contents = [contents.slice(0, insertAt - commaOffset), sourceString, contents.slice(insertAt)].join(''); - console.log('Added:', issue.key); - locales[locale.locale].parsed[issue.key] = locales[sourceLocale].parsed[issue.key]; - } - } - else if (program.translatorOutput) { - if (['missing', 'untranslated'].includes(issue.type)) { - translatorOutput[issue.key] = issue.source; - } - } - else if (!program.ignoreIssueTypes || !program.ignoreIssueTypes - .replace(' ','') - .split(',') - .includes(issue.type) - ) { - console.log([ - ' ', chalk.grey(`${issue.line}:${issue.column}`), - ' ', chalk[issue.level == 'error' ? 'red' : 'yellow'](issue.level), - ' ', chalk.grey(issue.type), - ' ', chalk.cyan(issue.key), - ' ', chalk.white(issue.msg) - ].join('')); - } - else { - locale.report.totals.ignored += 1; - } - }); - } - - if (program.removeExtraneous || program.addMissing || program.sort) { - await writeFile(localePath, locales[locale.locale].contents); - } - } - - if (program.translatorOutput) { - console.log(JSON.stringify(translatorOutput, null, 2)); - } - - if (program.removeExtraneous) { - const count = locale.report.errors ? locale.report.errors.extraneous || 0 : 0; - const cliReport = `\n ${chalk.green('\u2714')} Removed ${count} extraneous strings`; - console.log(cliReport); - } - else if (program.addMissing) { - const count = locale.report.errors ? locale.report.errors.missing || 0 : 0; - const cliReport = `\n ${chalk.green('\u2714')} Added ${count} missing strings`; - console.log(cliReport); - } - else if (program.sort) { - console.log('\nSorted'); - } - else if (locale.report.totals.errors || locale.report.totals.warnings) { - const color = locale.report.totals.errors ? 'red' : 'yellow'; - const total = locale.report.totals.errors + locale.report.totals.warnings; - const cliReport = chalk[color](`\n\u2716 ${total} issues (${locale.report.totals.errors} errors, ${locale.report.totals.warnings} warnings)${locale.report.totals.ignored ? chalk.grey(` - ${locale.report.totals.ignored} Ignored`) : ''}`); - console.log(cliReport); - } - else { - const cliReport = `\n ${chalk.green('\u2714')} Passed`; - console.log(cliReport); - } - } - })); - - if (program.throwErrors && output.some(locale => locale.report.totals.errors)) { - console.error('\nErrors were reported in at least one locale. See details above.'); - return 1; - } -}); + console.log(chalk.underline(localesPath)); + console.log(`Formatting:`, targetLocales.join(', ')); + + const sourceLocaleParsed = locales[sourceLocale].parsed; + + await Promise.all(Object.keys(locales).map(async locale => { + if (!allowedLocales || allowedLocales.includes(locale)) { + + let localeContents = locales[locale].contents; + await Promise.all(Object.values(locales[locale].parsed).map(async t => { + + const source = commandOpts.correct ? sourceLocaleParsed[t.key] : null; + + if (localeContents.includes(t)) { + const baseTabs = t.valSpace.includes('\n') && !commandOpts.newlines + ? t.valSpace.match(/^\n?(?\t*)/).groups.tabs + : t.match(/^\n?(?\t*)/).groups.tabs; + + const unescapedValue = t.val.replaceAll(`\\${t.valQuote}`, t.valQuote); + let newVal = await formatMessage(unescapedValue, { + locale, + sourceLocale, + add: commandOpts.add, + remove: commandOpts.remove, + newlines: commandOpts.newlines, + dedupe: commandOpts.dedupe, + trim: commandOpts.trim, + collapse: commandOpts.collapse, + quotes: commandOpts.quotes, + expandHashes: true, + hoist: commandOpts.hoist, + + baseTabs: baseTabs.length + (commandOpts.newlines ? 1 : 0), + key: t.key, + source, + target: t + }); + const valQuote = commandOpts.newlines && newVal.includes('\n') ? '`' : t.valQuote; + const valSpace = commandOpts.newlines && newVal.includes('\n') ? `\n\t${baseTabs}` : t.valSpace; + + newVal = newVal.replaceAll(valQuote, `\\${valQuote}`); + + const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}${t.valQuote}${t.comma}${t.comment}`; + const noo = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${valSpace}${valQuote}${newVal}${valQuote}${t.comma}${t.comment}`; + + if (old !== noo) count += 1; + localeContents = localeContents.replace(old, noo); + } + })); + + await writeFile(absLocalesPath + locales[locale].file, localeContents); + } + })); + + const cliReport = `\n ${chalk.green('\u2714')} Formatted ${count} messages`; + console.log(cliReport); + + return; + } + + if (program.rename) { + let count = 0; + await Promise.all(Object.keys(locales).map(async locale => { + if (!allowedLocales || allowedLocales.includes(locale)) { + + const localeContents = locales[locale].contents; + const t = locales[locale].parsed[programArgs.oldKey]; + + if (localeContents.includes(t)) { + const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; + const noo = `${t.keyQuote}${programArgs.newKey}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; + + count += 1; + const newLocaleContents = localeContents.replace(old, noo); + + await writeFile(absLocalesPath + locales[locale].file, newLocaleContents); + console.log(`${chalk.green('\u2714')} ${locales[locale].file} - Renamed`); + } + else { + console.log(`${chalk.red('\u2716')} ${locales[locale].file} - Missing`); + } + } + })); + + const cliReport = `\n ${chalk.green('\u2714')} Renamed ${count} messages`; + console.log(cliReport); + + return; + } + + if (!sourceLocale) noSource(); + + const output = validateLocales({ locales, sourceLocale }); + + await Promise.all(output.map(async(locale, idx) => { + const printMissing = {}; + const localeFile = `${locales[locale.locale].file}`; + const localeFilePath = `${localesPath}${localeFile}`; + if (!allowedLocales || allowedLocales.includes(locale.locale)) { + + if (program.sort || + program.removeExtraneous || + program.addMissing || + program.printMissing || + locale.report.totals.errors || + locale.report.totals.warnings) { + console.log((idx > 0 ? '\n' : '') + chalk.underline(localeFilePath)); + } + + if (programOpts.issues) { + + locale.report.totals.ignored = { warnings: 0, errors: 0 }; + + if (program.sort) { + let lastComma = ','; + const sorted = Object.values(locales[locale.locale].parsed) + .reduce((acc, val) => { + const block = !val.startsWith('\n\n') ? acc.pop() || [] : []; + if (!val.comma) { + lastComma = ''; + val.comma = ','; + val = `${val.match(/\s*/)[0]}${val.keyQuote}${val.key}${val.keyQuote}${val.keySpace}:${val.valSpace}${val.valQuote}${val.val}${val.valQuote}${val.comma}${val.comment}`; + } + block.push(val.replace('\n\n', '\n')); + acc.push(block); + return acc; + }, []) + .map(block => block.sort(sortFn).join('')) + .sort(sortFn) + + const last = sorted.pop(); + sorted.push(last.replace(/,(\s*(\/\/[^\n]*)?$)/, `${lastComma}$1`)); + locales[locale.locale].contents = locales[locale.locale].contents.replace(Object.values(locales[locale.locale].parsed).join(''), sorted.join('\n')); + } + else { + + locale.issues.forEach(issue => { + if (program.removeExtraneous) { + if (issue.type === 'extraneous') { + locales[locale.locale].contents = locales[locale.locale].contents.replace(locales[locale.locale].parsed[issue.key], '') + console.log('Removed:', issue.key); + } + } + else if (program.addMissing) { + if (issue.type === 'missing') { + const keys = Object.keys(locales[sourceLocale].parsed); + const targetKeys = Object.keys(locales[locale.locale].parsed); + const keyIdx = keys.indexOf(issue.key); + const nextKey = keys[keyIdx + 1]; + const previousKey = keys[keyIdx - 1]; + const nextMessage = locales[locale.locale].parsed[nextKey]; + const siblingMessage = nextMessage || locales[locale.locale].parsed[previousKey] || locales[locale.locale].parsed[targetKeys[targetKeys.length - 1]]; + const contents = locales[locale.locale].contents; + const insertAt = contents.indexOf(siblingMessage) + Number(!nextMessage ? String(siblingMessage).length : 0); + const comma = !nextMessage && !siblingMessage.comma ? `,${siblingMessage.comment}` : ''; + const commaOffset = comma ? siblingMessage.comment.length : 0; + const sourceMessage = `${comma}${locales[sourceLocale].parsed[issue.key]}`; + locales[locale.locale].contents = [contents.slice(0, insertAt - commaOffset), sourceMessage, contents.slice(insertAt)].join(''); + console.log('Added:', issue.key); + locales[locale.locale].parsed[issue.key] = locales[sourceLocale].parsed[issue.key]; + } + } + else if (program.printMissing) { + if (['missing', 'untranslated'].includes(issue.type)) { + printMissing[issue.key] = issue.source; + } + } + else if (!programOpts.ignore || !programOpts.ignore + .replace(' ', '') + .split(',') + .includes(issue.type) + ) { + console.log([ + ' ', chalk.grey(`${issue.line}:${issue.column}`), + ' ', chalk[issue.level == 'error' ? 'red' : 'yellow'](issue.level), + ' ', chalk.grey(issue.type), + ' ', chalk.cyan(issue.key), + ' ', chalk.white(issue.msg) + ].join('')); + } + else { + locale.report.totals.ignored[`${issue.level}s`] += 1; + } + }); + } + + if (program.removeExtraneous || program.addMissing || program.sort) { + writes.push([localeFilePath, locales[locale.locale].contents]); + } + } + + if (program.printMissing) { + console.log(JSON.stringify(printMissing, null, 2)); + } + else if (program.removeExtraneous) { + const count = locale.report.errors ? locale.report.errors.extraneous || 0 : 0; + const cliReport = `\n ${chalk.green('\u2714')} Removed ${count} extraneous messages`; + console.log(cliReport); + } + else if (program.addMissing) { + const count = locale.report.errors ? locale.report.errors.missing || 0 : 0; + const cliReport = `\n ${chalk.green('\u2714')} Added ${count} missing messages`; + console.log(cliReport); + } + else if (program.sort) { + console.log('\nSorted'); + } + else if (locale.report.totals.errors || locale.report.totals.warnings) { + const color = locale.report.totals.errors ? 'red' : 'yellow'; + const total = locale.report.totals.errors + locale.report.totals.warnings; + const ignored = locale.report.totals.ignored.errors + locale.report.totals.ignored.warnings; + const cliReport = chalk[color](`\n\u2716 ${total} issues (${locale.report.totals.errors} errors, ${locale.report.totals.warnings} warnings)${ignored ? chalk.grey(` - ${ignored} Ignored`) : ''}`); + console.log(cliReport); + return; + } + else { + // passed + } + } + + locale.report = undefined; + + })); + + const totals = { + errors: 0, + warnings: 0, + ignored: { + errors: 0, + warnings: 0 + } + }; + + output.forEach(locale => { + if (locale.report) { + totals.errors += locale.report.totals.errors; + totals.warnings += locale.report.totals.warnings; + totals.ignored.errors += locale.report.totals.ignored.errors; + totals.ignored.warnings += locale.report.totals.ignored.warnings; + } + }); + + if (idx < localesPaths.length - 1) console.log(`\n\n`); + + return totals; +})); + +for (const [path, contents] of writes) { + await writeFile(path, contents); +} + +if (results.filter(r => r).length) { + + let exitCode = 0; + + const totals = { + errors: 0, + warnings: 0, + ignored: { + errors: 0, + warnings: 0 + } + }; + + results.forEach(result => { + if (result) { + totals.errors += result.errors; + totals.warnings += result.warnings; + totals.ignored.errors += result.ignored.errors; + totals.ignored.warnings += result.ignored.warnings; + } + }); + + console.log(chalk.bold(`\n\nTotal ${chalk.grey(chalk.grey(` ${pathCombined} `))}`)); + + if (totals.errors || totals.warnings) { + const color = totals.errors ? 'red' : 'yellow'; + const total = totals.errors + totals.warnings; + const ignored = totals.ignored.errors + totals.ignored.warnings; + const cliReport = chalk[color](`\u2716 ${total} issues (${totals.errors} errors, ${totals.warnings} warnings)${ignored ? chalk.grey(` - ${ignored} Ignored`) : ''}`); + console.log(chalk.bold(cliReport)); + } else { + const cliReport = `\n ${chalk.green('\u2714')} Passed`; + console.log(cliReport); + } + + if (totals.errors - totals.ignored.errors) { + console.error('\nErrors were reported in at least one locale. See details above.'); + exitCode = 1; + } + + process.exit(exitCode); +} diff --git a/build-locale-data.js b/build-locale-data.js new file mode 100644 index 0000000..3216c97 --- /dev/null +++ b/build-locale-data.js @@ -0,0 +1,70 @@ +import { readdir, writeFile } from 'node:fs/promises'; +import { env, stderr } from 'node:process'; +import { formatList, getConfig } from './src/utils.js'; +import { dirname, join, posix } from 'node:path'; + +const defaultLocales = ['ar', 'cy', 'da', 'de', 'en', 'en-gb', 'es', 'es-es', 'fr', 'fr-ca', 'fr-fr', 'haw', 'hi', 'ja', 'ko', 'mi', 'nl', 'pt', 'sv', 'th', 'tr', 'vi', 'zh-cn', 'zh-tw']; +const defaultLocaleMap = { 'fr-on': 'fr-ca' }; + +const SAVE_PATH = posix.join(dirname(import.meta.url), 'src/locale-data.js').replace(/file:(\/c:)?/i, ''); + +function getDelimiters(locale) { + let delimiters; + const lang = locale.split('-')[0]; + try { + delimiters = cldr.extractDelimiters(locale); + } catch(err) { + delimiters = cldr.extractDelimiters(lang); + } + + switch (lang) { + case 'haw': + delimiters.apostrophe = "'"; + break; + default: + delimiters.apostrophe = '’'; + break; + } + + return delimiters +} + +let cldr; +await (async() => { + let contents = `import localeData from './locale-data-default.js';\nexport default localeData;\n`; + + const config = await getConfig(); + const localeMap = config.localeMap || defaultLocaleMap; + + let locales = env.MFV_LOCALES?.split(',') ?? config.locales; + locales ??= config.path && (await readdir(join(dirname(config.__configPath), config.path)).catch(() => {}))?.map(f => f.split('.')[0]); + locales ??= defaultLocales; + const nonDefaultLocales = locales?.filter(l => !defaultLocales.includes(localeMap[l] ?? l)); + + if (nonDefaultLocales?.length) { + let cldrImport; + try { + cldrImport = await import('cldr'); + } catch(e) { + stderr.write(`\n\nSome configured locales (${formatList(nonDefaultLocales.map(l => `"${l}"`))}) require the 'cldr' package: npm i -D cldr\n\n`); + process.exitCode = 1; + return; + } + cldr = (cldrImport).default; + const data = {}; + + nonDefaultLocales.forEach(locale => { + try { + locale = Intl.getCanonicalLocales(locale.trim().toLowerCase())[0]; + } catch(e) { + stderr.write(e.message); + process.exit(1); + } + data[locale] = {}; + data[locale].delimiters = getDelimiters(locale); + }); + + contents = `import defaultLocaleData from './locale-data-default.js';\nexport default { ...defaultLocaleData, ...${JSON.stringify(data, null, '\t')}}\n`; + } + await writeFile(SAVE_PATH, contents); +})(); diff --git a/package-lock.json b/package-lock.json index 79aae13..4c23b86 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,21 +1,21 @@ { "name": "messageformat-validator", - "version": "2.6.7", + "version": "3.0.0-beta.17", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "2.6.7", + "version": "3.0.0-beta.17", + "hasInstallScript": true, "license": "MIT", "dependencies": { + "@formatjs/icu-messageformat-parser": "^2.7.8", "chalk": "^4.1.0", - "commander": "^6.1.0", + "commander": "^12.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", - "glob": "^7.1.6", - "make-plural": "^7.4.0", - "messageformat-parser": "^4.1.3" + "glob": "^7.1.6" }, "bin": { "mfv": "bin/cli.js" @@ -27,6 +27,7 @@ "@eslint/eslintrc": "^3.1.0", "@eslint/js": "^9.10.0", "chai": "^5.1.1", + "cldr": "^7.5.0", "eslint": "^8.57.0", "globals": "^15.9.0", "mocha": "^10.7.3" @@ -119,15 +120,6 @@ "eslint": "^7.5.0 || ^8.0.0 || ^9.0.0" } }, - "node_modules/@babel/eslint-parser/node_modules/eslint-visitor-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", - "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", - "dev": true, - "engines": { - "node": ">=10" - } - }, "node_modules/@babel/generator": { "version": "7.25.6", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.25.6.tgz", @@ -161,6 +153,16 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "peer": true, + "dependencies": { + "yallist": "^3.0.2" + } + }, "node_modules/@babel/helper-module-imports": { "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz", @@ -473,9 +475,9 @@ } }, "node_modules/@eslint-community/regexpp": { - "version": "4.11.0", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.11.0.tgz", - "integrity": "sha512-G/M/tIiMrTAxEWRfLfQJMmGNX28IxBg4PBz8XqQhqUHLFI6TL2htpIB1iQCj144V5ee/JaKyT9/WZ0MGZWfA7A==", + "version": "4.11.1", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.11.1.tgz", + "integrity": "sha512-m4DVN9ZqskZoLU5GlWZadwDnYo3vAEydiUayB9widCl9ffWx2IvPnp6n3on5rJmziJSw9Bv+Z3ChDVdMwXCY8Q==", "dev": true, "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" @@ -534,14 +536,50 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@formatjs/ecma402-abstract": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@formatjs/ecma402-abstract/-/ecma402-abstract-2.0.0.tgz", + "integrity": "sha512-rRqXOqdFmk7RYvj4khklyqzcfQl9vEL/usogncBHRZfZBDOwMGuSRNFl02fu5KGHXdbinju+YXyuR+Nk8xlr/g==", + "dependencies": { + "@formatjs/intl-localematcher": "0.5.4", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/icu-messageformat-parser": { + "version": "2.7.8", + "resolved": "https://registry.npmjs.org/@formatjs/icu-messageformat-parser/-/icu-messageformat-parser-2.7.8.tgz", + "integrity": "sha512-nBZJYmhpcSX0WeJ5SDYUkZ42AgR3xiyhNCsQweFx3cz/ULJjym8bHAzWKvG5e2+1XO98dBYC0fWeeAECAVSwLA==", + "dependencies": { + "@formatjs/ecma402-abstract": "2.0.0", + "@formatjs/icu-skeleton-parser": "1.8.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/icu-skeleton-parser": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@formatjs/icu-skeleton-parser/-/icu-skeleton-parser-1.8.2.tgz", + "integrity": "sha512-k4ERKgw7aKGWJZgTarIcNEmvyTVD9FYh0mTrrBMHZ1b8hUu6iOJ4SzsZlo3UNAvHYa+PnvntIwRPt1/vy4nA9Q==", + "dependencies": { + "@formatjs/ecma402-abstract": "2.0.0", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/intl-localematcher": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/@formatjs/intl-localematcher/-/intl-localematcher-0.5.4.tgz", + "integrity": "sha512-zTwEpWOzZ2CiKcB93BLngUX59hQkuZjT2+SAQEscSm52peDW/getsawMcWF1rGRpMCX6D7nSJA3CzJ8gn13N/g==", + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", "deprecated": "Use @eslint/config-array instead", "dev": true, "dependencies": { - "@humanwhocodes/object-schema": "^2.0.2", + "@humanwhocodes/object-schema": "^2.0.3", "debug": "^4.3.1", "minimatch": "^3.0.5" }, @@ -631,28 +669,6 @@ "eslint-scope": "5.1.1" } }, - "node_modules/@nicolo-ribaudo/eslint-scope-5-internals/node_modules/eslint-scope": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", - "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", - "dev": true, - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^4.1.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@nicolo-ribaudo/eslint-scope-5-internals/node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -699,6 +715,15 @@ "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", "dev": true }, + "node_modules/@xmldom/xmldom": { + "version": "0.8.10", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.10.tgz", + "integrity": "sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==", + "dev": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/acorn": { "version": "8.12.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", @@ -932,6 +957,18 @@ "node": ">=12" } }, + "node_modules/chainsaw": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.0.9.tgz", + "integrity": "sha512-nG8PYH+/4xB+8zkV4G844EtfvZ5tTiLFoX3dZ4nhF4t3OCKIb9UvaFyNmeZO2zOSmRWzBoTD+napN6hiL+EgcA==", + "dev": true, + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/chalk": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", @@ -980,6 +1017,23 @@ "fsevents": "~2.3.2" } }, + "node_modules/cldr": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/cldr/-/cldr-7.5.0.tgz", + "integrity": "sha512-2qy3ASYFbNToTujNnk5Y8ak++B4TH/G+S8AEOrN1xUFZhxhmqWDPUGnOFGyId61vD2Trf+yE65wVzIcdE/bpPg==", + "dev": true, + "dependencies": { + "@xmldom/xmldom": "^0.8.0", + "escodegen": "^2.0.0", + "esprima": "^4.0.1", + "memoizeasync": "^1.1.0", + "passerror": "^1.1.1", + "pegjs": "^0.10.0", + "seq": "^0.3.5", + "unicoderegexp": "^0.4.1", + "xpath": "^0.0.33" + } + }, "node_modules/cliui": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", @@ -1008,11 +1062,11 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/commander": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-6.1.0.tgz", - "integrity": "sha512-wl7PNrYWd2y5mp1OK/LhTlv8Ff4kQJQRXXAvF+uU/TPNiVJUxZLRYGj/B0y/lPGAVcSbJqH2Za/cvHmrPMC8mA==", + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", "engines": { - "node": ">= 6" + "node": ">=18" } }, "node_modules/concat-map": { @@ -1107,9 +1161,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.19", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.19.tgz", - "integrity": "sha512-kpLJJi3zxTR1U828P+LIUDZ5ohixyo68/IcYOHLqnbTPr/wdgn4i1ECvmALN9E16JPA6cvCG5UG79gVwVdEK5w==", + "version": "1.5.25", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.25.tgz", + "integrity": "sha512-kMb204zvK3PsSlgvvwzI3wBIcAw15tRkYk+NQdsjdDtcQWTp2RABbMQ9rUBy8KNEOM+/E6ep+XC3AykiWZld4g==", "dev": true, "peer": true }, @@ -1140,17 +1194,38 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, "node_modules/eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "@ungap/structured-clone": "^1.2.0", @@ -1196,31 +1271,34 @@ } }, "node_modules/eslint-scope": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", - "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", "dev": true, "dependencies": { "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" + "estraverse": "^4.1.1" }, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" + "node": ">=8.0.0" + } + }, + "node_modules/eslint-scope/node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "engines": { + "node": ">=4.0" } }, "node_modules/eslint-visitor-keys": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.0.0.tgz", - "integrity": "sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", "dev": true, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" + "node": ">=10" } }, "node_modules/eslint/node_modules/@eslint/eslintrc": { @@ -1247,12 +1325,28 @@ } }, "node_modules/eslint/node_modules/@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/eslint/node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", "dev": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" } }, "node_modules/eslint/node_modules/eslint-visitor-keys": { @@ -1336,6 +1430,31 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/espree/node_modules/eslint-visitor-keys": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.0.0.tgz", + "integrity": "sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw==", + "dev": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", @@ -1589,6 +1708,18 @@ "node": ">=8" } }, + "node_modules/hashish": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz", + "integrity": "sha512-xyD4XgslstNAs72ENaoFvgMwtv8xhiDtC2AtzCG+8yF7W/Knxxm9BX+e2s25mm+HxMKh0rBmXVOEGF3zNImXvA==", + "dev": true, + "dependencies": { + "traverse": ">=0.2.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -1865,25 +1996,21 @@ } }, "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.0.tgz", + "integrity": "sha512-dVmQmXPBlTgFw77hm60ud//l2bCuDKkqC2on1EBoM7s9Urm9IQDrnujwZ93NFnAq0dVZ0HBXTS7PwEG+YE7+EQ==", + "dev": true + }, + "node_modules/memoizeasync": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/memoizeasync/-/memoizeasync-1.1.0.tgz", + "integrity": "sha512-HMfzdLqClZo8HMyuM9B6TqnXCNhw82iVWRLqd2cAdXi063v2iJB4mQfWFeKVByN8VUwhmDZ8NMhryBwKrPRf8Q==", "dev": true, - "peer": true, "dependencies": { - "yallist": "^3.0.2" + "lru-cache": "2.5.0", + "passerror": "1.1.1" } }, - "node_modules/make-plural": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/make-plural/-/make-plural-7.4.0.tgz", - "integrity": "sha512-4/gC9KVNTV6pvYg2gFeQYTW3mWaoJt7WZE5vrp1KnQDgW92JtYZnzmZT81oj/dUTqAIu0ufI2x3dkgu3bB1tYg==" - }, - "node_modules/messageformat-parser": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-4.1.3.tgz", - "integrity": "sha512-2fU3XDCanRqeOCkn7R5zW5VQHWf+T3hH65SzuqRvjatBK7r4uyFa5mEX+k6F9Bd04LVM5G4/BHBTUJsOdW7uyg==" - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -2089,6 +2216,15 @@ "node": ">=6" } }, + "node_modules/passerror": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/passerror/-/passerror-1.1.1.tgz", + "integrity": "sha512-PwrEQJBkJMxnxG+tdraz95vTstYnCRqiURNbGtg/vZHLgcAODc9hbiD5ZumGUoh3bpw0F0qKLje7Vd2Fd5Lx3g==", + "dev": true, + "engines": { + "node": "*" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -2124,6 +2260,18 @@ "node": ">= 14.16" } }, + "node_modules/pegjs": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/pegjs/-/pegjs-0.10.0.tgz", + "integrity": "sha512-qI5+oFNEGi3L5HAxDwN2LA4Gg7irF70Zs25edhjld9QemOgp0CbvMtbFcMvFtEo1OityPrcCzkQFB8JP/hxgow==", + "dev": true, + "bin": { + "pegjs": "bin/pegjs" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/picocolors": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.0.tgz", @@ -2298,6 +2446,19 @@ "semver": "bin/semver.js" } }, + "node_modules/seq": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/seq/-/seq-0.3.5.tgz", + "integrity": "sha512-sisY2Ln1fj43KBkRtXkesnRHYNdswIkIibvNe/0UKm2GZxjMbqmccpiatoKr/k2qX5VKiLU8xm+tz/74LAho4g==", + "dev": true, + "dependencies": { + "chainsaw": ">=0.0.7 <0.1", + "hashish": ">=0.0.2 <0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/serialize-javascript": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", @@ -2328,6 +2489,16 @@ "node": ">=8" } }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -2405,6 +2576,20 @@ "node": ">=8.0" } }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -2429,6 +2614,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/unicoderegexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/unicoderegexp/-/unicoderegexp-0.4.1.tgz", + "integrity": "sha512-ydh8D5mdd2ldTS25GtZJEgLciuF0Qf2n3rwPhonELk3HioX201ClYGvZMc1bCmx6nblZiADQwbMWekeIqs51qw==", + "dev": true + }, "node_modules/update-browserslist-db": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz", @@ -2532,6 +2723,15 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "node_modules/xpath": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.33.tgz", + "integrity": "sha512-NNXnzrkDrAzalLhIUc01jO2mOzXGXh1JwPgkihcLLzw98c0WgYDmmjSh1Kl3wzaxSVWMuA+fe0WTWOBDWCBmNA==", + "dev": true, + "engines": { + "node": ">=0.6.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 6c1f53e..54a13da 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "messageformat-validator", - "version": "2.6.7", - "description": "Validates that ICU MessageFormat strings are well-formed, and that translated target strings are compatible with their source.", + "version": "3.0.0-beta.17", + "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { "type": "git", @@ -13,7 +13,9 @@ "lint:eslint": "eslint . --ext .js --ignore-path .gitignore", "test": "npm run lint && npm run test:unit", "test:unit": "mocha 'test/**/*.test.js'", - "prepublishOnly": "npm t" + "prepack": "npm run build && npm t", + "postinstall": "npm run build", + "build": "node build-locale-data.js" }, "bin": { "mfv": "bin/cli.js" @@ -22,18 +24,18 @@ ".": "./src/index.js" }, "files": [ - "/src" + "/src", + "/build-locale-data.js" ], - "author": "Daniel Gleckler ", + "author": "Danny Gleckler ", "license": "MIT", "dependencies": { + "@formatjs/icu-messageformat-parser": "^2.7.8", "chalk": "^4.1.0", - "commander": "^6.1.0", + "commander": "^12.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", - "glob": "^7.1.6", - "make-plural": "^7.4.0", - "messageformat-parser": "^4.1.3" + "glob": "^7.1.6" }, "devDependencies": { "@babel/eslint-parser": "^7.25.1", @@ -42,6 +44,7 @@ "@eslint/eslintrc": "^3.1.0", "@eslint/js": "^9.10.0", "chai": "^5.1.1", + "cldr": "^7.5.0", "eslint": "^8.57.0", "globals": "^15.9.0", "mocha": "^10.7.3" diff --git a/src/format.js b/src/format.js new file mode 100644 index 0000000..ee1d326 --- /dev/null +++ b/src/format.js @@ -0,0 +1,389 @@ +import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.js'; +import { parse } from '@formatjs/icu-messageformat-parser'; +import { getArgs, getLocaleData, getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from './utils.js'; + +function expandASTHashes(ast, parentValue) { + if (Array.isArray(ast)) { + ast.map(ast => expandASTHashes(ast, parentValue)); + } + + if (ast.type === 7 && parentValue) { // # + ast.type = 1; + ast.value = parentValue; + } + else if (ast.type === 6 && !ast.offset) { // plural, selectordinal + expandASTHashes(Object.values(ast.options).map(o => o.value), ast.value); + } +} + +function mfEscape(msg) { + return msg.replace(/'([{}](?:.*?[{}])?)'/gsu, "'''$1'''"); +} + +export async function formatMessage(msg, options = {}) { + let ast; + + try { + msg = options.quotes === 'straight' ? msg.replace(/'/g, "'''") : mfEscape(msg); + ast = parse(msg, { requiresOtherClause: false }); + } catch(err) { + try { + const alteredMsg = msg.replace('\'{', '’{'); + ast = parse(alteredMsg, { requiresOtherClause: false }); + msg = alteredMsg; + } catch(err2) { + if (err.location) { + console.log(`\nERROR: ${err.message}`); + console.log(`\tLocale: ${options.locale}`); + console.log(`\tKey: ${options.key}`); + console.log(`\tOriginal message: ${err.originalMessage}`); + console.log('\tAt or near:', msg.slice(err.location.start.offset, Math.max(err.location.end.offset, err.location.start.offset + 4))); + } else { + console.log(err); + console.log(`\tLocale: ${options.locale}`); + console.log(`\tKey: ${options.key}`); + } + return msg; + } + } + if (options.expandHashes) { + expandASTHashes(ast); + } + + if ((options.hoist ?? true) && !options.trim) { + try { + ast = hoistSelectors(ast); + } catch(e) { + console.log(e); + } + } + + const localeData = { [options.locale]: await getLocaleData(options.locale) }; + if (options.sourceLocale) localeData[options.sourceLocale] = await getLocaleData(options.sourceLocale); + + let args = []; + if (options.source) { + try { + args = getArgs(parse(options.source, { requiresOtherClause: false })); + } catch(e) { + console.warn('WARN Source error:', options.key); + } + } + + return printAST(ast, { + useNewlines: options.newlines ?? msg.match(structureRegEx)?.join('').includes('\n'), + add: options.add ?? false, + remove: options.remove ?? false, + dedupe: options.dedupe ?? false, + trim: options.trim ?? false, + quotes: options.quotes, + + locale: options.locale, + sourceLocale: options.sourceLocale, + localeData, + args + }, options.baseTabs); +} + +function normalizeArgName(name, type, availableArgs) { + if (availableArgs.length) { + const idx = availableArgs.findIndex(([n, t]) => n === name && t === type); + if (idx > -1) { + availableArgs.splice(idx, 1); + return name; + } + else if (availableArgs.length === 1) { + return availableArgs.pop()[0]; + } else { + const idx = availableArgs.findIndex(([n, t]) => t === type && n.toLowerCase() === name.toLowerCase()); + if (idx > -1) { + // case match + return availableArgs.splice(idx, 1)[0][0]; + } + return [...new Set(availableArgs + .reduce((acc, [n, t]) => t === type && acc.push(n) && acc || acc, []) + )].join('|'); + } + } + return name; +} + +function printAST(ast, options, level = 0, parentValue) { + const { + locale, + sourceLocale, + quotes, + swapOne = new Set(), + useNewlines = false, + add = false, + remove = false, + dedupe = false, + trim = false, + args = [], + localeData, + } = options; + + const localeLower = locale.toLowerCase(); + + if (Array.isArray(ast)) { + const swapOneClone = new Set(swapOne); + ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) + + let msg = ast + .filter((i, idx) => !trim || i.type !== 0 || (idx !== 0 && idx !== ast.length - 1) || i.value.trim()) // filter out leading and trailing whitespace + .map((ast, idx, arr) => { + let trim = options.trim; + if (trim && ast.type === 0) { + if (arr.length === 1) { + trim = 'trim'; + } else if (!idx) { + trim = 'trimStart'; + } else if (idx === arr.length - 1) { + trim = 'trimEnd'; + } + } + return printAST(ast, { ...options, + swapOne: swapOneClone, + trim + }, + level); + }).join(''); + + if (quotes) { + const { delimiters } = localeData[locale]; + + for (const k in delimiters) { + if (paddedQuoteLocales.includes(localeLower)) { + if (k.endsWith('Start')) { + delimiters[k] = delimiters[k].padEnd(2,'\u202f'); + } else if (k.endsWith('End')) { + delimiters[k] = delimiters[k].padStart(2,'\u202f'); + } + } + } + + let + quoteStart = delimiters.quotationStart, + quoteEnd = delimiters.quotationEnd, + altStart = delimiters.alternateQuotationStart, + altEnd = delimiters.alternateQuotationEnd, + apostrophe = delimiters.apostrophe ?? '’'; + + //if (1) { // todo: fromSource + if (localeLower.endsWith('-gb')) { + quoteStart = delimiters.alternateQuotationStart; + quoteEnd = delimiters.alternateQuotationEnd; + altStart = delimiters.quotationStart; + altEnd = delimiters.quotationEnd; + } + //} + + if (quotes === 'straight' || quotes === 'both') { + msg = msg + .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") + .replace(/(?<=\s)\\?'|^\\?'/g, altStart) // opening ' + .replace(/(?<=\S)'(?=\S)/g, '|_apostrophe_|') // apostrophe + .replace(/\\?'/g, altEnd) // closing ' + .replace(/\|_apostrophe_\|/g, apostrophe) + .replace(/(?<=\s(\u0648)?)\\?"|^\\?"/g, quoteStart) // opening " + .replace(/\\?"/g, quoteEnd); // closing " + } + + if (quotes === 'source' || quotes === 'both' && !locale.endsWith('-gb')) { + const { + quoteEnd: sourceQuoteEnd, + quoteStart: sourceQuoteStart, + altEnd: sourceAltEnd, + altStart: sourceAltStart, + apostrophe: sourceApostrophe + } = (locale => { + const { delimiters } = localeData[locale]; + + for (const k in delimiters) { + if (paddedQuoteLocales.includes(locale.toLowerCase())) { + if (k.endsWith('Start')) { + delimiters[k] = delimiters[k].padEnd(2,'\u202f'); + } else if (k.endsWith('End')) { + delimiters[k] = delimiters[k].padStart(2,'\u202f'); + } + } + } + + let + quoteStart = delimiters.quotationStart, + quoteEnd = delimiters.quotationEnd, + altStart = delimiters.alternateQuotationStart, + altEnd = delimiters.alternateQuotationEnd, + apostrophe = delimiters.apostrophe ?? '’'; + + //if (1) { // todo: fromSource + if (locale.toLowerCase().endsWith('-gb')) { + quoteStart = delimiters.alternateQuotationStart; + quoteEnd = delimiters.alternateQuotationEnd; + altStart = delimiters.quotationStart; + altEnd = delimiters.quotationEnd; + } + + return { quoteStart, quoteEnd, altStart, altEnd, apostrophe }; + + })(sourceLocale); + + /* eslint-disable no-useless-escape */ + msg = msg + .replace(new RegExp(`''`, 'g'), '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") + .replace(new RegExp(`(?<=\s)\\\\?${sourceAltStart}|^\\\\?${sourceAltStart}`, 'g'), '|_altStart_|') // opening alt + .replace(new RegExp(`(?<=\\S)${sourceApostrophe}(?=\\S)`, 'g'), '|_apostrophe_|') // apostrophe + .replace(new RegExp(`\\\\?${sourceAltEnd}`, 'g'), '|_altEnd_|') // closing alt + .replace(new RegExp(`\\\\?${sourceQuoteStart}(\\b|(?=\\p{Sc}|\\p{P}))`, 'g'), '|_quoteStart_|') // opening quote + .replace(new RegExp(`\\b\\\\?${sourceQuoteEnd}`, 'g'), '|_quoteEnd_|') // closing quote + .replace(/\|_apostrophe_\|/g, apostrophe) + .replace(/\|_quoteStart_\|/g, quoteStart) + .replace(/\|_quoteEnd_\|/g, quoteEnd) + .replace(/\|_altStart_\|/g, altStart) + .replace(/\|_altEnd_\|/g, altEnd); + /* eslint-enable no-useless-escape */ + } + } + if (!quotes && localeData?.[locale]) { + const { delimiters } = localeData[locale]; + const apostrophe = delimiters.apostrophe ?? '\u2019'; + msg = msg.replace(/'/g, apostrophe); + } + return msg.replace(/\|_escape_\|/g, "'"); + } + + let text = ''; + const indent = useNewlines ? Array(level).fill('\t').join('') : ' '; + const newline = useNewlines ? `\n${indent}` : indent; + const type = ast.type; + + if (type === 0) { // straight text + let escaped = ast.value; + escaped = escaped.replace(/'([{}](?:.*[{}])?)'/gsu, "|_escape_|$1|_escape_|"); + escaped = parentValue ? escaped.replace("'#'", "|_escape_|#|_escape_|") : escaped; + + escaped = swapOne.size ? escaped.replace(/1/g, `{${[...swapOne].join('|')}}`) : escaped; + text += escaped[trim]?.() ?? escaped; + } + else if (type === 1) { // simple arg + text += `{${normalizeArgName(ast.value, type, args)}}`; + } + else if ([2, 3, 4].includes(type)) { // number, date, time + const style = (() => { + if (ast.style) { + if (typeof ast.style === 'string') return `, ${ast.style}`; + else return `, ::${ast.style.pattern || ast.style.tokens.map(t => t.stem).join(' ')}`; + } else { + return ''; + } + })(); + + const typesText = ['number', 'date', 'time']; + text += `{${normalizeArgName(ast.value, type, args)}, ${typesText[type - 2]}${style}}`; + } + else if (type === 5) { // select + + const argName = normalizeArgName(ast.value, type, args); + const optionsText = Object.entries(ast.options) + .sort((a, b) => { + return a[0] === 'other' ? 1 : (b[0] === 'other' ? -1 : 0); + }) + .map(([opt, { value }]) => { + return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, args }, level + 1)}}`; + }).join('') + (useNewlines ? newline : ''); + + text += `{${argName}, select,${optionsText}}`; + } + else if (type === 6) { // plural, selectordinal + const supportedCats = getPluralCats(locale, ast.pluralType); + const unsupportedCats = [ ...Object.keys(ast.options).filter(o => !/^=\d+$/.test(o)) , ...sortedCats].filter(cat => !supportedCats.includes(cat)); + if (add) { + supportedCats.forEach(cat => { + // TODO: check actual locale plural categories + if (!/^(fr|pt)/.test(locale) && cat === 'one' && ast.options['=1']) return; // don't create orphaned `one` + // add missing supported categories + ast.options[cat] ??= { ...(ast.options.other || ast.options.many || ast.options.few || Object.values(ast.options).at(-1)) }; + }); + } + + if (ast.options['=1'] && !ast.offset) { + if (ast.options['=1'].value.filter(o => o.type === 0).map(o => o.value).join('').includes('1')) { // TODO: recursively check actual options + // `=1` exists with literal "1" text + ast.options.one = ast.options['=1']; + delete ast.options['=1']; + swapOne.add(ast.value); + } else if (locale === sourceLocale && locale.startsWith('en')) { + // `=1` exists with basic Englished pluralization differences + const enBasicPluralsRegEx = /e|s|es(?=\s|$|\p{P})/gv; + if (printAST(ast.options.other.value, { locale, args }).replace(enBasicPluralsRegEx, '') === printAST(ast.options['=1'].value, { locale, args }).replace(enBasicPluralsRegEx, '')) { + ast.options.one = ast.options['=1']; + delete ast.options['=1']; + swapOne.add(ast.value); + } + } + } + + if (ast.options.one && !ast.offset) { + // `one` and `=1` are the same + if (ast.options['=1'] && JSON.stringify(ast.options['=1']) === JSON.stringify(ast.options['one'])) { + delete ast.options['=1']; + } + } + + if (dedupe && ast.options.other) { + const otherPrinted = printAST(ast.options.other.value, { locale, args }); + Object.entries(ast.options).forEach(([k, v]) => { + if (k !== 'other' && printAST(v.value, { locale, swapOne, args }) === otherPrinted) { + delete ast.options[k]; + } + }); + } + + // replace a bad category if possible + const usedCats = Object.keys(ast.options); + const unusedCats = supportedCats.filter(c => !usedCats.includes(c)); + if (unusedCats.length === 1) { + const unrecognizedCats = usedCats.filter(c => !/^=\d+$/.test(c) && !supportedCats.includes(c)); + if (unrecognizedCats.length === 1) { + ast.options[unusedCats[0]] = ast.options[unrecognizedCats[0]]; + delete ast.options[unrecognizedCats[0]]; + } + } + + remove && unsupportedCats.forEach(cat => { + const currentKeys = Object.keys(ast.options); + if (currentKeys.includes(cat)) { + if (currentKeys.length === 1) { + ast.options.other = Object.assign({}, ast.options[cat]); + } + delete ast.options[cat]; + } + }); + + const argName = normalizeArgName(ast.value, type, args); + + const typeText = ast.pluralType === 'ordinal' ? 'selectordinal' : 'plural'; + const offsetText = + ast.offset !== 0 ? ` offset:${ast.offset}` : ''; + const optionsText = Object.entries(ast.options).sort((a, b) => { + if (a[0].startsWith('=') || b[0].startsWith('=')) { + return a[0].localeCompare(b[0]); + } + return sortedCats.indexOf(a[0]) > sortedCats.indexOf(b[0]) ? 1 : -1; + }).map(([opt, { value }]) => { + return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, swapOne, args: [...args] }, level + 1, ast.value)}}`; + }).join('') + (useNewlines ? newline : ''); + + text += `{${argName}, ${typeText},${offsetText}${optionsText}}`; + } + else if (type === 7) { // # + text += '#'; + } + else if (type === 8) { // tag + text += `<${normalizeArgName(ast.value, type, args)}>${printAST(ast.children, options, level + 1)}`; + } + else { // unhandled + console.warn('unhandled type:', type); + } + + return text; +} diff --git a/src/index.js b/src/index.js index f8cfe95..5639098 100644 --- a/src/index.js +++ b/src/index.js @@ -1,2 +1,4 @@ export { Reporter } from './reporter.js'; -export { structureRegEx, validateMessage } from './validate.js'; +export { validateMessage } from './validate.js'; +export { structureRegEx } from './utils.js'; +export { formatMessage } from './format.js'; diff --git a/src/locale-data-default.js b/src/locale-data-default.js new file mode 100644 index 0000000..f9a8c48 --- /dev/null +++ b/src/locale-data-default.js @@ -0,0 +1,209 @@ +export default { + "ar": { + "delimiters": { + "quotationStart": "”", + "quotationEnd": "“", + "alternateQuotationStart": "’", + "alternateQuotationEnd": "‘", + "apostrophe": "’" + } + }, + "cy": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "da": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "de": { + "delimiters": { + "quotationStart": "„", + "quotationEnd": "“", + "alternateQuotationStart": "‚", + "alternateQuotationEnd": "‘", + "apostrophe": "’" + } + }, + "en-gb": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "en": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "es-es": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "es": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "fr-ca": { + "delimiters": { + "quotationStart": "«", + "quotationEnd": "»", + "alternateQuotationStart": "«", + "alternateQuotationEnd": "»", + "apostrophe": "’" + } + }, + "fr": { + "delimiters": { + "quotationStart": "«", + "quotationEnd": "»", + "alternateQuotationStart": "«", + "alternateQuotationEnd": "»", + "apostrophe": "’" + } + }, + "haw": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "'" + } + }, + "hi": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "ja": { + "delimiters": { + "quotationStart": "「", + "quotationEnd": "」", + "alternateQuotationStart": "『", + "alternateQuotationEnd": "』", + "apostrophe": "’" + } + }, + "ko": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "mi": { + "delimiters": { + "quotationStart": "\"", + "quotationEnd": "\"", + "alternateQuotationStart": "\"", + "alternateQuotationEnd": "\"", + "apostrophe": "’" + } + }, + "nl": { + "delimiters": { + "quotationStart": "‘", + "quotationEnd": "’", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "pt": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "sv": { + "delimiters": { + "quotationStart": "”", + "alternateQuotationStart": "’", + "quotationEnd": "”", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "th": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "tr": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’", + } + }, + "vi": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "zh-cn": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, + "zh-tw": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + } +} diff --git a/src/locale-data.js b/src/locale-data.js new file mode 100644 index 0000000..f7204c4 --- /dev/null +++ b/src/locale-data.js @@ -0,0 +1,2 @@ +import localeData from './locale-data-default.js'; +export default localeData; diff --git a/src/reporter.js b/src/reporter.js index 2582c22..f6e3b9b 100644 --- a/src/reporter.js +++ b/src/reporter.js @@ -1,100 +1,100 @@ export function Reporter(locale, fileContents = '') { - this._config = {}; - this._config.locale = locale; - this._config.fileContents = fileContents; - this.report = { totals: { errors: 0, warnings: 0 } }; - this.issues = []; + this._config = {}; + this._config.locale = locale; + this._config.fileContents = fileContents; + this.report = { totals: { errors: 0, warnings: 0 } }; + this.issues = []; } -Reporter.prototype.config = function(targetString, sourceString, key) { - this._config.key = key || targetString.key; - if (typeof targetString !== "undefined") this._config.target = targetString; - if (typeof sourceString !== "undefined") this._config.source = sourceString; +Reporter.prototype.config = function(targetMessage, sourceMessage, key) { + this._config.key = key || targetMessage.key; + if (typeof targetMessage !== "undefined") this._config.target = targetMessage; + if (typeof sourceMessage !== "undefined") this._config.source = sourceMessage; }; Reporter.prototype.log = function(level, type, msg, column = 0, givenLine) { - const levels = level + 's'; - this.report.totals[levels]++; + const levels = level + 's'; + this.report.totals[levels]++; - this.report[levels] = this.report[levels] || {}; - this.report[levels][type] = this.report[levels][type] || 0; - this.report[levels][type]++; + this.report[levels] = this.report[levels] || {}; + this.report[levels][type] = this.report[levels][type] || 0; + this.report[levels][type]++; - const start = Math.max(this._config.fileContents.indexOf(this._config.target), 0) + column - const newlines = this._config.target.split(this._config.target.val)[0].match(/\n/)?.length || 0; - const line = givenLine || this._config.fileContents.substring(0, start).split('\n').length + newlines; + const start = Math.max(this._config.fileContents.indexOf(this._config.target), 0) + column + const newlines = this._config.target.split(this._config.target.val)[0].match(/\n/)?.length || 0; + const line = givenLine || this._config.fileContents.substring(0, start).split('\n').length + newlines; - const issue = { - locale: this._config.locale, - line, - column, - type, - level, - msg, - target: this._config.target.val || this._config.target, - source: this._config.source.val || this._config.source - }; + const issue = { + locale: this._config.locale, + line, + column, + type, + level, + msg, + target: this._config.target?.val ?? this._config.target, + source: this._config.source?.val ?? this._config.source + }; - if (this._config.key) issue.key = this._config.key; + if (this._config.key) issue.key = this._config.key; - this.issues.push(issue); + this.issues.push(issue); - return issue; + return issue; }; Reporter.prototype.warning = function(type, msg, details = {}) { - const relativeColumn = details.column || 0; + const relativeColumn = details.column || 0; - let column, keyPos, linePos, valPos; + let column, keyPos, linePos, valPos; - const cleanTarget = this._config.target - .replace(/\n/g, '\\n') - .replace(/"/g, '\\"'); + const cleanTarget = this._config.target + .replace(/\n/g, '\\n') + .replace(/"/g, '\\"'); - if (['split', 'newline'].includes(type)) { - column = 0; - } - else if (this._config.key) { - keyPos = this._config.fileContents.indexOf(`"${this._config.key}"`); - valPos = this._config.fileContents.indexOf(`"${cleanTarget}"`, keyPos); - linePos = this._config.fileContents.lastIndexOf(String.fromCharCode(10), keyPos); - column = (valPos + 1) - linePos + relativeColumn; + if (['split', 'newline'].includes(type)) { + column = 0; + } + else if (this._config.key) { + keyPos = this._config.fileContents.indexOf(`"${this._config.key}"`); + valPos = this._config.fileContents.indexOf(`"${cleanTarget}"`, keyPos); + linePos = this._config.fileContents.lastIndexOf(String.fromCharCode(10), keyPos); + column = (valPos + 1) - linePos + relativeColumn; - if (valPos === -1) { - // the target string likely contains a backslash that does not escape anything - column = 0; - } - } + if (valPos === -1) { + // the target message likely contains a backslash that does not escape anything + column = 0; + } + } - return this.log('warning', type, msg, column); + return this.log('warning', type, msg, column); }; Reporter.prototype.error = function(type, msg, details = {}) { - const relativeColumn = details.column || 0; - - let column = relativeColumn, - keyPos, line, linePos, valPos; - - if (type === 'missing') { - column = 0; - } - else if (this._config.key) { - // todo: this seems json-specific - const keyQuote = this._config.target.keyQuote; - const valQuote = this._config.target.valQuote; - keyPos = this._config.fileContents.indexOf(`${keyQuote}${this._config.key}${keyQuote}`); - valPos = this._config.fileContents.indexOf(`${valQuote}${this._config.target.val}${valQuote}`, keyPos); - linePos = this._config.fileContents.lastIndexOf(String.fromCharCode(10), keyPos); - column = (valPos + 1) - linePos + relativeColumn; - line = (this._config.fileContents.substring(0, linePos + column).match(/\n/g)?.length ?? -1) + 1; - column -= this._config.target.lastIndexOf('\n', column); - - if (valPos === -1) { - // the target string likely contains a backslash that does not escape anything - column = 0; - } - } - return this.log('error', type, msg, column, line); + const relativeColumn = details.column || 0; + + let column = relativeColumn, + keyPos, line, linePos, valPos; + + if (type === 'missing') { + column = 0; + } + else if (this._config.key) { + // todo: this seems json-specific + const keyQuote = this._config.target.keyQuote; + const valQuote = this._config.target.valQuote; + keyPos = this._config.fileContents.indexOf(`${keyQuote}${this._config.key}${keyQuote}`); + valPos = this._config.fileContents.indexOf(`${valQuote}${this._config.target.val}${valQuote}`, keyPos); + linePos = this._config.fileContents.lastIndexOf(String.fromCharCode(10), keyPos); + column = (valPos + 1) - linePos + relativeColumn; + line = (this._config.fileContents.substring(0, linePos + column).match(/\n/g)?.length ?? -1) + 1; + column -= this._config.target.lastIndexOf('\n', column); + + if (valPos === -1) { + // the target message likely contains a backslash that does not escape anything + column = 0; + } + } + return this.log('error', type, msg, column, line); }; diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..7d4b30e --- /dev/null +++ b/src/utils.js @@ -0,0 +1,48 @@ +import { env } from 'node:process'; +import findConfig from 'find-config'; +import localeData from './locale-data.js'; + +export function getArgs(asts, types = [1,2,3,4,5,6,8], args = []) { + asts.forEach(ast => { + if (types.includes(ast.type)) { + if (ast.type === 8) args.push([ast.value, ast.type]); // account for closing tag + args.push([ast.value, ast.type]); + } + if (ast.options) Object.values(ast.options).map(({ value: asts }) => getArgs(asts, types, args)); + if (ast.children) args.concat(getArgs(ast.children, types, args)); + }) + return args; +} + +export async function getConfig(cwd) { + const configPath = findConfig('mfv.config.json', { cwd }); + const config = configPath ? (await import(`file://${configPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; + config.__configPath = configPath; + + if (config.locales !== undefined && !Array.isArray(config.locales)) { + console.error('locales config must be an array'); + process.exit(1); + } + + return config; +} + +export const sortedCats = ['zero', 'one', 'two', 'few', 'many', 'other']; +export const paddedQuoteLocales = ['fr', 'fr-ca', 'fr-fr', 'fr-on', 'vi-vn']; +export const structureRegEx = /(?<=\s*)(?}](?:[^']*?[{<>}])?))')(''|([{<]([^']|\n)*?[{<>}])|\s*}([^']|\n)*?[{}]|#)/g + +export async function getLocaleData(locale) { + locale = (await getConfig(env.PWD))?.localesMap?.[locale] || locale; + return (localeData[locale] ?? localeData[locale.split('-')[0]] ?? localeData['en']); +} + +export function getPluralCats(locale, pluralType) { + return new Intl.PluralRules(locale, { type: pluralType }).resolvedOptions().pluralCategories; +} + +export function formatList(arr, locale = 'en') { + return new Intl.ListFormat(locale).format(arr); +} + +// sort '.' above '-' +export const sortFn = (a, b) => a.replace(/\./g, '\u0000') >= b.replace(/\./g, '\u0000') ? 1 : -1; diff --git a/src/validate.js b/src/validate.js index 462aa1c..37d03e8 100644 --- a/src/validate.js +++ b/src/validate.js @@ -1,284 +1,313 @@ -import * as pluralCats from 'make-plural/pluralCategories' +import { formatList, getArgs, getPluralCats, sortedCats, structureRegEx } from './utils.js'; import { Reporter } from './reporter.js'; -import { parse } from 'messageformat-parser'; +import { parse } from '@formatjs/icu-messageformat-parser'; -function getPluralCats(locale) { - return pluralCats[locale.split('-')[0]] || pluralCats.en; -} +const SELECT = 5; +const PLURAL = 6; +const ARGUMENT = 1; -export const structureRegEx = /(?<=\s*){(.|\n)*?[{}]|\s*}(.|\n)*?[{}]|[{#]|(\s*)}/g; let reporter; export function validateLocales({ locales, sourceLocale }, localesReporter) { - const sourceStrings = locales[sourceLocale].parsed; + const sourceMessages = locales[sourceLocale].parsed; + + return Object.keys(locales).map((targetLocale) => { + + reporter = localesReporter ?? new Reporter(targetLocale, locales[targetLocale].contents); + const targetMessages = locales[targetLocale].parsed; + const checkedKeys = []; - return Object.keys(locales).map((targetLocale) => { + Object.keys(targetMessages).forEach(key => { - reporter = localesReporter ?? new Reporter(targetLocale, locales[targetLocale].contents); - const targetStrings = locales[targetLocale].parsed; - const checkedKeys = []; + checkedKeys.push(key); + const targetMessage = targetMessages?.[key].val; + const sourceMessage = sourceMessages?.[key]?.val || ''; + const overrides = Array.from(targetMessages?.[key].comment.matchAll(/mfv-(?[a-z]+)/g)).map(m => m.groups.override) - Object.keys(targetStrings).forEach(key => { - checkedKeys.push(key); - const targetString = targetStrings?.[key].val; - const sourceString = sourceStrings?.[key]?.val || ''; - const overrides = Array.from(targetStrings?.[key].comment.matchAll(/mfv-(?[a-z]+)/g)).map(m => m.groups.override) + reporter.config(targetMessages[key], sourceMessages[key]); - reporter.config(targetStrings[key], sourceStrings[key]); + if (!sourceMessage) { + reporter.error('extraneous', 'Message does not exist in the source locale'); + } + else { + if (locales[targetLocale].duplicateKeys.has(key)) reporter.error('duplicate', `Multiple messages named "${key}"`); - if (!sourceString) { - reporter.error('extraneous', 'This string does not exist in the source file.'); - } - else { - if (locales[targetLocale].duplicateKeys.has(key)) reporter.error('duplicate-keys', 'Key appears multiple times'); + validateMessage({ + targetMessage, + targetLocale, + sourceMessage, + sourceLocale, + overrides + }, reporter); + } + }); - validateMessage({ - targetString, - targetLocale, - sourceString, - sourceLocale, - overrides - }, reporter); - } - }); + const missingKeys = Object.keys(sourceMessages).filter(arg => !checkedKeys.includes(arg)); - const missingKeys = Object.keys(sourceStrings).filter(arg => !checkedKeys.includes(arg)); + if (missingKeys.length) { + missingKeys.forEach((key) => { + reporter.config(sourceMessages[key], sourceMessages[key]); + reporter.error('missing', 'Message missing from the target locale') + }) + } - if (missingKeys.length) { - missingKeys.forEach((key) => { - reporter.config(sourceStrings[key], sourceStrings[key]); - reporter.error('missing', `String missing from locale file.`) - }) - } + return { + locale: targetLocale, + issues: reporter.issues || [], + report: reporter.report, + parsed: true + } - return { - locale: targetLocale, - issues: reporter.issues || [], - report: reporter.report, - parsed: true - } + }); +} + +function checkNbsp(message, reporter) { + const structure = message.match(structureRegEx)?.join('') || ''; + const nbspPos = structure.indexOf(String.fromCharCode(160)); - }); + if (nbspPos > -1) { + reporter.error('nbsp', `Message structure contains non-breaking space at position ${nbspPos}`, { column: nbspPos }); + return true; + } } -export function validateMessage({ targetString, targetLocale, sourceString, sourceLocale, overrides }, msgReporter = reporter) { +export function validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale, overrides }, msgReporter = reporter) { const re = /[\u2000-\u206F\u2E00-\u2E7F\n\r\\'!"#$%&()*+,\-.\/∕:;<=>?@\[\]^_`{|}~]/g; // eslint-disable-line - if (sourceLocale - && targetLocale.split('-')[0] !== sourceLocale.split('-')[0] - && targetString.replace(re,'') === sourceString.replace(re,'')) { - - if (!overrides?.includes('translated') - && sourceString - .replace(structureRegEx, '') - .replace(re,'') - .replace(/\s/g, '')) { - - msgReporter.warning('untranslated', `String has not been translated.`); - } - } - - let parsedTarget; - try { - parsedTarget = Object.freeze(parse(targetString, getPluralCats(targetLocale))); - } - catch(e) { - - if (e.message.indexOf('Invalid key') === 0) { - const backtickCaptures = e.message.match(/`([^`]*)`/g); - const badKey = backtickCaptures[0].slice(1, -1); - const pluralArg = backtickCaptures[1].slice(1, -1) - const column = targetString.indexOf(badKey, targetString.indexOf(`{${pluralArg}, plural, {`)); - msgReporter.error('plural-key', e.message, { column }); - } - else if ((targetString.match(/{/g) || 0).length !== (targetString.match(/}/g) || 0).length) { - msgReporter.error('brace', 'Mismatched braces (i.e. {}). ' + e.message, { column: e.location.start.column }); - } - else { - msgReporter.error('parse', e.message, { column: e.location.start.column - 1 }); - } - } - - if (parsedTarget) { - - const targetTokens = parsedTarget; - let sourceTokens; - - try { - sourceTokens = parse(sourceString, getPluralCats(sourceLocale)); - } - catch(e) { - msgReporter.error('source-error', 'Failed to parse source string.'); - return; - } - - const checkCases = target => { - target?.forEach(part => { - if (['select', 'selectordinal', 'plural'].includes(part.type)) { - const hasOther = part.cases.find(c => c.key.trim() === 'other'); - if (!hasOther) { - msgReporter.error('other', 'Missing "other" case'); - } - - if (part.type !== 'select') { - const pluralType = part.type.includes('ordinal') ? 'ordinal' : 'cardinal'; - const allowedCats = getPluralCats(targetLocale)[pluralType]; - const cats = part.cases.map(c => c.key); - const missingCats = allowedCats.filter(c => c !== 'other' && !cats.includes(c)); - if (missingCats.length) msgReporter.warning('categories', `Missing categories: ${JSON.stringify(missingCats)}`); - } - } - checkCases(target.cases); - }); - }; - - checkCases(parsedTarget); - - const targetMap = _map(targetTokens); - const sourceMap = _map(sourceTokens); - - const argDiff = Array.from(targetMap.arguments).filter(arg => !Array.from(sourceMap.arguments).includes(arg)); - - const badArgPos = targetString.indexOf(argDiff[0]); - if (argDiff.length) { - msgReporter.error('argument', `Unrecognized arguments ${JSON.stringify(argDiff)}`, { column: badArgPos }); - } - - // remove all translated content, leaving only the messageformat structure - const structure = targetString.match(structureRegEx)?.join('') || ''; - - const nbspPos = structure.indexOf(String.fromCharCode(160)); - if (nbspPos > -1) { - msgReporter.error('nbsp', `String contains invalid non-breaking space at position ${nbspPos}.`, { column: nbspPos }); - } - - if (targetMap.cases.join(',') !== sourceMap.cases.join(',')) { - - const cleanTargetCases = targetMap.cases.map(c => c.replace(/.+(?<=\|(plural|selectordinal)\|).*/, '')); - const cleanSourceCases = sourceMap.cases.map(c => c.replace(/.+(?<=\|(plural|selectordinal)\|).*/, '')); - const caseDiff = cleanTargetCases.filter(arg => !cleanSourceCases.includes(arg)); - - if (caseDiff.length) { - msgReporter.error('case', `Unrecognized cases ${JSON.stringify(caseDiff.map(c => c.replace(/.+\|select\|/, '')))}`); - } - else if (targetMap.nested && targetMap.cases.length === sourceMap.cases.length) { - // TODO: better identify case order vs nesting order - msgReporter.warning('nest-order', `Nesting order does not match source.`); - } - } - - const firstPlural = targetMap.cases.findIndex(i => i.match(/^.+\|(plural|selectordinal)\|/)) + 1; - const lastSelect = targetMap.cases.findLastIndex(i => i.match(/^.+\|select\|/)) + 1; - - if (targetMap.nested && firstPlural && lastSelect && firstPlural < lastSelect) { - msgReporter.warning('nest-ideal', '"plural" and "selectordinal" should always nest inside "select".'); - } - - if (targetTokens.length > 1) { - if (targetLocale == sourceLocale && targetTokens.find((token) => typeof token !== 'string' && token.type.match(/plural|select/))) { - msgReporter.warning('split','String split by non-argument (e.g. select; plural).') - } - } - } + if (sourceLocale + && targetLocale.split('-')[0] !== sourceLocale.split('-')[0] + && targetMessage.replace(re,'') === sourceMessage.replace(re,'')) { + + if (!overrides?.includes('translated') + && sourceMessage + .replace(structureRegEx, '') + .replace(re,'') + .replace(/\s/g, '')) { + + msgReporter.warning('untranslated', 'Message has not been translated'); + } + } + + let parsedTarget; + try { + parsedTarget = Object.freeze(parse(targetMessage, { captureLocation: true, requiresOtherClause: false })); + } + catch(e) { + if ((targetMessage.match(/{/g) || 0).length !== (targetMessage.match(/}/g) || 0).length) { + msgReporter.error('brace', 'Mismatched braces', { column: e.location.start.column }); + } + else if (!checkNbsp(targetMessage, msgReporter)) { + if (e.message === 'EXPECT_SELECT_ARGUMENT_OPTIONS') { + msgReporter.error('parse', `Expected "," but "${e.originalMessage.substr(e.location.start.column, 1)}" found`, { column: e.location.start.column - 1 }); + } + else { + const near = `at or near: ${targetMessage.slice(e.location.start.offset, Math.max(e.location.end.offset, e.location.start.offset + 4))}`; + msgReporter.error('parse', `${e.message} ${near}`, { column: e.location.start.column - 1 }); + } + } + } + + if (parsedTarget) { + + const targetTokens = parsedTarget; + let sourceTokens; + + try { + sourceTokens = parse(sourceMessage, { requiresOtherClause: false }); + } + catch(e) { + msgReporter.error('source', 'Failed to parse source message'); + return; + } + + const checkCases = (target, msg) => { + target?.forEach(part => { + if ([SELECT, PLURAL].includes(part.type)) { + if (!part.options.other) { + msgReporter.error('other', 'Missing "other" option'); + } + + if (part.type === PLURAL) { + const supportedCats = getPluralCats(targetLocale, part.pluralType); + const cats = Object.keys(part.options); + const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); + + if (missingCats.includes('one') && cats.includes('=1')) { + // locales where `one` can match something other than 1 + const oneIsNot1Regex = /^(fr|da|hi|pt(?!-pt))(-?|$)/; + if (!oneIsNot1Regex.test(targetLocale)) { + missingCats.splice(missingCats.indexOf('one'), 1); + } + } + + if (missingCats.length) msgReporter.warning('category-missing', `Missing ${missingCats.length === 1 ? 'category' : 'categories'} ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); + const unsupportedCats = cats.filter(c => !/^=\d+$/.test(c) && !supportedCats.includes(c)); + + unsupportedCats.forEach(cat => { + const column = part.options[cat].location.start.offset; + msgReporter.error('category', `Unsupported category "${cat}". Locale supports "${supportedCats.join('", "')}", and explicit keys like "=0".`, { column }); + }); + } + + Object.values(part.options).forEach(o => { + checkCases(o.value, msg); + }); + } + }); + }; + + checkCases(parsedTarget, targetMessage); + + const targetMap = _map(targetTokens); + const sourceMap = _map(sourceTokens); + + const srcArgs = new Set(getArgs(sourceTokens).map(e => e.join(''))); + const tgtArgs = new Set(getArgs(targetTokens).map(e => e.join(''))); + + const argDiff = Array.from(tgtArgs).filter(arg => !Array.from(srcArgs).includes(arg)); + + const badArgPos = targetMessage.indexOf(argDiff[0]); + if (argDiff.length) { + msgReporter.error('argument', `Unrecognized ${argDiff.length === 1 ? 'argument' : 'arguments'} ${formatList(argDiff.map(i => `"${i.slice(0, -1)}"`))}. Source message uses ${formatList([...new Set([...srcArgs].map(a => a.slice(0, -1)))].map(i => `"${i}"`))}.`, { column: badArgPos }); + } + + checkNbsp(targetMessage, msgReporter); + + if (targetMap.cases.join(',') !== sourceMap.cases.join(',')) { + + const cleanTargetCases = targetMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); + const cleanSourceCases = sourceMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); + + const extraOptions = cleanTargetCases.filter(arg => !cleanSourceCases.includes(arg)); + extraOptions.forEach(o => { + msgReporter.error('option', `Unrecognized option "${o.replace(/.+\|5undefined\|/, '')}". Argument uses ${formatList(cleanSourceCases.map(o => `"${o.replace(/.+\|5undefined\|/, '')}"`))}.`); + }); + + const missingOptions = cleanSourceCases.filter(arg => !cleanTargetCases.includes(arg)); + missingOptions.forEach(o => { + o = o.replace(/.+\|5undefined\|/, ''); + o !== 'other' && msgReporter.error('option-missing', `Missing option "${o}"`); + }); + + if (targetMap.nested && targetMap.cases.length === sourceMap.cases.length) { + // TODO: better identify case order vs nesting order + msgReporter.warning('nest-order', 'Nesting order does not match source'); + } + } + + const firstPlural = targetMap.cases.findIndex(i => i.match(/^.+\|(6(ordinal|cardinal))\|/)) + 1; + const lastSelect = targetMap.cases.findLastIndex(i => i.match(/^.+\|5undefined\|/)) + 1; + + if (targetMap.nested && firstPlural && lastSelect && firstPlural < lastSelect) { + msgReporter.warning('nest-ideal', '"plural" and "selectordinal" should always nest inside "select"'); + } + + if (targetTokens.length > 1) { + if (targetLocale == sourceLocale && targetTokens.find((token) => typeof token !== 'string' && [PLURAL, SELECT].includes(token.type))) { + msgReporter.warning('split','Message split by complex argument') + } + } + } } export function parseLocales(locales, useJSONObj) { - return locales.reduce((acc, { contents, file }) => { - const locale = file.split('.')[0]; - acc[locale] = { - contents, - duplicateKeys: new Set(), - parsed: {}, - file - }; - - const regex = useJSONObj - //[ ][ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] - ? /("(?.*)"(\s*):(\s*){)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?,?)(?.*)/g - //[ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] - : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?,?)(?.*)/g; - const matches = Array.from(contents.matchAll(regex)); - - let findContext = false; - let findValue = false; - - matches.forEach(match => { - - if (useJSONObj) { - if (findContext && match.groups.key === 'context') { - acc[locale].parsed[findContext].comment = match.groups.val; - findContext = false; - return; - } - - if (findValue && match.groups.key === 'translation') { - acc[locale].parsed[findValue].val = match.groups.val; - findValue = false; - return; - } - - if (match.groups.realKey) { - - if (match.groups.key === 'translation') { - findContext = match.groups.realKey; - } - if (match.groups.key === 'context') { - match.groups.comment = match.groups.val; - findValue = match.groups.realKey; - } - - match.groups.key = match.groups.realKey; - } - } - - if (!acc[locale].parsed[match.groups.key]) { - acc[locale].parsed[match.groups.key] = Object.assign(String(match[0]), match.groups); - } - else { - acc[locale].duplicateKeys.add(match.groups.key); - } - }); - return acc; - }, {}); + return locales.reduce((acc, { contents, file }) => { + const locale = file.split('.')[0]; + acc[locale] = { + contents, + duplicateKeys: new Set(), + parsed: {}, + file + }; + + const regex = useJSONObj + ? /("(?.*)"(\s*):(\s*)\{)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(?:[^\\]|\\[\s\S])*?)\k(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/gv + //[ ][ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] + : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(?:[^\\]|\\[\s\S])*?)\k(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/gv; + //[ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] + + const matches = Array.from(contents.matchAll(regex)); + + let findContext = false; + let findValue = false; + + matches.forEach(match => { + + if (useJSONObj) { + if (findContext && match.groups.key === 'context') { + acc[locale].parsed[findContext].comment = match.groups.val; + findContext = false; + return; + } + + if (findValue && match.groups.key === 'translation') { + acc[locale].parsed[findValue].val = match.groups.val; + findValue = false; + return; + } + + if (match.groups.realKey) { + + if (match.groups.key === 'translation') { + findContext = match.groups.realKey; + } + if (match.groups.key === 'context') { + match.groups.comment = match.groups.val; + findValue = match.groups.realKey; + } + + match.groups.key = match.groups.realKey; + } + } + + if (!acc[locale].parsed[match.groups.key]) { + acc[locale].parsed[match.groups.key] = Object.assign(String(match[0]), match.groups); + } + else { + acc[locale].duplicateKeys.add(match.groups.key); + } + }); + return acc; + }, {}); } -function _map(tokens, partsMap = { nested: false, arguments: new Set(), cases: [], stringTokens: [] }) { - - tokens.forEach(token => { +function _map(ast, partsMap = { nestIdeal: false, arguments: new Set(), cases: [], messageTokens: [] }) { + //console.log(ast); + //console.log(JSON.stringify(partsMap, null, '\t')); + ast.forEach(token => { - if (typeof token !== 'string') { + if (typeof token !== 'string') { - if (token.arg) { - partsMap.arguments.add(token.arg); - } + if (token.type === ARGUMENT) { + partsMap.arguments.add(token.value); + } - if (token.cases) { + if (token.options) { - if (partsMap.cases.length) { - partsMap.nested = true; - } + if (partsMap.cases.length) { + partsMap.nested = true; + } - token.cases.forEach((case_) => { - switch (token.type) { - case 'select': - case 'plural': - case 'selectordinal': - partsMap.cases.push(`${token.arg}|${token.type}|${case_.key}`); - break; - } + Object.entries(token.options).forEach(([k, option]) => { + switch (token.type) { + case SELECT: + case PLURAL: + partsMap.cases.push(`${token.value}|${token.type}${token.pluralType}|${k}`); + break; + } - _map(case_.tokens, partsMap); - }); - } - } - else { - partsMap.stringTokens.push(token); - } + _map(option.value, partsMap); + }); + } + } + else { + partsMap.messageTokens.push(token); + } - }); + }); - return partsMap; + return partsMap; } diff --git a/test.js b/test.js index df25e8f..1967d0b 100644 --- a/test.js +++ b/test.js @@ -12,16 +12,16 @@ Object.keys(locales).forEach(key => { */ const locales = { - en: fs.readFileSync('./test/locales/json/bigfiles/en.json', 'utf8'), - fr: fs.readFileSync('./test/locales/json/bigfiles/fr.json', 'utf8'), - ar: fs.readFileSync('./test/locales/json/bigfiles/ar.json', 'utf8'), - es: fs.readFileSync('./test/locales/json/bigfiles/es.json', 'utf8') + String.fromCharCode(65279) + en: fs.readFileSync('./test/locales/json/bigfiles/en.json', 'utf8'), + fr: fs.readFileSync('./test/locales/json/bigfiles/fr.json', 'utf8'), + ar: fs.readFileSync('./test/locales/json/bigfiles/ar.json', 'utf8'), + es: fs.readFileSync('./test/locales/json/bigfiles/es.json', 'utf8') + String.fromCharCode(65279) } const issues = validateLocales({ - locales, - sourceLocale: 'en' + locales, + sourceLocale: 'en' }); console.log(JSON.stringify(issues, null, 2)); // eslint-disable-line no-console diff --git a/test/format.test.js b/test/format.test.js new file mode 100644 index 0000000..ab7efbd --- /dev/null +++ b/test/format.test.js @@ -0,0 +1,281 @@ +import { expect } from 'chai'; +import { formatMessage } from '../src/format.js'; + +describe('formatMessage', () => { + + let locale; + beforeEach(() => { + locale = 'en'; + }); + + [ + { locale: 'ar', expected: `This isn’t ”correct“.` }, + { locale: 'cy', expected: `This isn’t “correct”.` }, + { locale: 'de', expected: `This isn’t „correct“.` }, + { locale: 'en', expected: `This isn’t “correct”.` }, + { locale: 'en-gb', expected: `This isn’t ‘correct’.` }, + { locale: 'fr', expected: `This isn’t «\u202fcorrect\u202f».` }, + { locale: 'haw', expected: `This isn't “correct”.` }, + { locale: 'sv', expected: `This isn’t ”correct”.` }, + { locale: 'mi', expected: `This isn’t "correct".` }, + ].forEach(({ locale, expected }) => { + it(`should replace straight quotes with "${locale}" quotes when "quotes" option is "straight"`, async() => { + const message = `This isn't "correct".`; + const formatted = await formatMessage(message, { locale, sourceLocale: 'en', quotes: 'straight' }); + expect(formatted).to.equal(expected); + }); + + it(`should replace source quotes with "${locale}" quotes when "quotes" option is "source"`, async() => { + const message = `This isn’t “correct”.`; + const formatted = await formatMessage(message, { locale, sourceLocale: 'en', quotes: 'source' }); + expect(formatted).to.equal(expected); + }); + + it(`should not alter own "${locale}" quotes when "quotes" option is "source"`, async() => { + const formattedSelf = await formatMessage(expected, { locale, sourceLocale: 'en', quotes: 'source' }); + expect(formattedSelf).to.equal(expected); + }); + }); + + it('should identifiy apostrophes', async () => { + const message = `{evidenceTitle} collected {dateCollected} - {studentName}'s Portfolio`; + const expected = `{evidenceTitle} collected {dateCollected} - {studentName}’s Portfolio`; + const formatted = await formatMessage(message, { locale: 'en', sourceLocale: 'en' }); + expect(formatted).to.equal(expected); + }); + + [ + { condition: 'no options are set', options: {} }, + { condition: 'the "quotes" option is "straight"', options: { quotes: 'straight' } } + ].forEach(({ condition, options }) => { + it(`should preserve escapes when ${condition}`, async() => { + const message = `An '{escaped}' argument and an {argument}'s escape`; + const expected = `An '{escaped}' argument and an {argument}’s escape`; + const formatted = await formatMessage(message, { locale: 'en', sourceLocale: 'en', ...options }); + expect(formatted).to.equal(expected); + }); + }); + + it('should convert apostrophes in plain text without arguments', async() => { + const message = `It's a nice day`; + const formatted = await formatMessage(message, { locale: 'en' }); + expect(formatted).to.equal(`It’s a nice day`); + }); + + it('should convert apostrophes followed by whitespace', async() => { + const message = `{students}' assignments' titles`; + const formatted = await formatMessage(message, { locale: 'en' }); + expect(formatted).to.equal(`{students}’ assignments’ titles`); + }); + + [ + { locale: 'ar', expected: +`{a, plural, + one {{b, selectordinal, + other {} + }} + two {} + few {} + many {} + other {} +}` }, + { locale: 'cy', expected: +`{a, plural, + one {{b, selectordinal, + one {} + two {} + few {} + many {} + other {} + }} + two {} + few {} + many {} + other {} +}` }, + { locale: 'es', expected: +`{a, plural, + one {{b, selectordinal, + other {} + }} + many {} + other {} +}` }, + { locale: 'fr', expected: +`{a, plural, + one {{b, selectordinal, + one {} + other {} + }} + many {} + other {} +}` }, + { locale: 'ja', expected: +`{a, plural, + other {} +}` }, + ].forEach(({ locale, expected }) => { + it(`should remove plural and selectordinal categories that are unsupported in "${locale}" with the "remove" option`, async() => { + const message = +`{a, plural, + one {{b, selectordinal, one {} two {} few {} many {} other {}}} + two {} + few {} + many {} + other {} +}`; + const formatted = await formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + }); + + it(`should not remove plural and selectordinal categories that are unsupported without the "remove" option`, async() => { + const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; + const formatted = await formatMessage(message, { locale }); + expect(formatted).to.equal(message); + }); + + it(`should insert newslines and tabs with the "newlines" option`, async() => { + const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; + const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; + const formatted = await formatMessage(message, { locale, newlines: true }); + expect(formatted).to.equal(expected); + }); + + it(`should insert newslines and tabs if the message structure already contains newlines with no option`, async() => { + const message = `{a, plural,\none {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; + const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; + const formatted = await formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it(`should remove categories that are copies of a lower-precedence key with the "dedupe" option`, async() => { + const message = `{a, plural, one {value} other {value}}`; + const expected = `{a, plural, other {value}}`; + const formatted = await formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should not convert "=1" keys to "one" by default`, async() => { + const message = `{a, plural, =1 {value}}`; + const expected = `{a, plural, =1 {value}}`; + const formatted = await formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should not convert "=1" keys to "one" with an argument`, async() => { + const message = `{a, plural, =1 {{value}}}`; + const expected = `{a, plural, =1 {{value}}}`; + const formatted = await formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert "=1" keys to "one" when it contains a literal "1"`, async() => { + const message = `{a, plural, =1 {value 1}}`; + const expected = `{a, plural, one {value {a}}}`; + const formatted = await formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should not convert "=1" keys to "one" with an offset`, async() => { + const message = `{a, plural, offset:2 =1 {value 1}}`; + const expected = `{a, plural, offset:2 =1 {value 1}}`; + const formatted = await formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert "=1" keys to "one" when the only differences from "other" are "s" or "es"`, async() => { + const message = `{a, plural, =1 {Some value} other {Some values}}`; + const expected = `{a, plural, one {Some value} other {Some values}}`; + const formatted = await formatMessage(message, { locale, sourceLocale: 'en' }); + expect(formatted).to.equal(expected); + }); + + it(`should remove "=1" cases when they can be converted to a duplicate case with the "dedupe" option`, async() => { + const message = `{a, plural, =1 {value 1} other {value {a}}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = await formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should remove "=1" cases when they can be converted to unsupported "one" cases with the "remove" option`, async() => { + locale = 'ja'; + const message = `{a, plural, =1 {value 1} other {value {a}}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = await formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert unsupported cases to "other" if there are no other cases`, async() => { + locale = 'ja'; + const message = `{a, plural, two {value {a}}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = await formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert "=1" keys to "other" keys when they can be converted to unsupported "one" cases and there are no other keys with the "remove" option`, async() => { + locale = 'ja'; + const message = `{a, plural, =1 {value 1}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = await formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + + it('should hoist selectors to the outside and nest appropriately with no options', async() => { + const message = `\t{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; + const expected = `{a, plural, =1 {{b, plural, =1 {\ta cat and a dog!} other {\ta cat and {b} dogs!}}} other {{b, plural, =1 {\t{a} cats and a dog!} other {\t{a} cats and {b} dogs!}}}}`; + const formatted = await formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it('should not hoist selectors with the "trim" option', async() => { + const message = `\t{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; + const expected = `{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; + const formatted = await formatMessage(message, { locale, trim: true }); + expect(formatted).to.equal(expected); + }); + + it(`should trim whitespace with the "trim" option`, async() => { + const message = `\n{a, plural, other { value }}\t`; + const expected = `{a, plural, other {value}}`; + const formatted = await formatMessage(message, { locale, trim: true }); + expect(formatted).to.equal(expected); + }); + + it(`should not trim internal whitespace with the "trim" option`, async() => { + const message = `\n{a, plural, other { value {value2} value3 }}`; + const expected = `{a, plural, other {value {value2} value3}}`; + const formatted = await formatMessage(message, { locale, trim: true }); + expect(formatted).to.equal(expected); + }); + + it(`should replace bad plural categories with no options`, async() => { + const message = `{a, plural, one {b} अन्य {च}}`; + const expected = `{a, plural, one {b} other {च}}`; + const formatted = await formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it(`should not replace bad plural categories when ambiguous`, async() => { + const message = `{a, plural, अन्य {च}}`; + const expected = `{a, plural, अन्य {च}}`; + const formatted = await formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it(`should replace hashes`, async() => { + const message = `{a, plural, one {There is # thing}}`; + const expected = `{a, plural, one {There is {a} thing}}`; + const formatted = await formatMessage(message, { locale, expandHashes: true }); + expect(formatted).to.equal(expected); + }); + + it(`should not replace hashes if there is an offset`, async() => { + const message = `{a, plural, offset:1 one {There is # thing}}`; + const expected = `{a, plural, offset:1 one {There is # thing}}`; + const formatted = await formatMessage(message, { locale, expandHashes: true }); + expect(formatted).to.equal(expected); + }); + +}); diff --git a/test/validate.test.js b/test/validate.test.js index 83fac3a..4d68dc2 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -1,4 +1,4 @@ -import { parseLocales, structureRegEx, validateLocales, validateMessage } from '../src/validate.js'; +import { parseLocales, validateLocales, validateMessage } from '../src/validate.js'; import { Reporter } from '../src/reporter.js'; import { expect } from 'chai'; @@ -11,236 +11,337 @@ describe('validate', () => { reporter = new Reporter(targetLocale); }); - describe('structureRegEx', () => { - - [{ - name: 'simple argument', - message: 'abc{def}hij', - structure: '{def}' - }, - { - name: 'multiple simple arguments', - message: 'abc{def}hij {klm} nop {qrs}', - structure: '{def}{klm}{qrs}' - }].forEach(({ name, message, structure }) => { - it(`captures messageformat structure - ${name}`, () => { - expect(message.match(structureRegEx).join('')).to.equal(structure); - }); - }) - }) - describe('validateMessage', () => { // untranslated it('generates no issues with identical same-language messages', () => { - const sourceString = 'An {arg}'; - const targetString = 'An {arg}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arg}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(0); }); - it('generates an untranslated warning when messages are the same and languages are different', () => { + it('generates an "untranslated" warning when messages are the same and languages are different', () => { targetLocale = 'es-mx'; - const sourceString = 'An {arg}'; - const targetString = 'An {arg}'; - reporter.config(targetString, sourceString, 'key'); + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arg}'; + reporter.config(targetMessage, sourceMessage, 'key'); reporter._config.locale = targetLocale; - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('untranslated'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('String has not been translated.'); + expect(reporter.issues[0].msg).to.equal('Message has not been translated'); }); - // categories + // category - it('generates a categories warning when a target message is missing supported plural categories', () => { + it('generates a "category-missing" warning when a target message is missing supported plural categories', () => { targetLocale = 'cy-gb'; - const sourceString = '{a, plural, one {} other {}}'; - const targetString = '{a, plural, one {} other {}}'; - reporter.config(targetString, sourceString, 'key'); + const sourceMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); reporter._config.locale = targetLocale; - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].type).to.equal('category-missing'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Missing categories: ["zero","two","few","many"]'); + expect(reporter.issues[0].msg).to.equal('Missing categories "zero", "one", "two", "few", and "many"'); }); - // plural-key + it('does not generate a "category-missing" warning for `one` in a target locale where `one` can only match 1 and `=1` exists', () => { + targetLocale = 'es'; + const sourceMessage = '{a, plural, =1 {} other {}}'; + const targetMessage = '{a, plural, =1 {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + reporter._config.locale = targetLocale; + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('category-missing'); + expect(reporter.issues[0].level).to.equal('warning'); + expect(reporter.issues[0].msg).to.equal('Missing category "many"'); + }); + + it('generates "category" errors when a target message uses unsupported plural categories', () => { + const sourceMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, =1 {} one {} two {} few {} many {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(3); + + expect(reporter.issues[0].type).to.equal('category'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Locale supports "one", "other", and explicit keys like "=0".'); + + expect(reporter.issues[1].type).to.equal('category'); + expect(reporter.issues[1].level).to.equal('error'); + expect(reporter.issues[1].msg).to.equal('Unsupported category "few". Locale supports "one", "other", and explicit keys like "=0".'); + + expect(reporter.issues[2].type).to.equal('category'); + expect(reporter.issues[2].level).to.equal('error'); + expect(reporter.issues[2].msg).to.equal('Unsupported category "many". Locale supports "one", "other", and explicit keys like "=0".'); + }); + + it('generates "category" errors when a target message uses nested unsupported plural categories', () => { + const sourceMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, one {{a, plural, one {} two {} other {}}} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - it('generates a plural-key error when a target message uses unsupported plural categories', () => { - const sourceString = '{a, plural, one {} other {}}'; - const targetString = '{a, plural, one {} two {} few {} many {} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('plural-key'); + expect(reporter.issues[0].type).to.equal('category'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Invalid key `two` for argument `a`. Valid plural keys for this locale are `one`, `other`, and explicit keys like `=0`.'); + expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Locale supports "one", "other", and explicit keys like "=0".'); }); // split - it('generates a plural-key error when a source message is split by a complex arguemnt', () => { + it('generates a "split" error when a source message is split by a complex argument', () => { targetLocale = 'en'; - const sourceString = '{a, plural, one {} other {}} b'; - const targetString = '{a, plural, one {} other {}} b'; - reporter.config(targetString, sourceString, 'key'); + const sourceMessage = '{a, plural, one {} other {}} b'; + const targetMessage = '{a, plural, one {} other {}} b'; + reporter.config(targetMessage, sourceMessage, 'key'); reporter._config.locale = targetLocale; - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('split'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('String split by non-argument (e.g. select; plural).'); + expect(reporter.issues[0].msg).to.equal('Message split by complex argument'); }); // arg - it('generates an argument error with unrecognized argument', () => { - const sourceString = 'An {arg}'; - const targetString = 'An {arG}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); + it('generates an "argument" error with unrecognized argument', () => { + const sourceMessage = 'An {arg} {arg} {arg2, plural, one {{arg2}} other {}}'; + const targetMessage = 'An {arG} {arg} {arg2}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(3); expect(reporter.issues[0].type).to.equal('argument'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized arguments ["arG"]'); + expect(reporter.issues[0].msg).to.equal('Unrecognized argument "arG". Source message uses "arg" and "arg2".'); + }); + + it('does not generate an "argument" error with repeat arguments', () => { + const sourceMessage = '{groupAmount, plural, one {{arg1}} other {{arg1}}}'; + const targetMessage = '{groupAmount, plural, zero {{arg1}} one {{arg1}} two {{arg1}} few {{arg1}} many {{arg1}} other {{arg1}}}' + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale: 'ar', sourceMessage, sourceLocale }, reporter); + console.log(reporter.issues); + expect(reporter.issues.length).to.equal(0); }); // brace - it.skip('does not generate a brace error with parseable mismatched braces', () => { - const sourceString = 'An {arg}'; - const targetString = 'An {arg}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it('does not generate a "brace" error with parseable mismatched braces', () => { + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arg}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(0); }); - it('generates a brace error with unparseable mismatched braces', () => { - const sourceString = '{a, plural, one {An {arg}} other {}}'; - const targetString = '{a, plural, one {An {arg} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it('generates a "brace" error with unparseable mismatched braces', () => { + const sourceMessage = '{a, plural, one {An {arg}} other {{a} args}}'; + const targetMessage = '{a, plural, one {An {arg} other {{a} args}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('brace'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Mismatched braces (i.e. {}). Expected identifier but "}" found.'); + expect(reporter.issues[0].msg).to.equal('Mismatched braces'); }); - it('does not generate a brace error with escaped mismatched braces', () => { - const sourceString = '{a, plural, one {An {arg}} other {}}'; - const targetString = '{a, plural, one {An {arg}\'}\'} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it('does not generate a "brace" error with escaped mismatched braces', () => { + const sourceMessage = '{a, plural, one {An {arg}} other {}}'; + const targetMessage = '{a, plural, one {An {arg}\'}\'} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(0); }); - // case + // option - it('generates a case error with unrecognized cases in select arguments', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select, b {} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('case'); + it('generates "option" errors with unrecognized options in select arguments', () => { + const sourceMessage = '{a, select, b {} other {}}'; + const targetMessage = '{a, select, b {} c {} d {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(2); + + expect(reporter.issues[0].type).to.equal('option'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized cases ["b"]'); + expect(reporter.issues[0].msg).to.equal('Unrecognized option "c". Argument uses "b" and "other".'); + + expect(reporter.issues[1].type).to.equal('option'); + expect(reporter.issues[1].level).to.equal('error'); + expect(reporter.issues[1].msg).to.equal('Unrecognized option "d". Argument uses "b" and "other".'); }); - it.skip('generates a case error with missing cases in select arguments', () => { - const sourceString = '{a, select, b {} other {}}'; - const targetString = '{a, select, other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + + + it('generates an "option-missing" error with missing options in select arguments', () => { + const sourceMessage = '{a, select, b {} other {}}'; + const targetMessage = '{a, select, other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('case'); + expect(reporter.issues[0].type).to.equal('option-missing'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Missing cases ["b"]'); + expect(reporter.issues[0].msg).to.equal('Missing option "b"'); }); // nbsp - it('generates an nbsp error with non-breaking space in the messageformat structure', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select,\u00A0other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(2); + it('generates an "nbsp" error with non-breaking space in the messageformat structure', () => { + const sourceMessage = '{a, select, a {} other {}}'; + const targetMessage = '{a, select,\u00A0a {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('nbsp'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('String contains invalid non-breaking space at position 11.'); - - expect(reporter.issues[1].type).to.equal('case'); - expect(reporter.issues[1].level).to.equal('error'); - expect(reporter.issues[1].msg).to.equal('Unrecognized cases ["\u00A0other"]'); + expect(reporter.issues[0].msg).to.equal('Message structure contains non-breaking space at position 11'); }); // nest - it('generates a nest-order error with mismatched complex argument order', () => { - const sourceString = '{a, select, other {{b, select, other {}}}}'; - const targetString = '{b, select, other {{a, select, other {}}}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it('generates a "nest-order" error with mismatched complex argument order', () => { + const sourceMessage = '{a, select, other {{b, select, other {}}}}'; + const targetMessage = '{b, select, other {{a, select, other {}}}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('nest-order'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Nesting order does not match source.'); + expect(reporter.issues[0].msg).to.equal('Nesting order does not match source'); }); - it('generates a nest-ideal error with plural inside select', () => { - const sourceString = '{a, plural, one {} other {{b, select, other {}}}}'; - const targetString = '{a, plural, one {} other {{b, select, other {}}}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it.skip('does not generate a "nest-order" error with non-nested messages', () => { + const sourceMessage = '{a, select, other {}} {b, select, other {}}'; + const targetMessage = '{b, select, other {}} {a, select, other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + console.log(reporter); + expect(reporter.issues.length).to.equal(0); + }); + + it('generates a "nest-ideal" error with select inside plural', () => { + const sourceMessage = `{outcomes, select, + competencies {{num, plural, + one {Remove lower-level competency?} + other {Remove lower-level competencies?} + }} + expectations {{num, plural, + one {Remove lower-level expectation?} + other {Remove lower-level expectations?} + }} + objectives {{num, plural, + one {Remove lower-level objective?} + other {Remove lower-level objectives?} + }} + outcomes {{num, plural, + one {Remove lower-level outcome?} + other {Remove lower-level outcomes?} + }} + standards {{num, plural, + one {Remove lower-level standard?} + other {Remove lower-level standards?} + }} + other {{num, plural, + one {Remove lower-level standard?} + other {Remove lower-level standards?} + }} + }`; + const targetMessage = `{outcomes, select, + competencies {{num, plural, + one {Remove lower-level competency?} + other {Remove lower-level competencies?} + }} + expectations {{num, plural, + one {Remove lower-level expectation?} + other {Remove lower-level expectations?} + }} + objectives {{num, plural, + one {Remove lower-level objective?} + other {Remove lower-level objectives?} + }} + outcomes {{num, plural, + one {Remove lower-level outcome?} + other {Remove lower-level outcomes?} + }} + standards {{num, plural, + one {Remove lower-level standard?} + other {Remove lower-level standards?} + }} + other {{num, plural, + one {Remove lower-level standard?} + other {Remove lower-level standards?} + }} + }`; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('nest-ideal'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('"plural" and "selectordinal" should always nest inside "select".'); + expect(reporter.issues[0].msg).to.equal('"plural" and "selectordinal" should always nest inside "select"'); }); // other - it('generates an other error with missing other case', () => { - const sourceString = '{a, select, b {}}'; - const targetString = '{a, select, b {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it('generates an "other" error with missing other case', () => { + const sourceMessage = '{a, select, b {}}'; + const targetMessage = '{a, select, b {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('other'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Missing "other" case'); + expect(reporter.issues[0].msg).to.equal('Missing "other" option'); + }); + + it('generates an "other" error with missing nested other case', () => { + const sourceMessage = '{a, select, other {{c, select, b {}}}}'; + const targetMessage = '{a, select, other {{c, select, b {}}}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('other'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Missing "other" option'); }); // parse - it('generates a parse error with an unparseable target message', () => { - const sourceString = '{a, select, b {}}'; - const targetString = '{a, select b {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it('generates a "parse" error with an unparseable target message', () => { + const sourceMessage = '{a, select, b {}}'; + const targetMessage = '{a, select b {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('parse'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found.'); + expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found'); }); // source - it('generates a source-error error an unparseable source message', () => { - const sourceString = '{a, select b {}}'; - const targetString = '{a, select, b {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it('generates a "source" error an unparseable source message', () => { + const sourceMessage = '{a, select other {}}'; + const targetMessage = '{a, select, other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('source-error'); + expect(reporter.issues[0].type).to.equal('source'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Failed to parse source string.'); + expect(reporter.issues[0].msg).to.equal('Failed to parse source message'); }); }); @@ -249,20 +350,20 @@ describe('validate', () => { // extraneous - it('generates an extraneous error with unexpected message in target locale', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select, other {}}'; + it('generates an "extraneous" error with unexpected message in target locale', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, other {}}'; const locales = parseLocales([{ file: `${targetLocale}.json`, contents: JSON.stringify({ - a: targetString, - b: targetString + a: targetMessage, + b: targetMessage }, null, '\t') }, { file: `${sourceLocale}.json`, contents: JSON.stringify({ - a: sourceString + a: sourceMessage }, null, '\t') }]); @@ -270,25 +371,25 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('extraneous'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('This string does not exist in the source file.'); + expect(reporter.issues[0].msg).to.equal('Message does not exist in the source locale'); }); // missing - it('generates a missing error with missing message in the target locale', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select, other {}}'; + it('generates a "missing" error with missing message in the target locale', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, other {}}'; const locales = parseLocales([{ file: `${targetLocale}.json`, contents: JSON.stringify({ - a: targetString + a: targetMessage }, null, '\t') }, { file: `${sourceLocale}.json`, contents: JSON.stringify({ - a: sourceString, - b: sourceString + a: sourceMessage, + b: sourceMessage }, null, '\t') }]); @@ -296,33 +397,50 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('missing'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('String missing from locale file.'); + expect(reporter.issues[0].msg).to.equal('Message missing from the target locale'); }); // duplicate-keys - it('generates a duplicate-keys error with duplicate messages in the target locale', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select, other {}}'; + it('generates a "duplicate-keys" error with duplicate messages in the target locale', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, other {}}'; const locales = parseLocales([{ file: `${targetLocale}.json`, contents: `{ - "a": "${sourceString}", - "a": "${sourceString}" - }` + "a": "${sourceMessage}", + a: "${sourceMessage}" + }` }, { file: `${sourceLocale}.json`, contents: `{ - "a": "${targetString}" - }` + "a": "${targetMessage}" + }` }]); validateLocales({ locales, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('duplicate-keys'); + expect(reporter.issues[0].type).to.equal('duplicate'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Key appears multiple times'); + expect(reporter.issues[0].msg).to.equal('Multiple messages named "a"'); + }); + + }); + + describe('parseLocales', () => { + + it('correctly parses a value that is a double backslash (\\\\)', () => { + const locales = parseLocales([{ + file: 'en.json', + contents: `{ + "intl-common:characters:backslash": "\\\\", // short name or description of the "\\" character +}` + }]); + + expect(locales['en'].parsed['intl-common:characters:backslash']).to.exist; + expect(locales['en'].parsed['intl-common:characters:backslash'].val).to.equal('\\\\'); + expect(locales['en'].parsed['intl-common:characters:backslash'].comment).to.equal(' // short name or description of the "\\" character'); }); });