From 9630b70acdb3e9a8e02438afd048a05cd8268290 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 6 Sep 2024 10:15:50 -0400 Subject: [PATCH 01/90] Add format --- bin/cli.js | 70 ++++++++++ package-lock.json | 346 ++++++++++++++++++++++++++++++++++++---------- package.json | 4 +- src/format.js | 220 +++++++++++++++++++++++++++++ src/validate.js | 8 +- 5 files changed, 567 insertions(+), 81 deletions(-) create mode 100644 src/format.js diff --git a/bin/cli.js b/bin/cli.js index f716fcf..8e63c4e 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -6,6 +6,7 @@ import { parseLocales, structureRegEx, validateLocales } from '../src/validate.j import { readFile, readdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; import findConfig from 'find-config'; +import { formatMessage } from '../src/format.js' import glob from 'glob'; import pkg from '../package.json' with { type: 'json' }; import { program } from 'commander'; @@ -58,6 +59,23 @@ program program.newKey = newKey; }); +program + .command('format') + .description('Rewrite messages to a standard format') + .option('-n, --newlines', 'When formatting complex arguments, use newlines and indentation for readability') + .option('-a, --add', 'Add cases for missing supported pural and selectordinal categories') + .option('-r, --remove', 'Remove cases for unsupported pural and selectordinal categories') + .option('-d, --dedupe', 'Remove complex argument cases that duplicate the `other` case. Takes precedence over --add.') + .action(function() { + program.format = true; + const opts = this.opts(); + program.newlines = opts.newlines; + program.add = opts.add; + program.remove = opts.remove; + console.log(opts); + program.dedupe = opts.dedupe + }); + program .command('highlight ') .description('Output a string with all non-translatable ICU MessageFormat structure highlighted') @@ -104,6 +122,11 @@ localesPaths.forEach(async localesPath => { console.log(`Renaming "${program.oldKey}" to "${program.newKey}" in:`, targetLocales.join(', ')); } + if (program.format) { + if (!sourceLocale) noSource(); + console.log(`Formatting:`, targetLocales.join(', ')); + } + const resources = await Promise.all(filteredFiles.map(file => readFile(absLocalesPath + file, 'utf8'))) .then(readFiles => readFiles.map((contents, idx) => ({ file: filteredFiles[idx], @@ -151,6 +174,53 @@ localesPaths.forEach(async localesPath => { return; } + if (program.format) { + let count = 0; + + const sourceLocaleParsed = locales[sourceLocale].parsed; + Object.keys(locales).forEach(async locale => { + if (!allowedLocales || allowedLocales.includes(locale)) { + + let localeContents = locales[locale].contents; + + Object.values(locales[locale].parsed).forEach(t => { + + const source = sourceLocaleParsed[t.key]; + + if (localeContents.includes(t)) { + const baseTabs = t.match('^\n?(?\t*)').groups.tabs + const newVal = formatMessage(t.val, { + locale, + add: program.add, + remove: program.remove, + newlines: program.newlines, + dedupe: program.dedupe, + + baseTabs: baseTabs.length, + key: t.key, + source, + target: t + }); + const valQuote = program.newlines && newVal.includes('\n') ? '`' : t.valQuote; + const valSpace = program.newlines && newVal.includes('\n') ? `\n${baseTabs}` : t.valSpace; + const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}${t.valQuote}${t.comma}${t.comment}`; + const noo = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${valSpace}${valQuote}${newVal}${valQuote}${t.comma}${t.comment}`; + + if (old !== noo) count += 1; + localeContents = localeContents.replace(old, noo); + } + }); + + await writeFile(absLocalesPath + locales[locale].file, localeContents); + }; + }); + + const cliReport = `\n ${chalk.green('\u2714')} Formatted ${count} messages`; + console.log(cliReport); + + return; + } + if (program.rename) { let count = 0; await Promise.all(Object.keys(locales).map(async locale => { diff --git a/package-lock.json b/package-lock.json index 79aae13..a748303 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,17 @@ { "name": "messageformat-validator", - "version": "2.6.7", + "version": "3.0.0-alpha.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "2.6.7", + "version": "3.0.0-alpha.1", "license": "MIT", "dependencies": { + "@formatjs/icu-messageformat-parser": "^2.7.8", "chalk": "^4.1.0", + "cldr": "^7.5.0", "commander": "^6.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", @@ -119,15 +121,6 @@ "eslint": "^7.5.0 || ^8.0.0 || ^9.0.0" } }, - "node_modules/@babel/eslint-parser/node_modules/eslint-visitor-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", - "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", - "dev": true, - "engines": { - "node": ">=10" - } - }, "node_modules/@babel/generator": { "version": "7.25.6", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.25.6.tgz", @@ -161,6 +154,16 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "peer": true, + "dependencies": { + "yallist": "^3.0.2" + } + }, "node_modules/@babel/helper-module-imports": { "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz", @@ -473,9 +476,9 @@ } }, "node_modules/@eslint-community/regexpp": { - "version": "4.11.0", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.11.0.tgz", - "integrity": "sha512-G/M/tIiMrTAxEWRfLfQJMmGNX28IxBg4PBz8XqQhqUHLFI6TL2htpIB1iQCj144V5ee/JaKyT9/WZ0MGZWfA7A==", + "version": "4.11.1", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.11.1.tgz", + "integrity": "sha512-m4DVN9ZqskZoLU5GlWZadwDnYo3vAEydiUayB9widCl9ffWx2IvPnp6n3on5rJmziJSw9Bv+Z3ChDVdMwXCY8Q==", "dev": true, "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" @@ -534,14 +537,50 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@formatjs/ecma402-abstract": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@formatjs/ecma402-abstract/-/ecma402-abstract-2.0.0.tgz", + "integrity": "sha512-rRqXOqdFmk7RYvj4khklyqzcfQl9vEL/usogncBHRZfZBDOwMGuSRNFl02fu5KGHXdbinju+YXyuR+Nk8xlr/g==", + "dependencies": { + "@formatjs/intl-localematcher": "0.5.4", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/icu-messageformat-parser": { + "version": "2.7.8", + "resolved": "https://registry.npmjs.org/@formatjs/icu-messageformat-parser/-/icu-messageformat-parser-2.7.8.tgz", + "integrity": "sha512-nBZJYmhpcSX0WeJ5SDYUkZ42AgR3xiyhNCsQweFx3cz/ULJjym8bHAzWKvG5e2+1XO98dBYC0fWeeAECAVSwLA==", + "dependencies": { + "@formatjs/ecma402-abstract": "2.0.0", + "@formatjs/icu-skeleton-parser": "1.8.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/icu-skeleton-parser": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@formatjs/icu-skeleton-parser/-/icu-skeleton-parser-1.8.2.tgz", + "integrity": "sha512-k4ERKgw7aKGWJZgTarIcNEmvyTVD9FYh0mTrrBMHZ1b8hUu6iOJ4SzsZlo3UNAvHYa+PnvntIwRPt1/vy4nA9Q==", + "dependencies": { + "@formatjs/ecma402-abstract": "2.0.0", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/intl-localematcher": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/@formatjs/intl-localematcher/-/intl-localematcher-0.5.4.tgz", + "integrity": "sha512-zTwEpWOzZ2CiKcB93BLngUX59hQkuZjT2+SAQEscSm52peDW/getsawMcWF1rGRpMCX6D7nSJA3CzJ8gn13N/g==", + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", "deprecated": "Use @eslint/config-array instead", "dev": true, "dependencies": { - "@humanwhocodes/object-schema": "^2.0.2", + "@humanwhocodes/object-schema": "^2.0.3", "debug": "^4.3.1", "minimatch": "^3.0.5" }, @@ -631,28 +670,6 @@ "eslint-scope": "5.1.1" } }, - "node_modules/@nicolo-ribaudo/eslint-scope-5-internals/node_modules/eslint-scope": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", - "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", - "dev": true, - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^4.1.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@nicolo-ribaudo/eslint-scope-5-internals/node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -699,6 +716,14 @@ "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", "dev": true }, + "node_modules/@xmldom/xmldom": { + "version": "0.8.10", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.10.tgz", + "integrity": "sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/acorn": { "version": "8.12.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", @@ -932,6 +957,17 @@ "node": ">=12" } }, + "node_modules/chainsaw": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.0.9.tgz", + "integrity": "sha512-nG8PYH+/4xB+8zkV4G844EtfvZ5tTiLFoX3dZ4nhF4t3OCKIb9UvaFyNmeZO2zOSmRWzBoTD+napN6hiL+EgcA==", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/chalk": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", @@ -980,6 +1016,22 @@ "fsevents": "~2.3.2" } }, + "node_modules/cldr": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/cldr/-/cldr-7.5.0.tgz", + "integrity": "sha512-2qy3ASYFbNToTujNnk5Y8ak++B4TH/G+S8AEOrN1xUFZhxhmqWDPUGnOFGyId61vD2Trf+yE65wVzIcdE/bpPg==", + "dependencies": { + "@xmldom/xmldom": "^0.8.0", + "escodegen": "^2.0.0", + "esprima": "^4.0.1", + "memoizeasync": "^1.1.0", + "passerror": "^1.1.1", + "pegjs": "^0.10.0", + "seq": "^0.3.5", + "unicoderegexp": "^0.4.1", + "xpath": "^0.0.33" + } + }, "node_modules/cliui": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", @@ -1107,9 +1159,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.19", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.19.tgz", - "integrity": "sha512-kpLJJi3zxTR1U828P+LIUDZ5ohixyo68/IcYOHLqnbTPr/wdgn4i1ECvmALN9E16JPA6cvCG5UG79gVwVdEK5w==", + "version": "1.5.25", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.25.tgz", + "integrity": "sha512-kMb204zvK3PsSlgvvwzI3wBIcAw15tRkYk+NQdsjdDtcQWTp2RABbMQ9rUBy8KNEOM+/E6ep+XC3AykiWZld4g==", "dev": true, "peer": true }, @@ -1140,17 +1192,37 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, "node_modules/eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "@ungap/structured-clone": "^1.2.0", @@ -1196,31 +1268,34 @@ } }, "node_modules/eslint-scope": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", - "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", "dev": true, "dependencies": { "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" + "estraverse": "^4.1.1" }, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" + "node": ">=8.0.0" + } + }, + "node_modules/eslint-scope/node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "engines": { + "node": ">=4.0" } }, "node_modules/eslint-visitor-keys": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.0.0.tgz", - "integrity": "sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", "dev": true, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" + "node": ">=10" } }, "node_modules/eslint/node_modules/@eslint/eslintrc": { @@ -1247,12 +1322,28 @@ } }, "node_modules/eslint/node_modules/@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/eslint/node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", "dev": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" } }, "node_modules/eslint/node_modules/eslint-visitor-keys": { @@ -1336,6 +1427,30 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/espree/node_modules/eslint-visitor-keys": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.0.0.tgz", + "integrity": "sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw==", + "dev": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", @@ -1364,7 +1479,6 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, "engines": { "node": ">=4.0" } @@ -1373,7 +1487,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -1589,6 +1702,17 @@ "node": ">=8" } }, + "node_modules/hashish": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz", + "integrity": "sha512-xyD4XgslstNAs72ENaoFvgMwtv8xhiDtC2AtzCG+8yF7W/Knxxm9BX+e2s25mm+HxMKh0rBmXVOEGF3zNImXvA==", + "dependencies": { + "traverse": ">=0.2.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -1865,20 +1989,24 @@ } }, "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "peer": true, - "dependencies": { - "yallist": "^3.0.2" - } + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.0.tgz", + "integrity": "sha512-dVmQmXPBlTgFw77hm60ud//l2bCuDKkqC2on1EBoM7s9Urm9IQDrnujwZ93NFnAq0dVZ0HBXTS7PwEG+YE7+EQ==" }, "node_modules/make-plural": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/make-plural/-/make-plural-7.4.0.tgz", "integrity": "sha512-4/gC9KVNTV6pvYg2gFeQYTW3mWaoJt7WZE5vrp1KnQDgW92JtYZnzmZT81oj/dUTqAIu0ufI2x3dkgu3bB1tYg==" }, + "node_modules/memoizeasync": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/memoizeasync/-/memoizeasync-1.1.0.tgz", + "integrity": "sha512-HMfzdLqClZo8HMyuM9B6TqnXCNhw82iVWRLqd2cAdXi063v2iJB4mQfWFeKVByN8VUwhmDZ8NMhryBwKrPRf8Q==", + "dependencies": { + "lru-cache": "2.5.0", + "passerror": "1.1.1" + } + }, "node_modules/messageformat-parser": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-4.1.3.tgz", @@ -2089,6 +2217,14 @@ "node": ">=6" } }, + "node_modules/passerror": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/passerror/-/passerror-1.1.1.tgz", + "integrity": "sha512-PwrEQJBkJMxnxG+tdraz95vTstYnCRqiURNbGtg/vZHLgcAODc9hbiD5ZumGUoh3bpw0F0qKLje7Vd2Fd5Lx3g==", + "engines": { + "node": "*" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -2124,6 +2260,17 @@ "node": ">= 14.16" } }, + "node_modules/pegjs": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/pegjs/-/pegjs-0.10.0.tgz", + "integrity": "sha512-qI5+oFNEGi3L5HAxDwN2LA4Gg7irF70Zs25edhjld9QemOgp0CbvMtbFcMvFtEo1OityPrcCzkQFB8JP/hxgow==", + "bin": { + "pegjs": "bin/pegjs" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/picocolors": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.0.tgz", @@ -2298,6 +2445,18 @@ "semver": "bin/semver.js" } }, + "node_modules/seq": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/seq/-/seq-0.3.5.tgz", + "integrity": "sha512-sisY2Ln1fj43KBkRtXkesnRHYNdswIkIibvNe/0UKm2GZxjMbqmccpiatoKr/k2qX5VKiLU8xm+tz/74LAho4g==", + "dependencies": { + "chainsaw": ">=0.0.7 <0.1", + "hashish": ">=0.0.2 <0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/serialize-javascript": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", @@ -2328,6 +2487,15 @@ "node": ">=8" } }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -2405,6 +2573,19 @@ "node": ">=8.0" } }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "engines": { + "node": "*" + } + }, + "node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -2429,6 +2610,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/unicoderegexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/unicoderegexp/-/unicoderegexp-0.4.1.tgz", + "integrity": "sha512-ydh8D5mdd2ldTS25GtZJEgLciuF0Qf2n3rwPhonELk3HioX201ClYGvZMc1bCmx6nblZiADQwbMWekeIqs51qw==" + }, "node_modules/update-browserslist-db": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz", @@ -2532,6 +2718,14 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "node_modules/xpath": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.33.tgz", + "integrity": "sha512-NNXnzrkDrAzalLhIUc01jO2mOzXGXh1JwPgkihcLLzw98c0WgYDmmjSh1Kl3wzaxSVWMuA+fe0WTWOBDWCBmNA==", + "engines": { + "node": ">=0.6.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 6c1f53e..8a899ce 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "2.6.7", + "version": "3.0.0-alpha.1", "description": "Validates that ICU MessageFormat strings are well-formed, and that translated target strings are compatible with their source.", "type": "module", "repository": { @@ -27,7 +27,9 @@ "author": "Daniel Gleckler ", "license": "MIT", "dependencies": { + "@formatjs/icu-messageformat-parser": "^2.7.8", "chalk": "^4.1.0", + "cldr": "^7.5.0", "commander": "^6.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", diff --git a/src/format.js b/src/format.js new file mode 100644 index 0000000..0bb4494 --- /dev/null +++ b/src/format.js @@ -0,0 +1,220 @@ +import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.js'; +import { parse } from '@formatjs/icu-messageformat-parser'; +import * as pluralCats from 'make-plural/pluralCategories'; +import cldr from 'cldr'; + +function getPluralCats(locale) { + return pluralCats[locale.split('-')[0]] || pluralCats.en; +} + +function expandASTHashes(ast, parentValue) { + if (Array.isArray(ast)) { + ast.map(ast => expandASTHashes(ast, parentValue)); + } + + if (ast.type === 7) { // # + ast.type = 1; + ast.value = parentValue; + } + else if (ast.type === 6) { // plural, selectordinal + expandASTHashes(Object.values(ast.options).map(o => o.value), ast.value); + } +} + +export function formatMessage(msg, options = {}) { + let ast; + //const trimmedMsg = msg.trim(); + //msg = trimmedMsg.length === 1 ? msg : trimmedMsg; + try { + ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); + } catch(err) { + try { + alteredMsg = msg.replace('\'{', '’{'); + ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); + msg = alteredMsg; + } catch(err2) { + if (err.location) { + console.log(`\nERROR: ${err.message}`); + console.log(`\tLocale: ${options.locale}`); + console.log(`\tKey: ${options.key}`); + console.log(`\tOriginal message: ${err.originalMessage}`); + console.log('\tAt or near:', msg.slice(err.location.start.offset, Math.max(err.location.end.offset, err.location.start.offset + 4))); + } else { + console.log(err); + console.log(`\tLocale: ${options.locale}`); + console.log(`\tKey: ${options.key}`); + } + return msg; + } + } + if (options.expandHashes) { + expandASTHashes(ast); + } + try { + ast = hoistSelectors(ast); + } catch(e) { + console.log(e); + } + + return printAST(ast, { + useNewlines: options.newlines ?? msg.includes('\n'), + add: options.add ?? false, + remove: options.remove ?? false, + dedupe: options.dedupe ?? false, + locale: options.locale, + args: options.source ? [...new Set(options.source.match(/(?<=[\{<])[^,\{\}<>]+(?=[\}>,])/g))] : [] + }, options.baseTabs); +} + +function normalizeArgName(argName, availableArgs) { + if (!availableArgs.includes(argName)) { + if (availableArgs.length === 1) { + return availableArgs[0]; + } else { + return availableArgs.find(a => a.toLowerCase() === argName.toLowerCase()) ?? argName; + } + } + return argName; +} + +function printAST(ast, options, level = 0) { + const { + locale, + swapOne = new Set(), + useNewlines = false, + add = false, + remove = false, + dedupe = false, + args = [] + } = options; + + if (Array.isArray(ast)) { + const swapOneClone = new Set(swapOne); + ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) + + const delimiters = (() => { + try { + return cldr.extractDelimiters(locale); + } catch(err) { + return cldr.extractDelimiters(locale.split('-')[0]); + } + })(); + let + quoteStart = delimiters.quotationStart, + quoteEnd = delimiters.quotationEnd, + singleQuoteStart = delimiters.alternateQuotationStart, + singleQuoteEnd = delimiters.alternateQuotationEnd; + + if (locale.toLowerCase().endsWith('-gb')) { + quoteStart = delimiters.alternateQuotationStart; + quoteEnd = delimiters.alternateQuotationEnd; + singleQuoteStart = delimiters.quotationStart; + singleQuoteEnd = delimiters.quotationEnd; + } + + return ast.map(ast => printAST(ast, { ...options, swapOne: swapOneClone }, level)).join('') + .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") + .replace(/(?<=\s)\\?'|^\\?'/g, singleQuoteStart) // opening ' + .replace(/(?<=\S)'(?=\S)/g, '’') // apostrophe + .replace(/\\?'/g, singleQuoteEnd) // closing ' + .replace(/(?<=\s(\u0648)?)\\?"|^\\?"/g, quoteStart) // opening " + .replace(/\\?"/g, quoteEnd) // closing " + .replace(/\|_escape_\|/g, "'"); + } + + let text = ''; + const indent = useNewlines ? Array(level).fill('\t').join('') : ' '; + const newline = useNewlines ? `\n${indent}` : indent; + const type = ast.type; + + if (type === 0) { // straight text + const value = swapOne.size ? ast.value.replace(/1/g, `{${[...swapOne].join('|')}}`) : ast.value; + text += value; + } + else if (type === 1) { // simple arg + text += `{${normalizeArgName(ast.value, args)}}`; + } + else if ([2, 3, 4].includes(type)) { // number, date, time + const style = (() => { + if (ast.style) { + if (typeof ast.style === 'string') return `, ${ast.style}`; + else return `, ::${ast.style.pattern || ast.style.tokens.map(t => t.stem).join(' ')}`; + } else { + return ''; + } + })(); + + const typesText = ['number', 'date', 'time']; + text += `{${normalizeArgName(ast.value, args)}, ${typesText[type - 2]}${style}}`; + } + else if (type === 5) { // select + const optionsText = Object.entries(ast.options) + .sort((a, b) => { + return a[0] === 'other' ? 1 : (b[0] === 'other' ? -1 : 0); + }) + .map(([opt, { value }]) => { + return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, options, level + 1)}}`; + }).join('') + (useNewlines ? newline : ''); + + text += `{${normalizeArgName(ast.value, args)}, select,${optionsText}}`; + } + else if (type === 6) { // plural, selectordinal + const pluralCats = ['zero', 'one', 'two', 'few', 'many', 'other']; + const supportedCats = new Intl.PluralRules(locale, { type: ast.pluralType }).resolvedOptions().pluralCategories; + const unsupportedCats = [ ...Object.keys(ast.options).filter(o => !/^=\d+$/.test(o)) , ...pluralCats].filter(cat => !supportedCats.includes(cat)); + if (add) { + supportedCats.forEach(cat => { + if (!/^(fr|pt)/.test(locale) && cat === 'one' && ast.options['=1']) return; // don't create orphaned `one` + // add missing supported categories + ast.options[cat] ??= { ...(ast.options.other || ast.options.many || ast.options.few || Object.values(ast.options).at(-1)) }; + }); + } + + if (ast.options.one) { + // `one` and `=1` are the same + if (ast.options['=1'] && JSON.stringify(ast.options['=1']) === JSON.stringify(ast.options['one'])) { + delete ast.options['=1']; + } + } else if (ast.options['=1'] && /(? { + if (k !== 'other' && printAST(v.value, { locale, swapOne, args }) === otherPrinted) { + delete ast.options[k]; + } + }); + } + + remove && unsupportedCats.forEach(cat => delete ast.options[cat]); + + const typeText = ast.pluralType === 'ordinal' ? 'selectordinal' : 'plural'; + const offsetText = + ast.offset !== 0 ? ` offset:${ast.offset}` : ''; + const optionsText = Object.entries(ast.options).sort((a, b) => { + if (a[0].startsWith('=') || b[0].startsWith('=')) { + return a[0].localeCompare(b[0]); + } + return pluralCats.indexOf(a[0]) > pluralCats.indexOf(b[0]) ? 1 : -1; + }).map(([opt, { value }]) => { + return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, swapOne }, level + 1)}}`; + }).join('') + (useNewlines ? newline : ''); + + text += `{${normalizeArgName(ast.value, args)}, ${typeText},${offsetText}${optionsText}}`; + } + else if (type === 7) { // # + text += '#'; + } + else if (type === 8) { // tag + text += `<${normalizeArgName(ast.value, args)}>${printAST(ast.children, options, level + 1)}`; + } + else { // unhandled + console.warn('unhandled type:', type); + } + + return text; +} diff --git a/src/validate.js b/src/validate.js index 462aa1c..c8369b0 100644 --- a/src/validate.js +++ b/src/validate.js @@ -196,10 +196,10 @@ export function parseLocales(locales, useJSONObj) { }; const regex = useJSONObj - //[ ][ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] - ? /("(?.*)"(\s*):(\s*){)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?,?)(?.*)/g - //[ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] - : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?,?)(?.*)/g; + //[ ][ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] + ? /("(?.*)"(\s*):(\s*)\{)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/g + //[ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] + : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/g; const matches = Array.from(contents.matchAll(regex)); let findContext = false; From bc7dca16a2b1a5b2bd2f88aabbb7c2e91032acd2 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 6 Sep 2024 10:23:09 -0400 Subject: [PATCH 02/90] Cleanup terminology, grammar, etc. --- bin/cli.js | 36 ++++++++++++++++++++++++------------ src/validate.js | 8 ++++---- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 8e63c4e..0898471 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -21,7 +21,7 @@ program .option('-i, --ignoreIssueTypes ', 'Ignore these comma-separated issue types') .option('-l, --locales ', 'Process only these comma-separated locales') .option('-p, --path ', 'Path to a directory containing locale files') - .option('-t, --translator-output', 'Output JSON of all source strings that are missing or untranslated in the target') + .option('-t, --translator-output', 'Output JSON of all source messages that are missing or untranslated in the target') .option('-s, --source-locale ', 'The locale to use as the source') .option('--json-obj', 'Indicate that the files to be parsed are JSON files with keys that have objects for values') .command('validate', { isDefault: true, hidden: true }) @@ -31,28 +31,28 @@ program program .command('remove-extraneous') - .description('Remove strings that do not exist in the source locale') + .description('Remove messages that do not exist in the source locale') .action(() => { program.removeExtraneous = true; }); program .command('add-missing') - .description('Add strings that do not exist in the target locale') + .description('Add messages that do not exist in the target locale') .action(() => { program.addMissing = true; }); program .command('sort') - .description('Sort strings alphabetically by key, maintaining any blocks') + .description('Sort messages alphabetically by key, maintaining any blocks') .action(() => { program.sort = true; }); program .command('rename ') - .description('Rename a string') + .description('Rename a message') .action((oldKey, newKey) => { program.rename = true; program.oldKey = oldKey; @@ -78,7 +78,7 @@ program program .command('highlight ') - .description('Output a string with all non-translatable ICU MessageFormat structure highlighted') + .description('Output a message with all non-translatable ICU MessageFormat structure highlighted') .action(key => { program.highlight = key; }); @@ -86,7 +86,15 @@ program program.parse(process.argv); const pathCombined = program.path || path; -if (!pathCombined) throw new Error('Must provide a path to the locale files using either the -p option or a config file.'); +if (!pathCombined) { + console.error('Must provide a path to the locale files using either the -p option or a config file.'); + process.exit(1); +} + +const noSource = () => { + console.error('Must provide a source locale using either the -s option or a config file.'); + process.exit(1); +}; const localesPaths = glob.sync(pathCombined); localesPaths.forEach(async localesPath => { @@ -111,11 +119,13 @@ localesPaths.forEach(async localesPath => { const targetLocales = filteredFiles.map(file => file.split('.')[0]); if (program.removeExtraneous) { - console.log('Removing extraneous strings from:', targetLocales.join(', ')); + if (!sourceLocale) noSource(); + console.log('Removing extraneous messages from:', targetLocales.join(', ')); } if (program.addMissing) { - console.log('Adding missing strings to:', targetLocales.join(', ')); + if (!sourceLocale) noSource(); + console.log('Adding missing messages to:', targetLocales.join(', ')); } if (program.rename) { @@ -245,12 +255,14 @@ localesPaths.forEach(async localesPath => { } })); - const cliReport = `\n ${chalk.green('\u2714')} Renamed ${count} strings`; + const cliReport = `\n ${chalk.green('\u2714')} Renamed ${count} messages`; console.log(cliReport); return; } + if (!sourceLocale) noSource(); + const output = validateLocales({ locales, sourceLocale }); const translatorOutput = {}; @@ -340,12 +352,12 @@ localesPaths.forEach(async localesPath => { if (program.removeExtraneous) { const count = locale.report.errors ? locale.report.errors.extraneous || 0 : 0; - const cliReport = `\n ${chalk.green('\u2714')} Removed ${count} extraneous strings`; + const cliReport = `\n ${chalk.green('\u2714')} Removed ${count} extraneous messages`; console.log(cliReport); } else if (program.addMissing) { const count = locale.report.errors ? locale.report.errors.missing || 0 : 0; - const cliReport = `\n ${chalk.green('\u2714')} Added ${count} missing strings`; + const cliReport = `\n ${chalk.green('\u2714')} Added ${count} missing messages`; console.log(cliReport); } else if (program.sort) { diff --git a/src/validate.js b/src/validate.js index c8369b0..c5c6541 100644 --- a/src/validate.js +++ b/src/validate.js @@ -92,10 +92,10 @@ export function validateMessage({ targetString, targetLocale, sourceString, sour const badKey = backtickCaptures[0].slice(1, -1); const pluralArg = backtickCaptures[1].slice(1, -1) const column = targetString.indexOf(badKey, targetString.indexOf(`{${pluralArg}, plural, {`)); - msgReporter.error('plural-key', e.message, { column }); + msgReporter.error('categories', e.message, { column }); } else if ((targetString.match(/{/g) || 0).length !== (targetString.match(/}/g) || 0).length) { - msgReporter.error('brace', 'Mismatched braces (i.e. {}). ' + e.message, { column: e.location.start.column }); + msgReporter.error('brace', 'Mismatched braces. ' + e.message, { column: e.location.start.column }); } else { msgReporter.error('parse', e.message, { column: e.location.start.column - 1 }); @@ -144,7 +144,7 @@ export function validateMessage({ targetString, targetLocale, sourceString, sour const badArgPos = targetString.indexOf(argDiff[0]); if (argDiff.length) { - msgReporter.error('argument', `Unrecognized arguments ${JSON.stringify(argDiff)}`, { column: badArgPos }); + msgReporter.error('argument', `Unrecognized arguments: ${argDiff.join(', ')}. Must be one of: ${Array.from(sourceMap.arguments).join(', ')}`, { column: badArgPos }); } // remove all translated content, leaving only the messageformat structure @@ -179,7 +179,7 @@ export function validateMessage({ targetString, targetLocale, sourceString, sour if (targetTokens.length > 1) { if (targetLocale == sourceLocale && targetTokens.find((token) => typeof token !== 'string' && token.type.match(/plural|select/))) { - msgReporter.warning('split','String split by non-argument (e.g. select; plural).') + msgReporter.warning('split','String split by complex argument') } } } From 5c0cd398ef2eb89ffe063a8e0d6590d28985c9ee Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 17 Sep 2024 10:56:38 -0400 Subject: [PATCH 03/90] Throw on error --- bin/cli.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 0898471..72db0bb 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -16,7 +16,6 @@ const { path, source: globalSource, locales: globalLocales, jsonObj: globalJsonO program .version(pkg.version) - .option('-e, --throw-errors', 'Throw an error if error issues are found') .option('--no-issues', 'Don\'t output issues') .option('-i, --ignoreIssueTypes ', 'Ignore these comma-separated issue types') .option('-l, --locales ', 'Process only these comma-separated locales') @@ -376,7 +375,7 @@ localesPaths.forEach(async localesPath => { } })); - if (program.throwErrors && output.some(locale => locale.report.totals.errors)) { + if (output.some(locale => locale.report.totals.errors)) { console.error('\nErrors were reported in at least one locale. See details above.'); return 1; } From 504bf97cdd16fe8c98f3081e4c0d0ce055e77530 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 17 Sep 2024 17:03:54 -0400 Subject: [PATCH 04/90] Add trim and collapse; Move translator-output to print-missing subcommand --- bin/cli.js | 18 +++++++++++++----- src/format.js | 6 ++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 72db0bb..1f4ca6e 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -20,7 +20,6 @@ program .option('-i, --ignoreIssueTypes ', 'Ignore these comma-separated issue types') .option('-l, --locales ', 'Process only these comma-separated locales') .option('-p, --path ', 'Path to a directory containing locale files') - .option('-t, --translator-output', 'Output JSON of all source messages that are missing or untranslated in the target') .option('-s, --source-locale ', 'The locale to use as the source') .option('--json-obj', 'Indicate that the files to be parsed are JSON files with keys that have objects for values') .command('validate', { isDefault: true, hidden: true }) @@ -28,6 +27,13 @@ program program.validate = true; }); +program + .command('print-missing') + .description('Output JSON of all source messages that are missing or untranslated in the target') + .action(() => { + program.printMissing = true; + }); + program .command('remove-extraneous') .description('Remove messages that do not exist in the source locale') @@ -65,6 +71,8 @@ program .option('-a, --add', 'Add cases for missing supported pural and selectordinal categories') .option('-r, --remove', 'Remove cases for unsupported pural and selectordinal categories') .option('-d, --dedupe', 'Remove complex argument cases that duplicate the `other` case. Takes precedence over --add.') + .option('-t, --trim', 'Trim whitespace from both ends of messages') + .option('-c, --collapse', 'Collapse repeating whitepace') .action(function() { program.format = true; const opts = this.opts(); @@ -316,7 +324,7 @@ localesPaths.forEach(async localesPath => { locales[locale.locale].parsed[issue.key] = locales[sourceLocale].parsed[issue.key]; } } - else if (program.translatorOutput) { + else if (program.printMissing) { if (['missing', 'untranslated'].includes(issue.type)) { translatorOutput[issue.key] = issue.source; } @@ -345,11 +353,10 @@ localesPaths.forEach(async localesPath => { } } - if (program.translatorOutput) { + if (program.printMissing) { console.log(JSON.stringify(translatorOutput, null, 2)); } - - if (program.removeExtraneous) { + else if (program.removeExtraneous) { const count = locale.report.errors ? locale.report.errors.extraneous || 0 : 0; const cliReport = `\n ${chalk.green('\u2714')} Removed ${count} extraneous messages`; console.log(cliReport); @@ -367,6 +374,7 @@ localesPaths.forEach(async localesPath => { const total = locale.report.totals.errors + locale.report.totals.warnings; const cliReport = chalk[color](`\n\u2716 ${total} issues (${locale.report.totals.errors} errors, ${locale.report.totals.warnings} warnings)${locale.report.totals.ignored ? chalk.grey(` - ${locale.report.totals.ignored} Ignored`) : ''}`); console.log(cliReport); + return; } else { const cliReport = `\n ${chalk.green('\u2714')} Passed`; diff --git a/src/format.js b/src/format.js index 0bb4494..c0889af 100644 --- a/src/format.js +++ b/src/format.js @@ -23,8 +23,10 @@ function expandASTHashes(ast, parentValue) { export function formatMessage(msg, options = {}) { let ast; - //const trimmedMsg = msg.trim(); - //msg = trimmedMsg.length === 1 ? msg : trimmedMsg; + if (options.trim) { + const trimmedMsg = msg.trim(); + msg = trimmedMsg.length === 1 ? msg : trimmedMsg; + } try { ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); } catch(err) { From e6432f8ade6a68a16be59309cccb297f7f7956c8 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 17 Sep 2024 17:10:16 -0400 Subject: [PATCH 05/90] Fix output order --- bin/cli.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 1f4ca6e..c97bf17 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -349,7 +349,7 @@ localesPaths.forEach(async localesPath => { } if (program.removeExtraneous || program.addMissing || program.sort) { - await writeFile(localePath, locales[locale.locale].contents); + writeFile(localePath, locales[locale.locale].contents); } } @@ -381,9 +381,12 @@ localesPaths.forEach(async localesPath => { console.log(cliReport); } } + + locale.report = undefined; + })); - if (output.some(locale => locale.report.totals.errors)) { + if (output.some(locale => locale.report?.totals.errors)) { console.error('\nErrors were reported in at least one locale. See details above.'); return 1; } From 300a03753d9540f11c41f0866a9ea25ce09fd51f Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 18 Sep 2024 08:46:00 -0400 Subject: [PATCH 06/90] Add trim and collapse options --- bin/cli.js | 5 ++++- src/format.js | 25 +++++++++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index c97bf17..5d31b22 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -79,7 +79,8 @@ program program.newlines = opts.newlines; program.add = opts.add; program.remove = opts.remove; - console.log(opts); + program.trim = opts.trim; + program.collapse = opts.collapse; program.dedupe = opts.dedupe }); @@ -212,6 +213,8 @@ localesPaths.forEach(async localesPath => { remove: program.remove, newlines: program.newlines, dedupe: program.dedupe, + trim: program.trim, + collapse: program.collapse, baseTabs: baseTabs.length, key: t.key, diff --git a/src/format.js b/src/format.js index c0889af..b2be370 100644 --- a/src/format.js +++ b/src/format.js @@ -23,10 +23,6 @@ function expandASTHashes(ast, parentValue) { export function formatMessage(msg, options = {}) { let ast; - if (options.trim) { - const trimmedMsg = msg.trim(); - msg = trimmedMsg.length === 1 ? msg : trimmedMsg; - } try { ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); } catch(err) { @@ -63,6 +59,9 @@ export function formatMessage(msg, options = {}) { add: options.add ?? false, remove: options.remove ?? false, dedupe: options.dedupe ?? false, + trim: options.trim ?? false, + collapse: options.collapse ?? false, + locale: options.locale, args: options.source ? [...new Set(options.source.match(/(?<=[\{<])[^,\{\}<>]+(?=[\}>,])/g))] : [] }, options.baseTabs); @@ -87,6 +86,8 @@ function printAST(ast, options, level = 0) { add = false, remove = false, dedupe = false, + trim = false, + collapse = false, args = [] } = options; @@ -114,7 +115,19 @@ function printAST(ast, options, level = 0) { singleQuoteEnd = delimiters.quotationEnd; } - return ast.map(ast => printAST(ast, { ...options, swapOne: swapOneClone }, level)).join('') + return ast.map((ast, idx, arr) => { + let trim; + if (options.trim) { + if (arr.length === 1) { + trim = 'trim'; + } else if (!idx) { + trim = 'trimStart'; + } else if (idx === arr.length - 1) { + trim = 'trimEnd'; + } + } + return printAST(ast, { ...options, swapOne: swapOneClone, trim }, level); + }).join('') .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") .replace(/(?<=\s)\\?'|^\\?'/g, singleQuoteStart) // opening ' .replace(/(?<=\S)'(?=\S)/g, '’') // apostrophe @@ -131,7 +144,7 @@ function printAST(ast, options, level = 0) { if (type === 0) { // straight text const value = swapOne.size ? ast.value.replace(/1/g, `{${[...swapOne].join('|')}}`) : ast.value; - text += value; + text += value[trim]?.() ?? value; } else if (type === 1) { // simple arg text += `{${normalizeArgName(ast.value, args)}}`; From adef6086cdb8d64c9e541dcb9ee8a46be9d0b436 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 18 Sep 2024 10:12:00 -0400 Subject: [PATCH 07/90] Add format tests --- test/format.test.js | 88 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 test/format.test.js diff --git a/test/format.test.js b/test/format.test.js new file mode 100644 index 0000000..d27aeb3 --- /dev/null +++ b/test/format.test.js @@ -0,0 +1,88 @@ +import { expect } from 'chai'; +import { formatMessage } from '../src/format.js'; + +describe('formatMessage', () => { + + let locale; + beforeEach(() => { + locale = 'en'; + }); + + [ + { locale: 'ar', expected: `This isn’t ”correct“` }, + { locale: 'cy', expected: `This isn’t “correct”` }, + { locale: 'de', expected: `This isn’t „correct“` }, + { locale: 'en', expected: `This isn’t “correct”` }, + { locale: 'en-gb', expected: `This isn’t ‘correct’` }, + { locale: 'fr', expected: `This isn’t «correct»` }, + { locale: 'sv', expected: `This isn’t ”correct”` }, + ].forEach(({ locale, expected }) => { + it(`should replace straight quotes with "${locale}" quotes with no options`, () => { + const message = `This isn't "correct"`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + }); + + [ + { locale: 'ar', expected: +`{a, plural, + one {{b, selectordinal, + other {} + }} + two {} + few {} + many {} + other {} +}` }, + { locale: 'cy', expected: +`{a, plural, + one {{b, selectordinal, + one {} + two {} + few {} + many {} + other {} + }} + two {} + few {} + many {} + other {} +}` }, + { locale: 'es', expected: +`{a, plural, + one {{b, selectordinal, + other {} + }} + many {} + other {} +}` }, + { locale: 'fr', expected: +`{a, plural, + one {{b, selectordinal, + one {} + other {} + }} + many {} + other {} +}` }, + { locale: 'ja', expected: +`{a, plural, + other {} +}` }, + ].forEach(({ locale, expected }) => { + it(`should remove plural and selectordinal categories that are unsupported in "${locale}" with the "remove" option`, () => { + const message = +`{a, plural, + one {{b, selectordinal, one {} two {} few {} many {} other {}}} + two {} + few {} + many {} + other {} +}`; + const formatted = formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + }); + +}); From a2b670804717f116027620a76292e5af1b0dace8 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 18 Sep 2024 10:31:05 -0400 Subject: [PATCH 08/90] Fix lint and tests --- bin/cli.js | 2 +- src/format.js | 10 ++------- src/reporter.js | 4 ++-- test/format.test.js | 5 ----- test/validate.test.js | 52 ++++++++++++++++++++++++++++++++++--------- 5 files changed, 47 insertions(+), 26 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 5d31b22..ae82bac 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -232,7 +232,7 @@ localesPaths.forEach(async localesPath => { }); await writeFile(absLocalesPath + locales[locale].file, localeContents); - }; + } }); const cliReport = `\n ${chalk.green('\u2714')} Formatted ${count} messages`; diff --git a/src/format.js b/src/format.js index b2be370..db12423 100644 --- a/src/format.js +++ b/src/format.js @@ -1,12 +1,7 @@ import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.js'; import { parse } from '@formatjs/icu-messageformat-parser'; -import * as pluralCats from 'make-plural/pluralCategories'; import cldr from 'cldr'; -function getPluralCats(locale) { - return pluralCats[locale.split('-')[0]] || pluralCats.en; -} - function expandASTHashes(ast, parentValue) { if (Array.isArray(ast)) { ast.map(ast => expandASTHashes(ast, parentValue)); @@ -27,7 +22,7 @@ export function formatMessage(msg, options = {}) { ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); } catch(err) { try { - alteredMsg = msg.replace('\'{', '’{'); + const alteredMsg = msg.replace('\'{', '’{'); ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); msg = alteredMsg; } catch(err2) { @@ -63,7 +58,7 @@ export function formatMessage(msg, options = {}) { collapse: options.collapse ?? false, locale: options.locale, - args: options.source ? [...new Set(options.source.match(/(?<=[\{<])[^,\{\}<>]+(?=[\}>,])/g))] : [] + args: options.source ? [...new Set(options.source.match(/(?<=[{<])[^,{}<>]+(?=[}>,])/g))] : [] }, options.baseTabs); } @@ -87,7 +82,6 @@ function printAST(ast, options, level = 0) { remove = false, dedupe = false, trim = false, - collapse = false, args = [] } = options; diff --git a/src/reporter.js b/src/reporter.js index 2582c22..3d4a3eb 100644 --- a/src/reporter.js +++ b/src/reporter.js @@ -31,8 +31,8 @@ Reporter.prototype.log = function(level, type, msg, column = 0, givenLine) { type, level, msg, - target: this._config.target.val || this._config.target, - source: this._config.source.val || this._config.source + target: this._config.target?.val ?? this._config.target, + source: this._config.source?.val ?? this._config.source }; if (this._config.key) issue.key = this._config.key; diff --git a/test/format.test.js b/test/format.test.js index d27aeb3..b1f348b 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -3,11 +3,6 @@ import { formatMessage } from '../src/format.js'; describe('formatMessage', () => { - let locale; - beforeEach(() => { - locale = 'en'; - }); - [ { locale: 'ar', expected: `This isn’t ”correct“` }, { locale: 'cy', expected: `This isn’t “correct”` }, diff --git a/test/validate.test.js b/test/validate.test.js index 83fac3a..96b2365 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -69,7 +69,16 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Missing categories: ["zero","two","few","many"]'); }); - // plural-key + it('generates a categories error when a target message uses unsupported plural categories', () => { + const sourceString = '{a, plural, one {} other {}}'; + const targetString = '{a, plural, one {} two {} few {} many {} other {}}'; + reporter.config(targetString, sourceString, 'key'); + validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Invalid key `two` for argument `a`. Valid plural keys for this locale are `one`, `other`, and explicit keys like `=0`.'); + }); it('generates a plural-key error when a target message uses unsupported plural categories', () => { const sourceString = '{a, plural, one {} other {}}'; @@ -82,7 +91,18 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Invalid key `two` for argument `a`. Valid plural keys for this locale are `one`, `other`, and explicit keys like `=0`.'); }); - // split + it('generates a split error when a source message is split by a complex argument', () => { + targetLocale = 'en'; + const sourceString = '{a, plural, one {} other {}} b'; + const targetString = '{a, plural, one {} other {}} b'; + reporter.config(targetString, sourceString, 'key'); + reporter._config.locale = targetLocale; + validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('split'); + expect(reporter.issues[0].level).to.equal('warning'); + expect(reporter.issues[0].msg).to.equal('String split by complex argument'); + }); it('generates a plural-key error when a source message is split by a complex arguemnt', () => { targetLocale = 'en'; @@ -97,7 +117,16 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('String split by non-argument (e.g. select; plural).'); }); - // arg + it('generates an argument error with unrecognized argument', () => { + const sourceString = 'An {arg}'; + const targetString = 'An {arG}'; + reporter.config(targetString, sourceString, 'key'); + validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('argument'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Unrecognized arguments: arG. Must be one of: arg'); + }); it('generates an argument error with unrecognized argument', () => { const sourceString = 'An {arg}'; @@ -112,13 +141,16 @@ describe('validate', () => { // brace - it.skip('does not generate a brace error with parseable mismatched braces', () => { - const sourceString = 'An {arg}'; - const targetString = 'An {arg}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(0); - }); + it('generates a brace error with unparseable mismatched braces', () => { + const sourceString = '{a, plural, one {An {arg}} other {}}'; + const targetString = '{a, plural, one {An {arg} other {}}'; + reporter.config(targetString, sourceString, 'key'); + validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('brace'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Mismatched braces. Expected identifier but "}" found.'); + }); it('generates a brace error with unparseable mismatched braces', () => { const sourceString = '{a, plural, one {An {arg}} other {}}'; From ca4ca048a2804204ce0f8272ee314d0d098f393b Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 18 Sep 2024 15:50:05 -0400 Subject: [PATCH 09/90] Add more format tests and improvements to make them pass --- src/format.js | 50 +++++++++++++++++----- test/format.test.js | 100 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 12 deletions(-) diff --git a/src/format.js b/src/format.js index db12423..fe16f10 100644 --- a/src/format.js +++ b/src/format.js @@ -1,7 +1,10 @@ import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.js'; import { parse } from '@formatjs/icu-messageformat-parser'; +import { structureRegEx } from './validate.js'; import cldr from 'cldr'; +const paddedQuoteLocales = ['fr', 'fr-ca', 'fr-fr', 'fr-on', 'vi-vn']; + function expandASTHashes(ast, parentValue) { if (Array.isArray(ast)) { ast.map(ast => expandASTHashes(ast, parentValue)); @@ -50,7 +53,7 @@ export function formatMessage(msg, options = {}) { } return printAST(ast, { - useNewlines: options.newlines ?? msg.includes('\n'), + useNewlines: options.newlines ?? msg.match(structureRegEx)?.join('').includes('\n'), add: options.add ?? false, remove: options.remove ?? false, dedupe: options.dedupe ?? false, @@ -85,6 +88,8 @@ function printAST(ast, options, level = 0) { args = [] } = options; + const localeLower = locale.toLowerCase(); + if (Array.isArray(ast)) { const swapOneClone = new Set(swapOne); ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) @@ -96,22 +101,37 @@ function printAST(ast, options, level = 0) { return cldr.extractDelimiters(locale.split('-')[0]); } })(); + + for (const k in delimiters) { + if (paddedQuoteLocales.includes(localeLower)) { + if (k.endsWith('Start')) { + delimiters[k] = delimiters[k].padEnd(2,'\u202f'); + } else if (k.endsWith('End')) { + delimiters[k] = delimiters[k].padStart(2,'\u202f'); + } + } + } + let quoteStart = delimiters.quotationStart, quoteEnd = delimiters.quotationEnd, singleQuoteStart = delimiters.alternateQuotationStart, singleQuoteEnd = delimiters.alternateQuotationEnd; - if (locale.toLowerCase().endsWith('-gb')) { - quoteStart = delimiters.alternateQuotationStart; - quoteEnd = delimiters.alternateQuotationEnd; - singleQuoteStart = delimiters.quotationStart; - singleQuoteEnd = delimiters.quotationEnd; + if (1) { // todo: fromSource + if (localeLower.endsWith('-gb')) { + quoteStart = delimiters.alternateQuotationStart; + quoteEnd = delimiters.alternateQuotationEnd; + singleQuoteStart = delimiters.quotationStart; + singleQuoteEnd = delimiters.quotationEnd; + } } - return ast.map((ast, idx, arr) => { - let trim; - if (options.trim) { + return ast + .filter((i, idx) => !trim || i.type !== 0 || (idx !== 0 && idx !== ast.length - 1) || i.value.trim()) // filter out leading and trailing whitespace + .map((ast, idx, arr) => { + let trim = options.trim; + if (trim && ast.type === 0) { if (arr.length === 1) { trim = 'trim'; } else if (!idx) { @@ -184,7 +204,7 @@ function printAST(ast, options, level = 0) { if (ast.options['=1'] && JSON.stringify(ast.options['=1']) === JSON.stringify(ast.options['one'])) { delete ast.options['=1']; } - } else if (ast.options['=1'] && /(? delete ast.options[cat]); + remove && unsupportedCats.forEach(cat => { + const currentKeys = Object.keys(ast.options); + if (currentKeys.includes(cat)) { + if (currentKeys.length === 1) { + ast.options.other = Object.assign({}, ast.options[cat]); + } + delete ast.options[cat]; + } + }); const typeText = ast.pluralType === 'ordinal' ? 'selectordinal' : 'plural'; const offsetText = + ast.offset !== 0 ? ` offset:${ast.offset}` : ''; diff --git a/test/format.test.js b/test/format.test.js index b1f348b..1405184 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -3,13 +3,18 @@ import { formatMessage } from '../src/format.js'; describe('formatMessage', () => { + let locale; + beforeEach(() => { + locale = 'en'; + }); + [ { locale: 'ar', expected: `This isn’t ”correct“` }, { locale: 'cy', expected: `This isn’t “correct”` }, { locale: 'de', expected: `This isn’t „correct“` }, { locale: 'en', expected: `This isn’t “correct”` }, { locale: 'en-gb', expected: `This isn’t ‘correct’` }, - { locale: 'fr', expected: `This isn’t «correct»` }, + { locale: 'fr', expected: `This isn’t «\u202fcorrect\u202f»` }, { locale: 'sv', expected: `This isn’t ”correct”` }, ].forEach(({ locale, expected }) => { it(`should replace straight quotes with "${locale}" quotes with no options`, () => { @@ -80,4 +85,97 @@ describe('formatMessage', () => { }); }); + it(`should not remove plural and selectordinal categories that are unsupported without the "remove" option`, () => { + const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(message); + }); + + it(`should insert newslines and tabs with the "newlines" option`, () => { + const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; + const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; + const formatted = formatMessage(message, { locale, newlines: true }); + expect(formatted).to.equal(expected); + }); + + it(`should insert newslines and tabs if the message structure already contains newlines with no option`, () => { + const message = `{a, plural,\none {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; + const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it.skip(`should remove duplicate categories with no options`, () => { + const message = `{a, plural, one {value} other {value2}}`; + const expected = `{a, plural, one {value}}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it(`should remove categories that are copies of a lower-precedence key with the "dedupe" option`, () => { + const message = `{a, plural, one {value} other {value}}`; + const expected = `{a, plural, other {value}}`; + const formatted = formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert "=1" keys to "one" when it contains a literal "1"`, () => { + const message = `{a, plural, =1 {value 1}}`; + const expected = `{a, plural, one {value {a}}}`; + const formatted = formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should remove "=1" cases when they can be converted to a duplicate case with the "dedupe" option`, () => { + const message = `{a, plural, =1 {value 1} other {value {a}}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should remove "=1" cases when they can be converted to unsupported "one" cases with the "remove" option`, () => { + locale = 'ja'; + const message = `{a, plural, =1 {value 1} other {value {a}}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert unsupported cases to "other" if there are no other cases`, () => { + locale = 'ja'; + const message = `{a, plural, two {value {a}}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert "=1" keys to "other" keys when they can be converted to unsupported "one" cases and there are no other keys with the "remove" option`, () => { + locale = 'ja'; + const message = `{a, plural, =1 {value 1}}}`; + const expected = `{a, plural, other {value {a}}}}`; + const formatted = formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + + it('should hoist complex selectors to the outside and nest appropriately with no options', () => { + const message = `\t{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; + const expected = `{a, plural, =1 {{b, plural, =1 {\ta cat and a dog!} other {\ta cat and {b} dogs!}}} other {{b, plural, =1 {\t{a} cats and a dog!} other {\t{a} cats and {b} dogs!}}}}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it(`should trim whitespace with the "trim" option`, () => { + const message = `\n{a, plural, other { value }}\t`; + const expected = `{a, plural, other {value}}`; + const formatted = formatMessage(message, { locale, trim: true }); + expect(formatted).to.equal(expected); + }); + + it(`should not trim internal whitespace with the "trim" option`, () => { + const message = `\n{a, plural, other { value {value2} value3 }}`; + const expected = `{a, plural, other {value {value2} value3}}`; + const formatted = formatMessage(message, { locale, trim: true }); + expect(formatted).to.equal(expected); + }); + }); From 6226ead9e196c772853ed8971657f79e788f9313 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 18 Sep 2024 17:35:44 -0400 Subject: [PATCH 10/90] "string" -> "message" --- bin/cli.js | 14 +- package.json | 2 +- src/format.js | 4 +- src/reporter.js | 12 +- src/validate.js | 59 ++++---- test/validate.test.js | 316 +++++++++++++++++++++++++++--------------- 6 files changed, 249 insertions(+), 158 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index ae82bac..2748168 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -315,14 +315,14 @@ localesPaths.forEach(async localesPath => { const keyIdx = keys.indexOf(issue.key); const nextKey = keys[keyIdx + 1]; const previousKey = keys[keyIdx - 1]; - const nextString = locales[locale.locale].parsed[nextKey]; - const siblingString = nextString || locales[locale.locale].parsed[previousKey] || locales[locale.locale].parsed[targetKeys[targetKeys.length - 1]]; + const nextMessage = locales[locale.locale].parsed[nextKey]; + const siblingMessage = nextMessage || locales[locale.locale].parsed[previousKey] || locales[locale.locale].parsed[targetKeys[targetKeys.length - 1]]; const contents = locales[locale.locale].contents; - const insertAt = contents.indexOf(siblingString) + Number(!nextString ? String(siblingString).length : 0); - const comma = !nextString && !siblingString.comma ? `,${siblingString.comment}` : ''; - const commaOffset = comma ? siblingString.comment.length : 0; - const sourceString = `${comma}${locales[sourceLocale].parsed[issue.key]}`; - locales[locale.locale].contents = [contents.slice(0, insertAt - commaOffset), sourceString, contents.slice(insertAt)].join(''); + const insertAt = contents.indexOf(siblingMessage) + Number(!nextMessage ? String(siblingMessage).length : 0); + const comma = !nextMessage && !siblingMessage.comma ? `,${siblingMessage.comment}` : ''; + const commaOffset = comma ? siblingMessage.comment.length : 0; + const sourceMessage = `${comma}${locales[sourceLocale].parsed[issue.key]}`; + locales[locale.locale].contents = [contents.slice(0, insertAt - commaOffset), sourceMessage, contents.slice(insertAt)].join(''); console.log('Added:', issue.key); locales[locale.locale].parsed[issue.key] = locales[sourceLocale].parsed[issue.key]; } diff --git a/package.json b/package.json index 8a899ce..90f432f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "messageformat-validator", "version": "3.0.0-alpha.1", - "description": "Validates that ICU MessageFormat strings are well-formed, and that translated target strings are compatible with their source.", + "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { "type": "git", diff --git a/src/format.js b/src/format.js index fe16f10..950144a 100644 --- a/src/format.js +++ b/src/format.js @@ -118,14 +118,14 @@ function printAST(ast, options, level = 0) { singleQuoteStart = delimiters.alternateQuotationStart, singleQuoteEnd = delimiters.alternateQuotationEnd; - if (1) { // todo: fromSource + //if (1) { // todo: fromSource if (localeLower.endsWith('-gb')) { quoteStart = delimiters.alternateQuotationStart; quoteEnd = delimiters.alternateQuotationEnd; singleQuoteStart = delimiters.quotationStart; singleQuoteEnd = delimiters.quotationEnd; } - } + //} return ast .filter((i, idx) => !trim || i.type !== 0 || (idx !== 0 && idx !== ast.length - 1) || i.value.trim()) // filter out leading and trailing whitespace diff --git a/src/reporter.js b/src/reporter.js index 3d4a3eb..b839997 100644 --- a/src/reporter.js +++ b/src/reporter.js @@ -6,10 +6,10 @@ export function Reporter(locale, fileContents = '') { this.issues = []; } -Reporter.prototype.config = function(targetString, sourceString, key) { - this._config.key = key || targetString.key; - if (typeof targetString !== "undefined") this._config.target = targetString; - if (typeof sourceString !== "undefined") this._config.source = sourceString; +Reporter.prototype.config = function(targetMessage, sourceMessage, key) { + this._config.key = key || targetMessage.key; + if (typeof targetMessage !== "undefined") this._config.target = targetMessage; + if (typeof sourceMessage !== "undefined") this._config.source = sourceMessage; }; Reporter.prototype.log = function(level, type, msg, column = 0, givenLine) { @@ -62,7 +62,7 @@ Reporter.prototype.warning = function(type, msg, details = {}) { column = (valPos + 1) - linePos + relativeColumn; if (valPos === -1) { - // the target string likely contains a backslash that does not escape anything + // the target message likely contains a backslash that does not escape anything column = 0; } } @@ -92,7 +92,7 @@ Reporter.prototype.error = function(type, msg, details = {}) { column -= this._config.target.lastIndexOf('\n', column); if (valPos === -1) { - // the target string likely contains a backslash that does not escape anything + // the target message likely contains a backslash that does not escape anything column = 0; } } diff --git a/src/validate.js b/src/validate.js index c5c6541..996f9d8 100644 --- a/src/validate.js +++ b/src/validate.js @@ -11,45 +11,46 @@ let reporter; export function validateLocales({ locales, sourceLocale }, localesReporter) { - const sourceStrings = locales[sourceLocale].parsed; + const sourceMessages = locales[sourceLocale].parsed; return Object.keys(locales).map((targetLocale) => { reporter = localesReporter ?? new Reporter(targetLocale, locales[targetLocale].contents); - const targetStrings = locales[targetLocale].parsed; + const targetMessages = locales[targetLocale].parsed; const checkedKeys = []; - Object.keys(targetStrings).forEach(key => { + Object.keys(targetMessages).forEach(key => { checkedKeys.push(key); - const targetString = targetStrings?.[key].val; - const sourceString = sourceStrings?.[key]?.val || ''; - const overrides = Array.from(targetStrings?.[key].comment.matchAll(/mfv-(?[a-z]+)/g)).map(m => m.groups.override) + const targetMessage = targetMessages?.[key].val; + const sourceMessage = sourceMessages?.[key]?.val || ''; + const overrides = Array.from(targetMessages?.[key].comment.matchAll(/mfv-(?[a-z]+)/g)).map(m => m.groups.override) - reporter.config(targetStrings[key], sourceStrings[key]); - if (!sourceString) { - reporter.error('extraneous', 'This string does not exist in the source file.'); + reporter.config(targetMessages[key], sourceMessages[key]); + + if (!sourceMessage) { + reporter.error('extraneous', 'Message does not exist in the source file.'); } else { if (locales[targetLocale].duplicateKeys.has(key)) reporter.error('duplicate-keys', 'Key appears multiple times'); validateMessage({ - targetString, + targetMessage, targetLocale, - sourceString, + sourceMessage, sourceLocale, overrides }, reporter); } }); - const missingKeys = Object.keys(sourceStrings).filter(arg => !checkedKeys.includes(arg)); + const missingKeys = Object.keys(sourceMessages).filter(arg => !checkedKeys.includes(arg)); if (missingKeys.length) { missingKeys.forEach((key) => { - reporter.config(sourceStrings[key], sourceStrings[key]); - reporter.error('missing', `String missing from locale file.`) + reporter.config(sourceMessages[key], sourceMessages[key]); + reporter.error('missing', `Message missing from locale file.`) }) } @@ -63,27 +64,27 @@ export function validateLocales({ locales, sourceLocale }, localesReporter) { }); } -export function validateMessage({ targetString, targetLocale, sourceString, sourceLocale, overrides }, msgReporter = reporter) { +export function validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale, overrides }, msgReporter = reporter) { const re = /[\u2000-\u206F\u2E00-\u2E7F\n\r\\'!"#$%&()*+,\-.\/∕:;<=>?@\[\]^_`{|}~]/g; // eslint-disable-line if (sourceLocale && targetLocale.split('-')[0] !== sourceLocale.split('-')[0] - && targetString.replace(re,'') === sourceString.replace(re,'')) { + && targetMessage.replace(re,'') === sourceMessage.replace(re,'')) { if (!overrides?.includes('translated') - && sourceString + && sourceMessage .replace(structureRegEx, '') .replace(re,'') .replace(/\s/g, '')) { - msgReporter.warning('untranslated', `String has not been translated.`); + msgReporter.warning('untranslated', `Message has not been translated.`); } } let parsedTarget; try { - parsedTarget = Object.freeze(parse(targetString, getPluralCats(targetLocale))); + parsedTarget = Object.freeze(parse(targetMessage, getPluralCats(targetLocale))); } catch(e) { @@ -91,10 +92,10 @@ export function validateMessage({ targetString, targetLocale, sourceString, sour const backtickCaptures = e.message.match(/`([^`]*)`/g); const badKey = backtickCaptures[0].slice(1, -1); const pluralArg = backtickCaptures[1].slice(1, -1) - const column = targetString.indexOf(badKey, targetString.indexOf(`{${pluralArg}, plural, {`)); + const column = targetMessage.indexOf(badKey, targetMessage.indexOf(`{${pluralArg}, plural, {`)); msgReporter.error('categories', e.message, { column }); } - else if ((targetString.match(/{/g) || 0).length !== (targetString.match(/}/g) || 0).length) { + else if ((targetMessage.match(/{/g) || 0).length !== (targetMessage.match(/}/g) || 0).length) { msgReporter.error('brace', 'Mismatched braces. ' + e.message, { column: e.location.start.column }); } else { @@ -108,10 +109,10 @@ export function validateMessage({ targetString, targetLocale, sourceString, sour let sourceTokens; try { - sourceTokens = parse(sourceString, getPluralCats(sourceLocale)); + sourceTokens = parse(sourceMessage, getPluralCats(sourceLocale)); } catch(e) { - msgReporter.error('source-error', 'Failed to parse source string.'); + msgReporter.error('source-error', 'Failed to parse source message.'); return; } @@ -142,17 +143,17 @@ export function validateMessage({ targetString, targetLocale, sourceString, sour const argDiff = Array.from(targetMap.arguments).filter(arg => !Array.from(sourceMap.arguments).includes(arg)); - const badArgPos = targetString.indexOf(argDiff[0]); + const badArgPos = targetMessage.indexOf(argDiff[0]); if (argDiff.length) { msgReporter.error('argument', `Unrecognized arguments: ${argDiff.join(', ')}. Must be one of: ${Array.from(sourceMap.arguments).join(', ')}`, { column: badArgPos }); } // remove all translated content, leaving only the messageformat structure - const structure = targetString.match(structureRegEx)?.join('') || ''; + const structure = targetMessage.match(structureRegEx)?.join('') || ''; const nbspPos = structure.indexOf(String.fromCharCode(160)); if (nbspPos > -1) { - msgReporter.error('nbsp', `String contains invalid non-breaking space at position ${nbspPos}.`, { column: nbspPos }); + msgReporter.error('nbsp', `Message contains invalid non-breaking space at position ${nbspPos}.`, { column: nbspPos }); } if (targetMap.cases.join(',') !== sourceMap.cases.join(',')) { @@ -179,7 +180,7 @@ export function validateMessage({ targetString, targetLocale, sourceString, sour if (targetTokens.length > 1) { if (targetLocale == sourceLocale && targetTokens.find((token) => typeof token !== 'string' && token.type.match(/plural|select/))) { - msgReporter.warning('split','String split by complex argument') + msgReporter.warning('split','Message split by complex argument') } } } @@ -245,7 +246,7 @@ export function parseLocales(locales, useJSONObj) { }, {}); } -function _map(tokens, partsMap = { nested: false, arguments: new Set(), cases: [], stringTokens: [] }) { +function _map(tokens, partsMap = { nested: false, arguments: new Set(), cases: [], messageTokens: [] }) { tokens.forEach(token => { @@ -275,7 +276,7 @@ function _map(tokens, partsMap = { nested: false, arguments: new Set(), cases: [ } } else { - partsMap.stringTokens.push(token); + partsMap.messageTokens.push(token); } }); diff --git a/test/validate.test.js b/test/validate.test.js index 96b2365..1d80026 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -31,15 +31,26 @@ describe('validate', () => { describe('validateMessage', () => { - // untranslated + it('generates no issues with identical same-language messages', () => { + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arg}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(0); + }); - it('generates no issues with identical same-language messages', () => { - const sourceString = 'An {arg}'; - const targetString = 'An {arg}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(0); - }); + it('generates an untranslated warning when messages are the same and languages are different', () => { + targetLocale = 'es-mx'; + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arg}'; + reporter.config(targetMessage, sourceMessage, 'key'); + reporter._config.locale = targetLocale; + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('untranslated'); + expect(reporter.issues[0].level).to.equal('warning'); + expect(reporter.issues[0].msg).to.equal('Message has not been translated.'); + }); it('generates an untranslated warning when messages are the same and languages are different', () => { targetLocale = 'es-mx'; @@ -54,26 +65,24 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('String has not been translated.'); }); - // categories - - it('generates a categories warning when a target message is missing supported plural categories', () => { - targetLocale = 'cy-gb'; - const sourceString = '{a, plural, one {} other {}}'; - const targetString = '{a, plural, one {} other {}}'; - reporter.config(targetString, sourceString, 'key'); - reporter._config.locale = targetLocale; - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('categories'); - expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Missing categories: ["zero","two","few","many"]'); - }); + it('generates a categories warning when a target message is missing supported plural categories', () => { + targetLocale = 'cy-gb'; + const sourceMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, one {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + reporter._config.locale = targetLocale; + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].level).to.equal('warning'); + expect(reporter.issues[0].msg).to.equal('Missing categories: ["zero","two","few","many"]'); + }); it('generates a categories error when a target message uses unsupported plural categories', () => { - const sourceString = '{a, plural, one {} other {}}'; - const targetString = '{a, plural, one {} two {} few {} many {} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + const sourceMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, one {} two {} few {} many {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('categories'); expect(reporter.issues[0].level).to.equal('error'); @@ -93,15 +102,15 @@ describe('validate', () => { it('generates a split error when a source message is split by a complex argument', () => { targetLocale = 'en'; - const sourceString = '{a, plural, one {} other {}} b'; - const targetString = '{a, plural, one {} other {}} b'; - reporter.config(targetString, sourceString, 'key'); + const sourceMessage = '{a, plural, one {} other {}} b'; + const targetMessage = '{a, plural, one {} other {}} b'; + reporter.config(targetMessage, sourceMessage, 'key'); reporter._config.locale = targetLocale; - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('split'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('String split by complex argument'); + expect(reporter.issues[0].msg).to.equal('Message split by complex argument'); }); it('generates a plural-key error when a source message is split by a complex arguemnt', () => { @@ -118,10 +127,10 @@ describe('validate', () => { }); it('generates an argument error with unrecognized argument', () => { - const sourceString = 'An {arg}'; - const targetString = 'An {arG}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arG}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('argument'); expect(reporter.issues[0].level).to.equal('error'); @@ -139,29 +148,32 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Unrecognized arguments ["arG"]'); }); - // brace + it.skip('does not generate a brace error with parseable mismatched braces', () => { + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arg}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(0); + }); it('generates a brace error with unparseable mismatched braces', () => { - const sourceString = '{a, plural, one {An {arg}} other {}}'; - const targetString = '{a, plural, one {An {arg} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + const sourceMessage = '{a, plural, one {An {arg}} other {}}'; + const targetMessage = '{a, plural, one {An {arg} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('brace'); expect(reporter.issues[0].level).to.equal('error'); expect(reporter.issues[0].msg).to.equal('Mismatched braces. Expected identifier but "}" found.'); }); - it('generates a brace error with unparseable mismatched braces', () => { - const sourceString = '{a, plural, one {An {arg}} other {}}'; - const targetString = '{a, plural, one {An {arg} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('brace'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Mismatched braces (i.e. {}). Expected identifier but "}" found.'); - }); + it('does not generate a brace error with escaped mismatched braces', () => { + const sourceMessage = '{a, plural, one {An {arg}} other {}}'; + const targetMessage = '{a, plural, one {An {arg}\'}\'} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(0); + }); it('does not generate a brace error with escaped mismatched braces', () => { const sourceString = '{a, plural, one {An {arg}} other {}}'; @@ -171,18 +183,27 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(0); }); - // case + it('generates a case error with unrecognized cases in select arguments', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, b {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('case'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Unrecognized cases ["b"]'); + }); - it('generates a case error with unrecognized cases in select arguments', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select, b {} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('case'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized cases ["b"]'); - }); + it.skip('generates a case error with missing cases in select arguments', () => { + const sourceMessage = '{a, select, b {} other {}}'; + const targetMessage = '{a, select, other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('case'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Missing cases ["b"]'); + }); it.skip('generates a case error with missing cases in select arguments', () => { const sourceString = '{a, select, b {} other {}}'; @@ -195,7 +216,15 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Missing cases ["b"]'); }); - // nbsp + it('generates an nbsp error with non-breaking space in the messageformat structure', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select,\u00A0other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(2); + expect(reporter.issues[0].type).to.equal('nbsp'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Message contains invalid non-breaking space at position 11.'); it('generates an nbsp error with non-breaking space in the messageformat structure', () => { const sourceString = '{a, select, other {}}'; @@ -212,18 +241,27 @@ describe('validate', () => { expect(reporter.issues[1].msg).to.equal('Unrecognized cases ["\u00A0other"]'); }); - // nest + it('generates a nest-order error with mismatched complex argument order', () => { + const sourceMessage = '{a, select, other {{b, select, other {}}}}'; + const targetMessage = '{b, select, other {{a, select, other {}}}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('nest-order'); + expect(reporter.issues[0].level).to.equal('warning'); + expect(reporter.issues[0].msg).to.equal('Nesting order does not match source.'); + }); - it('generates a nest-order error with mismatched complex argument order', () => { - const sourceString = '{a, select, other {{b, select, other {}}}}'; - const targetString = '{b, select, other {{a, select, other {}}}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('nest-order'); - expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Nesting order does not match source.'); - }); + it('generates a nest-ideal error with plural inside select', () => { + const sourceMessage = '{a, plural, one {} other {{b, select, other {}}}}'; + const targetMessage = '{a, plural, one {} other {{b, select, other {}}}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('nest-ideal'); + expect(reporter.issues[0].level).to.equal('warning'); + expect(reporter.issues[0].msg).to.equal('"plural" and "selectordinal" should always nest inside "select".'); + }); it('generates a nest-ideal error with plural inside select', () => { const sourceString = '{a, plural, one {} other {{b, select, other {}}}}'; @@ -236,7 +274,16 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('"plural" and "selectordinal" should always nest inside "select".'); }); - // other + it('generates an other error with missing other case', () => { + const sourceMessage = '{a, select, b {}}'; + const targetMessage = '{a, select, b {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('other'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Missing "other" case'); + }); it('generates an other error with missing other case', () => { const sourceString = '{a, select, b {}}'; @@ -249,7 +296,16 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Missing "other" case'); }); - // parse + it('generates a parse error with an unparseable target message', () => { + const sourceMessage = '{a, select, b {}}'; + const targetMessage = '{a, select b {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('parse'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found.'); + }); it('generates a parse error with an unparseable target message', () => { const sourceString = '{a, select, b {}}'; @@ -262,7 +318,16 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found.'); }); - // source + it('generates a source-error error an unparseable source message', () => { + const sourceMessage = '{a, select b {}}'; + const targetMessage = '{a, select, b {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('source-error'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Failed to parse source message.'); + }); it('generates a source-error error an unparseable source message', () => { const sourceString = '{a, select b {}}'; @@ -279,24 +344,29 @@ describe('validate', () => { describe('validateLocales', () => { - // extraneous - - it('generates an extraneous error with unexpected message in target locale', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select, other {}}'; - const locales = parseLocales([{ - file: `${targetLocale}.json`, - contents: JSON.stringify({ - a: targetString, - b: targetString - }, null, '\t') - }, - { - file: `${sourceLocale}.json`, - contents: JSON.stringify({ - a: sourceString - }, null, '\t') - }]); + it('generates an extraneous error with unexpected message in target locale', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, other {}}'; + const locales = parseLocales([{ + file: `${targetLocale}.json`, + contents: JSON.stringify({ + a: targetMessage, + b: targetMessage + }, null, '\t') + }, + { + file: `${sourceLocale}.json`, + contents: JSON.stringify({ + a: sourceMessage + }, null, '\t') + }]); + + validateLocales({ locales, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('extraneous'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Message does not exist in the source file.'); + }); validateLocales({ locales, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); @@ -305,24 +375,29 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('This string does not exist in the source file.'); }); - // missing - - it('generates a missing error with missing message in the target locale', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select, other {}}'; - const locales = parseLocales([{ - file: `${targetLocale}.json`, - contents: JSON.stringify({ - a: targetString - }, null, '\t') - }, - { - file: `${sourceLocale}.json`, - contents: JSON.stringify({ - a: sourceString, - b: sourceString - }, null, '\t') - }]); + it('generates a missing error with missing message in the target locale', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, other {}}'; + const locales = parseLocales([{ + file: `${targetLocale}.json`, + contents: JSON.stringify({ + a: targetMessage + }, null, '\t') + }, + { + file: `${sourceLocale}.json`, + contents: JSON.stringify({ + a: sourceMessage, + b: sourceMessage + }, null, '\t') + }]); + + validateLocales({ locales, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('missing'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Message missing from locale file.'); + }); validateLocales({ locales, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); @@ -331,7 +406,22 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('String missing from locale file.'); }); - // duplicate-keys + it('generates a duplicate-keys error with duplicate messages in the target locale', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, other {}}'; + const locales = parseLocales([{ + file: `${targetLocale}.json`, + contents: `{ + "a": "${sourceMessage}", + "a": "${sourceMessage}" + }` + }, + { + file: `${sourceLocale}.json`, + contents: `{ + "a": "${targetMessage}" + }` + }]); it('generates a duplicate-keys error with duplicate messages in the target locale', () => { const sourceString = '{a, select, other {}}'; From 6b2b3aa39a410227e7fbbb3e82c7c4734788bf90 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 18 Sep 2024 21:45:38 -0400 Subject: [PATCH 11/90] Use @formatjs/icu-messageformat-parser in the validator --- package-lock.json | 8 +-- package.json | 5 +- src/validate.js | 123 +++++++++++++++++++++++------------------- test/validate.test.js | 111 +++++++++++++++++++------------------- 4 files changed, 128 insertions(+), 119 deletions(-) diff --git a/package-lock.json b/package-lock.json index a748303..6dcf13c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,8 +16,7 @@ "esm": "^3.2.25", "find-config": "^1.0.0", "glob": "^7.1.6", - "make-plural": "^7.4.0", - "messageformat-parser": "^4.1.3" + "make-plural": "^7.4.0" }, "bin": { "mfv": "bin/cli.js" @@ -2007,11 +2006,6 @@ "passerror": "1.1.1" } }, - "node_modules/messageformat-parser": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-4.1.3.tgz", - "integrity": "sha512-2fU3XDCanRqeOCkn7R5zW5VQHWf+T3hH65SzuqRvjatBK7r4uyFa5mEX+k6F9Bd04LVM5G4/BHBTUJsOdW7uyg==" - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", diff --git a/package.json b/package.json index 90f432f..1bbd119 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "files": [ "/src" ], - "author": "Daniel Gleckler ", + "author": "Danny Gleckler ", "license": "MIT", "dependencies": { "@formatjs/icu-messageformat-parser": "^2.7.8", @@ -34,8 +34,7 @@ "esm": "^3.2.25", "find-config": "^1.0.0", "glob": "^7.1.6", - "make-plural": "^7.4.0", - "messageformat-parser": "^4.1.3" + "make-plural": "^7.4.0" }, "devDependencies": { "@babel/eslint-parser": "^7.25.1", diff --git a/src/validate.js b/src/validate.js index 996f9d8..12c4aa8 100644 --- a/src/validate.js +++ b/src/validate.js @@ -1,6 +1,11 @@ import * as pluralCats from 'make-plural/pluralCategories' import { Reporter } from './reporter.js'; -import { parse } from 'messageformat-parser'; +import { parse } from '@formatjs/icu-messageformat-parser'; + +const SELECT = 5; +const SELECTORDINAL = 6; +const PLURAL = 6; +const ARGUMENT = 1; function getPluralCats(locale) { return pluralCats[locale.split('-')[0]] || pluralCats.en; @@ -64,6 +69,16 @@ export function validateLocales({ locales, sourceLocale }, localesReporter) { }); } +function checkNbsp(message, reporter) { + const structure = message.match(structureRegEx)?.join('') || ''; + const nbspPos = structure.indexOf(String.fromCharCode(160)); + + if (nbspPos > -1) { + reporter.error('nbsp', `Message contains invalid non-breaking space at position ${nbspPos}.`, { column: nbspPos }); + return true; + } +} + export function validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale, overrides }, msgReporter = reporter) { const re = /[\u2000-\u206F\u2E00-\u2E7F\n\r\\'!"#$%&()*+,\-.\/∕:;<=>?@\[\]^_`{|}~]/g; // eslint-disable-line @@ -84,22 +99,19 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so let parsedTarget; try { - parsedTarget = Object.freeze(parse(targetMessage, getPluralCats(targetLocale))); + parsedTarget = Object.freeze(parse(targetMessage, { captureLocation: true, requiresOtherClause: false })); } catch(e) { - - if (e.message.indexOf('Invalid key') === 0) { - const backtickCaptures = e.message.match(/`([^`]*)`/g); - const badKey = backtickCaptures[0].slice(1, -1); - const pluralArg = backtickCaptures[1].slice(1, -1) - const column = targetMessage.indexOf(badKey, targetMessage.indexOf(`{${pluralArg}, plural, {`)); - msgReporter.error('categories', e.message, { column }); - } - else if ((targetMessage.match(/{/g) || 0).length !== (targetMessage.match(/}/g) || 0).length) { - msgReporter.error('brace', 'Mismatched braces. ' + e.message, { column: e.location.start.column }); + if ((targetMessage.match(/{/g) || 0).length !== (targetMessage.match(/}/g) || 0).length) { + msgReporter.error('brace', 'Mismatched braces', { column: e.location.start.column }); } - else { - msgReporter.error('parse', e.message, { column: e.location.start.column - 1 }); + else if (!checkNbsp(targetMessage, msgReporter)) { + if (e.message === 'EXPECT_SELECT_ARGUMENT_OPTIONS') { + msgReporter.error('parse', `Expected "," but "${e.originalMessage.substr(e.location.start.column, 1)}" found`, { column: e.location.start.column - 1 }); + } + else { + msgReporter.error('parse', e.message, { column: e.location.start.column - 1 }); + } } } @@ -109,34 +121,41 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so let sourceTokens; try { - sourceTokens = parse(sourceMessage, getPluralCats(sourceLocale)); + sourceTokens = parse(sourceMessage, { requiresOtherClause: false }); } catch(e) { msgReporter.error('source-error', 'Failed to parse source message.'); return; } - const checkCases = target => { + const checkCases = (target, msg) => { target?.forEach(part => { - if (['select', 'selectordinal', 'plural'].includes(part.type)) { - const hasOther = part.cases.find(c => c.key.trim() === 'other'); - if (!hasOther) { - msgReporter.error('other', 'Missing "other" case'); + if ([SELECT, PLURAL].includes(part.type)) { + if (!part.options.other) { + msgReporter.error('other', 'Missing "other" option'); } - if (part.type !== 'select') { - const pluralType = part.type.includes('ordinal') ? 'ordinal' : 'cardinal'; - const allowedCats = getPluralCats(targetLocale)[pluralType]; - const cats = part.cases.map(c => c.key); - const missingCats = allowedCats.filter(c => c !== 'other' && !cats.includes(c)); + if (part.type === PLURAL) { + const supportedCats = getPluralCats(targetLocale)[part.pluralType]; + const cats = Object.keys(part.options); + const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); if (missingCats.length) msgReporter.warning('categories', `Missing categories: ${JSON.stringify(missingCats)}`); + const unsupportedCats = cats.filter(c => !supportedCats.includes(c)); + + unsupportedCats.forEach(cat => { + const column = part.options[cat].location.start.offset; + msgReporter.error('categories', `Unsupported category "${cat}". Must be one of: "${supportedCats.join('", "')}", or explicit keys like "=0"`, { column }); + }); } + + Object.values(part.options).forEach(o => { + checkCases(o.value, msg); + }); } - checkCases(target.cases); }); }; - checkCases(parsedTarget); + checkCases(parsedTarget, targetMessage); const targetMap = _map(targetTokens); const sourceMap = _map(sourceTokens); @@ -148,38 +167,33 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so msgReporter.error('argument', `Unrecognized arguments: ${argDiff.join(', ')}. Must be one of: ${Array.from(sourceMap.arguments).join(', ')}`, { column: badArgPos }); } - // remove all translated content, leaving only the messageformat structure - const structure = targetMessage.match(structureRegEx)?.join('') || ''; - - const nbspPos = structure.indexOf(String.fromCharCode(160)); - if (nbspPos > -1) { - msgReporter.error('nbsp', `Message contains invalid non-breaking space at position ${nbspPos}.`, { column: nbspPos }); - } + checkNbsp(targetMessage, msgReporter); if (targetMap.cases.join(',') !== sourceMap.cases.join(',')) { - const cleanTargetCases = targetMap.cases.map(c => c.replace(/.+(?<=\|(plural|selectordinal)\|).*/, '')); - const cleanSourceCases = sourceMap.cases.map(c => c.replace(/.+(?<=\|(plural|selectordinal)\|).*/, '')); - const caseDiff = cleanTargetCases.filter(arg => !cleanSourceCases.includes(arg)); + const cleanTargetCases = targetMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); + const cleanSourceCases = sourceMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); + const optionDiff = cleanTargetCases.filter(arg => !cleanSourceCases.includes(arg)); - if (caseDiff.length) { - msgReporter.error('case', `Unrecognized cases ${JSON.stringify(caseDiff.map(c => c.replace(/.+\|select\|/, '')))}`); - } - else if (targetMap.nested && targetMap.cases.length === sourceMap.cases.length) { + optionDiff.forEach(o => { + msgReporter.error('option', `Unrecognized option "${o.replace(/.+\|5undefined\|/, '')}". Must be one of "${cleanSourceCases.map(o => o.replace(/.+\|5undefined\|/, '')).join('", "')}".`); + }); + + if (targetMap.nested && targetMap.cases.length === sourceMap.cases.length) { // TODO: better identify case order vs nesting order msgReporter.warning('nest-order', `Nesting order does not match source.`); } } - const firstPlural = targetMap.cases.findIndex(i => i.match(/^.+\|(plural|selectordinal)\|/)) + 1; - const lastSelect = targetMap.cases.findLastIndex(i => i.match(/^.+\|select\|/)) + 1; + const firstPlural = targetMap.cases.findIndex(i => i.match(/^.+\|(6(ordinal|cardinal))\|/)) + 1; + const lastSelect = targetMap.cases.findLastIndex(i => i.match(/^.+\|5undefined\|/)) + 1; if (targetMap.nested && firstPlural && lastSelect && firstPlural < lastSelect) { msgReporter.warning('nest-ideal', '"plural" and "selectordinal" should always nest inside "select".'); } if (targetTokens.length > 1) { - if (targetLocale == sourceLocale && targetTokens.find((token) => typeof token !== 'string' && token.type.match(/plural|select/))) { + if (targetLocale == sourceLocale && targetTokens.find((token) => typeof token !== 'string' && [PLURAL, SELECT].includes(token.type))) { msgReporter.warning('split','Message split by complex argument') } } @@ -246,32 +260,31 @@ export function parseLocales(locales, useJSONObj) { }, {}); } -function _map(tokens, partsMap = { nested: false, arguments: new Set(), cases: [], messageTokens: [] }) { +function _map(ast, partsMap = { nested: false, arguments: new Set(), cases: [], messageTokens: [] }) { - tokens.forEach(token => { + ast.forEach(token => { if (typeof token !== 'string') { - if (token.arg) { - partsMap.arguments.add(token.arg); + if (token.type === ARGUMENT) { + partsMap.arguments.add(token.value); } - if (token.cases) { + if (token.options) { if (partsMap.cases.length) { partsMap.nested = true; } - token.cases.forEach((case_) => { + Object.entries(token.options).forEach(([k, option]) => { switch (token.type) { - case 'select': - case 'plural': - case 'selectordinal': - partsMap.cases.push(`${token.arg}|${token.type}|${case_.key}`); + case SELECT: + case PLURAL: + partsMap.cases.push(`${token.value}|${token.type}${token.pluralType}|${k}`); break; } - _map(case_.tokens, partsMap); + _map(option.value, partsMap); }); } } diff --git a/test/validate.test.js b/test/validate.test.js index 1d80026..95721b0 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -39,7 +39,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(0); }); - it('generates an untranslated warning when messages are the same and languages are different', () => { + it('generates an "untranslated" warning when messages are the same and languages are different', () => { targetLocale = 'es-mx'; const sourceMessage = 'An {arg}'; const targetMessage = 'An {arg}'; @@ -65,7 +65,7 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('String has not been translated.'); }); - it('generates a categories warning when a target message is missing supported plural categories', () => { + it('generates a "categories" warning when a target message is missing supported plural categories', () => { targetLocale = 'cy-gb'; const sourceMessage = '{a, plural, one {} other {}}'; const targetMessage = '{a, plural, one {} other {}}'; @@ -78,15 +78,25 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Missing categories: ["zero","two","few","many"]'); }); - it('generates a categories error when a target message uses unsupported plural categories', () => { + it('generates "categories" errors when a target message uses unsupported plural categories', () => { const sourceMessage = '{a, plural, one {} other {}}'; const targetMessage = '{a, plural, one {} two {} few {} many {} other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); + + expect(reporter.issues.length).to.equal(3); + expect(reporter.issues[0].type).to.equal('categories'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Invalid key `two` for argument `a`. Valid plural keys for this locale are `one`, `other`, and explicit keys like `=0`.'); + expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Must be one of: "one", "other", or explicit keys like "=0"'); + + expect(reporter.issues[1].type).to.equal('categories'); + expect(reporter.issues[1].level).to.equal('error'); + expect(reporter.issues[1].msg).to.equal('Unsupported category "few". Must be one of: "one", "other", or explicit keys like "=0"'); + + expect(reporter.issues[2].type).to.equal('categories'); + expect(reporter.issues[2].level).to.equal('error'); + expect(reporter.issues[2].msg).to.equal('Unsupported category "many". Must be one of: "one", "other", or explicit keys like "=0"'); }); it('generates a plural-key error when a target message uses unsupported plural categories', () => { @@ -100,7 +110,7 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Invalid key `two` for argument `a`. Valid plural keys for this locale are `one`, `other`, and explicit keys like `=0`.'); }); - it('generates a split error when a source message is split by a complex argument', () => { + it('generates a "split" error when a source message is split by a complex argument', () => { targetLocale = 'en'; const sourceMessage = '{a, plural, one {} other {}} b'; const targetMessage = '{a, plural, one {} other {}} b'; @@ -126,7 +136,7 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('String split by non-argument (e.g. select; plural).'); }); - it('generates an argument error with unrecognized argument', () => { + it('generates an "argument" error with unrecognized argument', () => { const sourceMessage = 'An {arg}'; const targetMessage = 'An {arG}'; reporter.config(targetMessage, sourceMessage, 'key'); @@ -148,7 +158,7 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Unrecognized arguments ["arG"]'); }); - it.skip('does not generate a brace error with parseable mismatched braces', () => { + it('does not generate a "brace" error with parseable mismatched braces', () => { const sourceMessage = 'An {arg}'; const targetMessage = 'An {arg}}'; reporter.config(targetMessage, sourceMessage, 'key'); @@ -156,18 +166,18 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(0); }); - it('generates a brace error with unparseable mismatched braces', () => { - const sourceMessage = '{a, plural, one {An {arg}} other {}}'; - const targetMessage = '{a, plural, one {An {arg} other {}}'; + it('generates a "brace" error with unparseable mismatched braces', () => { + const sourceMessage = '{a, plural, one {An {arg}} other {{a} args}}'; + const targetMessage = '{a, plural, one {An {arg} other {{a} args}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('brace'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Mismatched braces. Expected identifier but "}" found.'); + expect(reporter.issues[0].msg).to.equal('Mismatched braces'); }); - it('does not generate a brace error with escaped mismatched braces', () => { + it('does not generate a "brace" error with escaped mismatched braces', () => { const sourceMessage = '{a, plural, one {An {arg}} other {}}'; const targetMessage = '{a, plural, one {An {arg}\'}\'} other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); @@ -175,32 +185,32 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(0); }); - it('does not generate a brace error with escaped mismatched braces', () => { - const sourceString = '{a, plural, one {An {arg}} other {}}'; - const targetString = '{a, plural, one {An {arg}\'}\'} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(0); - }); + // option - it('generates a case error with unrecognized cases in select arguments', () => { - const sourceMessage = '{a, select, other {}}'; - const targetMessage = '{a, select, b {} other {}}'; + it('generates "option" errors with unrecognized cases in select arguments', () => { + const sourceMessage = '{a, select, b {} other {}}'; + const targetMessage = '{a, select, B {} C {} other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('case'); + + expect(reporter.issues.length).to.equal(2); + + expect(reporter.issues[0].type).to.equal('option'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized cases ["b"]'); + expect(reporter.issues[0].msg).to.equal('Unrecognized option "B". Must be one of "b", "other".'); + + expect(reporter.issues[1].type).to.equal('option'); + expect(reporter.issues[1].level).to.equal('error'); + expect(reporter.issues[1].msg).to.equal('Unrecognized option "C". Must be one of "b", "other".'); }); - it.skip('generates a case error with missing cases in select arguments', () => { + it.skip('generates a "option" error with missing cases in select arguments', () => { const sourceMessage = '{a, select, b {} other {}}'; const targetMessage = '{a, select, other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('case'); + expect(reporter.issues[0].type).to.equal('option'); expect(reporter.issues[0].level).to.equal('error'); expect(reporter.issues[0].msg).to.equal('Missing cases ["b"]'); }); @@ -216,32 +226,24 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Missing cases ["b"]'); }); - it('generates an nbsp error with non-breaking space in the messageformat structure', () => { - const sourceMessage = '{a, select, other {}}'; - const targetMessage = '{a, select,\u00A0other {}}'; + it('generates an "nbsp" error with non-breaking space in the messageformat structure', () => { + const sourceMessage = '{a, select, a {} other {}}'; + const targetMessage = '{a, select,\u00A0a {} other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(2); + + expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('nbsp'); expect(reporter.issues[0].level).to.equal('error'); expect(reporter.issues[0].msg).to.equal('Message contains invalid non-breaking space at position 11.'); - - it('generates an nbsp error with non-breaking space in the messageformat structure', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select,\u00A0other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(2); - expect(reporter.issues[0].type).to.equal('nbsp'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('String contains invalid non-breaking space at position 11.'); + }); expect(reporter.issues[1].type).to.equal('case'); expect(reporter.issues[1].level).to.equal('error'); expect(reporter.issues[1].msg).to.equal('Unrecognized cases ["\u00A0other"]'); }); - it('generates a nest-order error with mismatched complex argument order', () => { + it('generates a "nest-order" error with mismatched complex argument order', () => { const sourceMessage = '{a, select, other {{b, select, other {}}}}'; const targetMessage = '{b, select, other {{a, select, other {}}}}'; reporter.config(targetMessage, sourceMessage, 'key'); @@ -252,7 +254,7 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Nesting order does not match source.'); }); - it('generates a nest-ideal error with plural inside select', () => { + it('generates a "nest-ideal" error with plural inside select', () => { const sourceMessage = '{a, plural, one {} other {{b, select, other {}}}}'; const targetMessage = '{a, plural, one {} other {{b, select, other {}}}}'; reporter.config(targetMessage, sourceMessage, 'key'); @@ -274,15 +276,16 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('"plural" and "selectordinal" should always nest inside "select".'); }); - it('generates an other error with missing other case', () => { + it('generates an "other" error with missing other case', () => { const sourceMessage = '{a, select, b {}}'; const targetMessage = '{a, select, b {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('other'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Missing "other" case'); + expect(reporter.issues[0].msg).to.equal('Missing "other" option'); }); it('generates an other error with missing other case', () => { @@ -296,7 +299,7 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Missing "other" case'); }); - it('generates a parse error with an unparseable target message', () => { + it('generates a "parse" error with an unparseable target message', () => { const sourceMessage = '{a, select, b {}}'; const targetMessage = '{a, select b {}}'; reporter.config(targetMessage, sourceMessage, 'key'); @@ -304,7 +307,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('parse'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found.'); + expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found'); }); it('generates a parse error with an unparseable target message', () => { @@ -318,9 +321,9 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found.'); }); - it('generates a source-error error an unparseable source message', () => { - const sourceMessage = '{a, select b {}}'; - const targetMessage = '{a, select, b {}}'; + it('generates a "source-error" error an unparseable source message', () => { + const sourceMessage = '{a, select other {}}'; + const targetMessage = '{a, select, other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); @@ -344,7 +347,7 @@ describe('validate', () => { describe('validateLocales', () => { - it('generates an extraneous error with unexpected message in target locale', () => { + it('generates an "extraneous" error with unexpected message in target locale', () => { const sourceMessage = '{a, select, other {}}'; const targetMessage = '{a, select, other {}}'; const locales = parseLocales([{ @@ -375,7 +378,7 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('This string does not exist in the source file.'); }); - it('generates a missing error with missing message in the target locale', () => { + it('generates a "missing" error with missing message in the target locale', () => { const sourceMessage = '{a, select, other {}}'; const targetMessage = '{a, select, other {}}'; const locales = parseLocales([{ @@ -406,7 +409,7 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('String missing from locale file.'); }); - it('generates a duplicate-keys error with duplicate messages in the target locale', () => { + it('generates a "duplicate-keys" error with duplicate messages in the target locale', () => { const sourceMessage = '{a, select, other {}}'; const targetMessage = '{a, select, other {}}'; const locales = parseLocales([{ From 26cc12dbff78b7aeb037acb3d238397279d81019 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 18 Sep 2024 23:01:16 -0400 Subject: [PATCH 12/90] Use Intl.PluralRules in validate --- package-lock.json | 8 +------- package.json | 3 +-- src/validate.js | 7 +++---- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6dcf13c..f977982 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,8 +15,7 @@ "commander": "^6.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", - "glob": "^7.1.6", - "make-plural": "^7.4.0" + "glob": "^7.1.6" }, "bin": { "mfv": "bin/cli.js" @@ -1992,11 +1991,6 @@ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.0.tgz", "integrity": "sha512-dVmQmXPBlTgFw77hm60ud//l2bCuDKkqC2on1EBoM7s9Urm9IQDrnujwZ93NFnAq0dVZ0HBXTS7PwEG+YE7+EQ==" }, - "node_modules/make-plural": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/make-plural/-/make-plural-7.4.0.tgz", - "integrity": "sha512-4/gC9KVNTV6pvYg2gFeQYTW3mWaoJt7WZE5vrp1KnQDgW92JtYZnzmZT81oj/dUTqAIu0ufI2x3dkgu3bB1tYg==" - }, "node_modules/memoizeasync": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/memoizeasync/-/memoizeasync-1.1.0.tgz", diff --git a/package.json b/package.json index 1bbd119..f9e1c08 100644 --- a/package.json +++ b/package.json @@ -33,8 +33,7 @@ "commander": "^6.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", - "glob": "^7.1.6", - "make-plural": "^7.4.0" + "glob": "^7.1.6" }, "devDependencies": { "@babel/eslint-parser": "^7.25.1", diff --git a/src/validate.js b/src/validate.js index 12c4aa8..5cc5d75 100644 --- a/src/validate.js +++ b/src/validate.js @@ -1,4 +1,3 @@ -import * as pluralCats from 'make-plural/pluralCategories' import { Reporter } from './reporter.js'; import { parse } from '@formatjs/icu-messageformat-parser'; @@ -7,8 +6,8 @@ const SELECTORDINAL = 6; const PLURAL = 6; const ARGUMENT = 1; -function getPluralCats(locale) { - return pluralCats[locale.split('-')[0]] || pluralCats.en; +function getPluralCats(locale, pluralType) { + return new Intl.PluralRules(locale, { type: pluralType }).resolvedOptions().pluralCategories; } export const structureRegEx = /(?<=\s*){(.|\n)*?[{}]|\s*}(.|\n)*?[{}]|[{#]|(\s*)}/g; @@ -136,7 +135,7 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so } if (part.type === PLURAL) { - const supportedCats = getPluralCats(targetLocale)[part.pluralType]; + const supportedCats = getPluralCats(targetLocale, part.pluralType); const cats = Object.keys(part.options); const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); if (missingCats.length) msgReporter.warning('categories', `Missing categories: ${JSON.stringify(missingCats)}`); From 3759129d30000fdee49a3d0a78372f13708cdb63 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 18 Sep 2024 23:29:36 -0400 Subject: [PATCH 13/90] Add utils --- src/format.js | 11 ++++------- src/validate.js | 9 ++------- test/validate.test.js | 2 +- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/format.js b/src/format.js index 950144a..ec21c0c 100644 --- a/src/format.js +++ b/src/format.js @@ -1,10 +1,8 @@ import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.js'; import { parse } from '@formatjs/icu-messageformat-parser'; -import { structureRegEx } from './validate.js'; +import { getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from './utils.js'; import cldr from 'cldr'; -const paddedQuoteLocales = ['fr', 'fr-ca', 'fr-fr', 'fr-on', 'vi-vn']; - function expandASTHashes(ast, parentValue) { if (Array.isArray(ast)) { ast.map(ast => expandASTHashes(ast, parentValue)); @@ -188,9 +186,8 @@ function printAST(ast, options, level = 0) { text += `{${normalizeArgName(ast.value, args)}, select,${optionsText}}`; } else if (type === 6) { // plural, selectordinal - const pluralCats = ['zero', 'one', 'two', 'few', 'many', 'other']; - const supportedCats = new Intl.PluralRules(locale, { type: ast.pluralType }).resolvedOptions().pluralCategories; - const unsupportedCats = [ ...Object.keys(ast.options).filter(o => !/^=\d+$/.test(o)) , ...pluralCats].filter(cat => !supportedCats.includes(cat)); + const supportedCats = getPluralCats(locale, ast.pluralType); + const unsupportedCats = [ ...Object.keys(ast.options).filter(o => !/^=\d+$/.test(o)) , ...sortedCats].filter(cat => !supportedCats.includes(cat)); if (add) { supportedCats.forEach(cat => { if (!/^(fr|pt)/.test(locale) && cat === 'one' && ast.options['=1']) return; // don't create orphaned `one` @@ -236,7 +233,7 @@ function printAST(ast, options, level = 0) { if (a[0].startsWith('=') || b[0].startsWith('=')) { return a[0].localeCompare(b[0]); } - return pluralCats.indexOf(a[0]) > pluralCats.indexOf(b[0]) ? 1 : -1; + return sortedCats.indexOf(a[0]) > sortedCats.indexOf(b[0]) ? 1 : -1; }).map(([opt, { value }]) => { return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, swapOne }, level + 1)}}`; }).join('') + (useNewlines ? newline : ''); diff --git a/src/validate.js b/src/validate.js index 5cc5d75..9223327 100644 --- a/src/validate.js +++ b/src/validate.js @@ -1,16 +1,11 @@ +import { formatList, getPluralCats, sortedCats, structureRegEx } from './utils.js'; import { Reporter } from './reporter.js'; import { parse } from '@formatjs/icu-messageformat-parser'; const SELECT = 5; -const SELECTORDINAL = 6; const PLURAL = 6; const ARGUMENT = 1; -function getPluralCats(locale, pluralType) { - return new Intl.PluralRules(locale, { type: pluralType }).resolvedOptions().pluralCategories; -} - -export const structureRegEx = /(?<=\s*){(.|\n)*?[{}]|\s*}(.|\n)*?[{}]|[{#]|(\s*)}/g; let reporter; export function validateLocales({ locales, sourceLocale }, localesReporter) { @@ -138,7 +133,7 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const supportedCats = getPluralCats(targetLocale, part.pluralType); const cats = Object.keys(part.options); const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); - if (missingCats.length) msgReporter.warning('categories', `Missing categories: ${JSON.stringify(missingCats)}`); + if (missingCats.length) msgReporter.warning('categories', `Missing categories ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); const unsupportedCats = cats.filter(c => !supportedCats.includes(c)); unsupportedCats.forEach(cat => { diff --git a/test/validate.test.js b/test/validate.test.js index 95721b0..4d549f7 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -75,7 +75,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('categories'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Missing categories: ["zero","two","few","many"]'); + expect(reporter.issues[0].msg).to.equal('Missing categories "zero", "two", "few", and "many"'); }); it('generates "categories" errors when a target message uses unsupported plural categories', () => { From 96f0b8a411a7a6ac59e2b9a1d47a02c34d09b564 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 19 Sep 2024 00:58:00 -0400 Subject: [PATCH 14/90] Add nested category tests --- bin/cli.js | 3 ++- src/format.js | 2 +- src/utils.js | 11 ++++++++++ src/validate.js | 2 +- test/validate.test.js | 48 ++++++++++++++++++++++++------------------- 5 files changed, 42 insertions(+), 24 deletions(-) create mode 100644 src/utils.js diff --git a/bin/cli.js b/bin/cli.js index 2748168..7304a80 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -2,7 +2,7 @@ /* eslint-disable no-console */ -import { parseLocales, structureRegEx, validateLocales } from '../src/validate.js'; +import { parseLocales, validateLocales } from '../src/validate.js'; import { readFile, readdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; import findConfig from 'find-config'; @@ -10,6 +10,7 @@ import { formatMessage } from '../src/format.js' import glob from 'glob'; import pkg from '../package.json' with { type: 'json' }; import { program } from 'commander'; +import { structureRegEx } from '../src/utils.js'; const configPath = findConfig('mfv.config.json'); const { path, source: globalSource, locales: globalLocales, jsonObj: globalJsonObj } = configPath ? (await import(`file://${configPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; diff --git a/src/format.js b/src/format.js index ec21c0c..adb9a68 100644 --- a/src/format.js +++ b/src/format.js @@ -45,7 +45,7 @@ export function formatMessage(msg, options = {}) { expandASTHashes(ast); } try { - ast = hoistSelectors(ast); + ast = hoistSelectors(ast); } catch(e) { console.log(e); } diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..21d3ba7 --- /dev/null +++ b/src/utils.js @@ -0,0 +1,11 @@ +export const sortedCats = ['zero', 'one', 'two', 'few', 'many', 'other']; +export const paddedQuoteLocales = ['fr', 'fr-ca', 'fr-fr', 'fr-on', 'vi-vn']; +export const structureRegEx = /(?<=\s*){(.|\n)*?[{}]|\s*}(.|\n)*?[{}]|[{#]|(\s*)}/g; + +export function getPluralCats(locale, pluralType) { + return new Intl.PluralRules(locale, { type: pluralType }).resolvedOptions().pluralCategories; +} + +export function formatList(arr, locale = 'en') { + return new Intl.ListFormat(locale).format(arr); +} diff --git a/src/validate.js b/src/validate.js index 9223327..5cc3d0b 100644 --- a/src/validate.js +++ b/src/validate.js @@ -134,7 +134,7 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const cats = Object.keys(part.options); const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); if (missingCats.length) msgReporter.warning('categories', `Missing categories ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); - const unsupportedCats = cats.filter(c => !supportedCats.includes(c)); + const unsupportedCats = cats.filter(c => !/^=\d+$/.test(c) && !supportedCats.includes(c)); unsupportedCats.forEach(cat => { const column = part.options[cat].location.start.offset; diff --git a/test/validate.test.js b/test/validate.test.js index 4d549f7..e65c014 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -80,7 +80,7 @@ describe('validate', () => { it('generates "categories" errors when a target message uses unsupported plural categories', () => { const sourceMessage = '{a, plural, one {} other {}}'; - const targetMessage = '{a, plural, one {} two {} few {} many {} other {}}'; + const targetMessage = '{a, plural, =1 {} one {} two {} few {} many {} other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); @@ -99,16 +99,19 @@ describe('validate', () => { expect(reporter.issues[2].msg).to.equal('Unsupported category "many". Must be one of: "one", "other", or explicit keys like "=0"'); }); - it('generates a plural-key error when a target message uses unsupported plural categories', () => { - const sourceString = '{a, plural, one {} other {}}'; - const targetString = '{a, plural, one {} two {} few {} many {} other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('plural-key'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Invalid key `two` for argument `a`. Valid plural keys for this locale are `one`, `other`, and explicit keys like `=0`.'); - }); + it('generates "categories" errors when a target message uses nested unsupported plural categories', () => { + const sourceMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, one {{a, plural, one {} two {} other {}}} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Must be one of: "one", "other", or explicit keys like "=0"'); + }); + + // split it('generates a "split" error when a source message is split by a complex argument', () => { targetLocale = 'en'; @@ -288,16 +291,19 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Missing "other" option'); }); - it('generates an other error with missing other case', () => { - const sourceString = '{a, select, b {}}'; - const targetString = '{a, select, b {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('other'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Missing "other" case'); - }); + it('generates an "other" error with missing nested other case', () => { + const sourceMessage = '{a, select, other {{c, select, b {}}}}'; + const targetMessage = '{a, select, other {{c, select, b {}}}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('other'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Missing "other" option'); + }); + + // parse it('generates a "parse" error with an unparseable target message', () => { const sourceMessage = '{a, select, b {}}'; From e1e68f0247258bad11887d17593b4800683869f4 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 19 Sep 2024 01:34:14 -0400 Subject: [PATCH 15/90] Use tabs for indentation --- .eslintrc.json | 3 + bin/cli.js | 684 +++++++++++++++++++++--------------------- src/format.js | 322 ++++++++++---------- src/reporter.js | 146 ++++----- src/utils.js | 4 +- src/validate.js | 485 +++++++++++++++--------------- test.js | 12 +- test/format.test.js | 254 ++++++++-------- test/validate.test.js | 613 ++++++++++++++++--------------------- 9 files changed, 1208 insertions(+), 1315 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index a13dc81..26d2cbd 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -13,5 +13,8 @@ "@babel/plugin-syntax-import-attributes" ] } + }, + "rules": { + "indent": ["error", "tab"] } } diff --git a/bin/cli.js b/bin/cli.js index 7304a80..151e7dd 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -16,382 +16,382 @@ const configPath = findConfig('mfv.config.json'); const { path, source: globalSource, locales: globalLocales, jsonObj: globalJsonObj } = configPath ? (await import(`file://${configPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; program - .version(pkg.version) - .option('--no-issues', 'Don\'t output issues') - .option('-i, --ignoreIssueTypes ', 'Ignore these comma-separated issue types') - .option('-l, --locales ', 'Process only these comma-separated locales') - .option('-p, --path ', 'Path to a directory containing locale files') - .option('-s, --source-locale ', 'The locale to use as the source') - .option('--json-obj', 'Indicate that the files to be parsed are JSON files with keys that have objects for values') - .command('validate', { isDefault: true, hidden: true }) - .action(() => { - program.validate = true; - }); + .version(pkg.version) + .option('--no-issues', 'Don\'t output issues') + .option('-i, --ignoreIssueTypes ', 'Ignore these comma-separated issue types') + .option('-l, --locales ', 'Process only these comma-separated locales') + .option('-p, --path ', 'Path to a directory containing locale files') + .option('-s, --source-locale ', 'The locale to use as the source') + .option('--json-obj', 'Indicate that the files to be parsed are JSON files with keys that have objects for values') + .command('validate', { isDefault: true, hidden: true }) + .action(() => { + program.validate = true; + }); program - .command('print-missing') - .description('Output JSON of all source messages that are missing or untranslated in the target') - .action(() => { - program.printMissing = true; - }); + .command('print-missing') + .description('Output JSON of all source messages that are missing or untranslated in the target') + .action(() => { + program.printMissing = true; + }); program - .command('remove-extraneous') - .description('Remove messages that do not exist in the source locale') - .action(() => { - program.removeExtraneous = true; - }); + .command('remove-extraneous') + .description('Remove messages that do not exist in the source locale') + .action(() => { + program.removeExtraneous = true; + }); program - .command('add-missing') - .description('Add messages that do not exist in the target locale') - .action(() => { - program.addMissing = true; - }); + .command('add-missing') + .description('Add messages that do not exist in the target locale') + .action(() => { + program.addMissing = true; + }); program - .command('sort') - .description('Sort messages alphabetically by key, maintaining any blocks') - .action(() => { - program.sort = true; - }); + .command('sort') + .description('Sort messages alphabetically by key, maintaining any blocks') + .action(() => { + program.sort = true; + }); program - .command('rename ') - .description('Rename a message') - .action((oldKey, newKey) => { - program.rename = true; - program.oldKey = oldKey; - program.newKey = newKey; - }); + .command('rename ') + .description('Rename a message') + .action((oldKey, newKey) => { + program.rename = true; + program.oldKey = oldKey; + program.newKey = newKey; + }); program - .command('format') - .description('Rewrite messages to a standard format') - .option('-n, --newlines', 'When formatting complex arguments, use newlines and indentation for readability') - .option('-a, --add', 'Add cases for missing supported pural and selectordinal categories') - .option('-r, --remove', 'Remove cases for unsupported pural and selectordinal categories') - .option('-d, --dedupe', 'Remove complex argument cases that duplicate the `other` case. Takes precedence over --add.') - .option('-t, --trim', 'Trim whitespace from both ends of messages') - .option('-c, --collapse', 'Collapse repeating whitepace') - .action(function() { - program.format = true; - const opts = this.opts(); - program.newlines = opts.newlines; - program.add = opts.add; - program.remove = opts.remove; - program.trim = opts.trim; - program.collapse = opts.collapse; - program.dedupe = opts.dedupe - }); + .command('format') + .description('Rewrite messages to a standard format') + .option('-n, --newlines', 'When formatting complex arguments, use newlines and indentation for readability') + .option('-a, --add', 'Add cases for missing supported pural and selectordinal categories') + .option('-r, --remove', 'Remove cases for unsupported pural and selectordinal categories') + .option('-d, --dedupe', 'Remove complex argument cases that duplicate the `other` case. Takes precedence over --add.') + .option('-t, --trim', 'Trim whitespace from both ends of messages') + .option('-c, --collapse', 'Collapse repeating whitepace') + .action(function() { + program.format = true; + const opts = this.opts(); + program.newlines = opts.newlines; + program.add = opts.add; + program.remove = opts.remove; + program.trim = opts.trim; + program.collapse = opts.collapse; + program.dedupe = opts.dedupe + }); program - .command('highlight ') - .description('Output a message with all non-translatable ICU MessageFormat structure highlighted') - .action(key => { - program.highlight = key; - }); + .command('highlight ') + .description('Output a message with all non-translatable ICU MessageFormat structure highlighted') + .action(key => { + program.highlight = key; + }); program.parse(process.argv); const pathCombined = program.path || path; if (!pathCombined) { - console.error('Must provide a path to the locale files using either the -p option or a config file.'); - process.exit(1); + console.error('Must provide a path to the locale files using either the -p option or a config file.'); + process.exit(1); } const noSource = () => { - console.error('Must provide a source locale using either the -s option or a config file.'); - process.exit(1); + console.error('Must provide a source locale using either the -s option or a config file.'); + process.exit(1); }; const localesPaths = glob.sync(pathCombined); localesPaths.forEach(async localesPath => { - const absLocalesPath = `${process.cwd()}/${localesPath}`; + const absLocalesPath = `${process.cwd()}/${localesPath}`; - const subConfigPath = findConfig('mfv.config.json', { cwd: absLocalesPath }); + const subConfigPath = findConfig('mfv.config.json', { cwd: absLocalesPath }); - const { source, locales: configLocales, jsonObj } = subConfigPath ? (await import(`file://${subConfigPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; /* eslint-disable-line global-require */ + const { source, locales: configLocales, jsonObj } = subConfigPath ? (await import(`file://${subConfigPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; /* eslint-disable-line global-require */ - const files = await readdir(absLocalesPath).catch(err => { - console.log(`Failed to read ${absLocalesPath}\n`); - throw err; - }); + const files = await readdir(absLocalesPath).catch(err => { + console.log(`Failed to read ${absLocalesPath}\n`); + throw err; + }); - const sourceLocale = program.sourceLocale || source || globalSource; - const allowedLocalesString = program.locales || configLocales || globalLocales; - const allowedLocales = allowedLocalesString && allowedLocalesString.split(',').concat(sourceLocale); - const filteredFiles = !allowedLocales ? - files.filter(file => !(/^\..*/g).test(file)) : - files.filter(file => allowedLocales.includes(file.split('.')[0])); - const targetLocales = filteredFiles.map(file => file.split('.')[0]); - - if (program.removeExtraneous) { - if (!sourceLocale) noSource(); - console.log('Removing extraneous messages from:', targetLocales.join(', ')); - } - - if (program.addMissing) { - if (!sourceLocale) noSource(); - console.log('Adding missing messages to:', targetLocales.join(', ')); - } - - if (program.rename) { - console.log(`Renaming "${program.oldKey}" to "${program.newKey}" in:`, targetLocales.join(', ')); - } - - if (program.format) { - if (!sourceLocale) noSource(); - console.log(`Formatting:`, targetLocales.join(', ')); - } - - const resources = await Promise.all(filteredFiles.map(file => readFile(absLocalesPath + file, 'utf8'))) - .then(readFiles => readFiles.map((contents, idx) => ({ - file: filteredFiles[idx], - contents - }))) - .catch(err => { - console.error(err); - process.exitCode = 1; - }); - if (!resources) return; + const sourceLocale = program.sourceLocale || source || globalSource; + const allowedLocalesString = program.locales || configLocales || globalLocales; + const allowedLocales = allowedLocalesString && allowedLocalesString.split(',').concat(sourceLocale); + const filteredFiles = !allowedLocales ? + files.filter(file => !(/^\..*/g).test(file)) : + files.filter(file => allowedLocales.includes(file.split('.')[0])); + const targetLocales = filteredFiles.map(file => file.split('.')[0]); + + if (program.removeExtraneous) { + if (!sourceLocale) noSource(); + console.log('Removing extraneous messages from:', targetLocales.join(', ')); + } + + if (program.addMissing) { + if (!sourceLocale) noSource(); + console.log('Adding missing messages to:', targetLocales.join(', ')); + } + + if (program.rename) { + console.log(`Renaming "${program.oldKey}" to "${program.newKey}" in:`, targetLocales.join(', ')); + } + + if (program.format) { + if (!sourceLocale) noSource(); + console.log(`Formatting:`, targetLocales.join(', ')); + } + + const resources = await Promise.all(filteredFiles.map(file => readFile(absLocalesPath + file, 'utf8'))) + .then(readFiles => readFiles.map((contents, idx) => ({ + file: filteredFiles[idx], + contents + }))) + .catch(err => { + console.error(err); + process.exitCode = 1; + }); + if (!resources) return; - const useJSONObj = program.jsonObj || jsonObj || globalJsonObj; - - const locales = parseLocales(resources, useJSONObj); - - if (program.highlight) { - - const showWS = str => str - .replace(/ /g, '·') - .replace(/\t/g, '··') - .replace(/\n/g, '␤\n'); - - Object.keys(locales).forEach(locale => { - if ((!allowedLocales || allowedLocales.includes(locale)) && locales[locale].parsed[program.highlight]) { - const str = String(locales[locale].parsed[program.highlight].val); - - let match; - let prevEnd = 0; - const sections = []; - - while((match = structureRegEx.exec(str)) !== null) { - sections.push(showWS(str.substring(prevEnd, match.index))); - sections.push(chalk.red(showWS(str.substr(match.index, match[0].length)))); - prevEnd = match.index + match[0].length; - } + const useJSONObj = program.jsonObj || jsonObj || globalJsonObj; + + const locales = parseLocales(resources, useJSONObj); + + if (program.highlight) { + + const showWS = str => str + .replace(/ /g, '·') + .replace(/\t/g, '··') + .replace(/\n/g, '␤\n'); + + Object.keys(locales).forEach(locale => { + if ((!allowedLocales || allowedLocales.includes(locale)) && locales[locale].parsed[program.highlight]) { + const str = String(locales[locale].parsed[program.highlight].val); + + let match; + let prevEnd = 0; + const sections = []; + + while((match = structureRegEx.exec(str)) !== null) { + sections.push(showWS(str.substring(prevEnd, match.index))); + sections.push(chalk.red(showWS(str.substr(match.index, match[0].length)))); + prevEnd = match.index + match[0].length; + } - sections.push(showWS(str.substring(prevEnd))); + sections.push(showWS(str.substring(prevEnd))); - const highlighted = sections.join(''); - console.log(highlighted); + const highlighted = sections.join(''); + console.log(highlighted); - } - }); + } + }); - return; - } - - if (program.format) { - let count = 0; - - const sourceLocaleParsed = locales[sourceLocale].parsed; - Object.keys(locales).forEach(async locale => { - if (!allowedLocales || allowedLocales.includes(locale)) { - - let localeContents = locales[locale].contents; - - Object.values(locales[locale].parsed).forEach(t => { - - const source = sourceLocaleParsed[t.key]; - - if (localeContents.includes(t)) { - const baseTabs = t.match('^\n?(?\t*)').groups.tabs - const newVal = formatMessage(t.val, { - locale, - add: program.add, - remove: program.remove, - newlines: program.newlines, - dedupe: program.dedupe, - trim: program.trim, - collapse: program.collapse, - - baseTabs: baseTabs.length, - key: t.key, - source, - target: t - }); - const valQuote = program.newlines && newVal.includes('\n') ? '`' : t.valQuote; - const valSpace = program.newlines && newVal.includes('\n') ? `\n${baseTabs}` : t.valSpace; - const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}${t.valQuote}${t.comma}${t.comment}`; - const noo = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${valSpace}${valQuote}${newVal}${valQuote}${t.comma}${t.comment}`; - - if (old !== noo) count += 1; - localeContents = localeContents.replace(old, noo); - } - }); - - await writeFile(absLocalesPath + locales[locale].file, localeContents); - } - }); - - const cliReport = `\n ${chalk.green('\u2714')} Formatted ${count} messages`; - console.log(cliReport); - - return; - } - - if (program.rename) { - let count = 0; - await Promise.all(Object.keys(locales).map(async locale => { - if (!allowedLocales || allowedLocales.includes(locale)) { - - const localeContents = locales[locale].contents; - const t = locales[locale].parsed[program.oldKey]; - - if (localeContents.includes(t)) { - const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; - const noo = `${t.keyQuote}${program.newKey}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; - - count += 1; - const newLocaleContents = localeContents.replace(old, noo); - - await writeFile(absLocalesPath + locales[locale].file, newLocaleContents); - console.log(`${chalk.green('\u2714')} ${locales[locale].file} - Renamed`); - } - else { - console.log(`${chalk.red('\u2716')} ${locales[locale].file} - Missing`); - } - } - })); - - const cliReport = `\n ${chalk.green('\u2714')} Renamed ${count} messages`; - console.log(cliReport); - - return; - } - - if (!sourceLocale) noSource(); - - const output = validateLocales({ locales, sourceLocale }); - const translatorOutput = {}; - - await Promise.all(output.map(async(locale, idx) => { - const localePath = `${absLocalesPath}${locales[locale.locale].file}`; - - if (!allowedLocales || allowedLocales.includes(locale.locale)) { - console.log((idx > 0 ? '\n' : '') + chalk.underline(localePath)); - if (program.issues) { - - locale.report.totals.ignored = 0; - - if (program.sort) { - const sorted = Object.values(locales[locale.locale].parsed) - .reduce((acc, val) => { - const block = !val.startsWith('\n\n') ? acc.pop() || [] : []; - block.push(val.replace('\n\n', '\n')); - acc.push(block); - return acc; - }, []) - .map(block => block.sort().join('')) - .sort() - .join('\n'); - - locales[locale.locale].contents = locales[locale.locale].contents.replace(Object.values(locales[locale.locale].parsed).join(''), sorted); - } - else { - - locale.issues.forEach(issue => { - if (program.removeExtraneous) { - if (issue.type === 'extraneous') { - locales[locale.locale].contents = locales[locale.locale].contents.replace(locales[locale.locale].parsed[issue.key], '') - console.log('Removed:', issue.key); - } - } - else if (program.addMissing) { - if (issue.type === 'missing') { - const keys = Object.keys(locales[sourceLocale].parsed); - const targetKeys = Object.keys(locales[locale.locale].parsed); - const keyIdx = keys.indexOf(issue.key); - const nextKey = keys[keyIdx + 1]; - const previousKey = keys[keyIdx - 1]; - const nextMessage = locales[locale.locale].parsed[nextKey]; - const siblingMessage = nextMessage || locales[locale.locale].parsed[previousKey] || locales[locale.locale].parsed[targetKeys[targetKeys.length - 1]]; - const contents = locales[locale.locale].contents; - const insertAt = contents.indexOf(siblingMessage) + Number(!nextMessage ? String(siblingMessage).length : 0); - const comma = !nextMessage && !siblingMessage.comma ? `,${siblingMessage.comment}` : ''; - const commaOffset = comma ? siblingMessage.comment.length : 0; - const sourceMessage = `${comma}${locales[sourceLocale].parsed[issue.key]}`; - locales[locale.locale].contents = [contents.slice(0, insertAt - commaOffset), sourceMessage, contents.slice(insertAt)].join(''); - console.log('Added:', issue.key); - locales[locale.locale].parsed[issue.key] = locales[sourceLocale].parsed[issue.key]; - } - } - else if (program.printMissing) { - if (['missing', 'untranslated'].includes(issue.type)) { - translatorOutput[issue.key] = issue.source; - } - } - else if (!program.ignoreIssueTypes || !program.ignoreIssueTypes - .replace(' ','') - .split(',') - .includes(issue.type) - ) { - console.log([ - ' ', chalk.grey(`${issue.line}:${issue.column}`), - ' ', chalk[issue.level == 'error' ? 'red' : 'yellow'](issue.level), - ' ', chalk.grey(issue.type), - ' ', chalk.cyan(issue.key), - ' ', chalk.white(issue.msg) - ].join('')); - } - else { - locale.report.totals.ignored += 1; - } - }); - } - - if (program.removeExtraneous || program.addMissing || program.sort) { - writeFile(localePath, locales[locale.locale].contents); - } - } - - if (program.printMissing) { - console.log(JSON.stringify(translatorOutput, null, 2)); - } - else if (program.removeExtraneous) { - const count = locale.report.errors ? locale.report.errors.extraneous || 0 : 0; - const cliReport = `\n ${chalk.green('\u2714')} Removed ${count} extraneous messages`; - console.log(cliReport); - } - else if (program.addMissing) { - const count = locale.report.errors ? locale.report.errors.missing || 0 : 0; - const cliReport = `\n ${chalk.green('\u2714')} Added ${count} missing messages`; - console.log(cliReport); - } - else if (program.sort) { - console.log('\nSorted'); - } - else if (locale.report.totals.errors || locale.report.totals.warnings) { - const color = locale.report.totals.errors ? 'red' : 'yellow'; - const total = locale.report.totals.errors + locale.report.totals.warnings; - const cliReport = chalk[color](`\n\u2716 ${total} issues (${locale.report.totals.errors} errors, ${locale.report.totals.warnings} warnings)${locale.report.totals.ignored ? chalk.grey(` - ${locale.report.totals.ignored} Ignored`) : ''}`); - console.log(cliReport); - return; - } - else { - const cliReport = `\n ${chalk.green('\u2714')} Passed`; - console.log(cliReport); - } - } - - locale.report = undefined; - - })); - - if (output.some(locale => locale.report?.totals.errors)) { - console.error('\nErrors were reported in at least one locale. See details above.'); - return 1; - } + return; + } + + if (program.format) { + let count = 0; + + const sourceLocaleParsed = locales[sourceLocale].parsed; + Object.keys(locales).forEach(async locale => { + if (!allowedLocales || allowedLocales.includes(locale)) { + + let localeContents = locales[locale].contents; + + Object.values(locales[locale].parsed).forEach(t => { + + const source = sourceLocaleParsed[t.key]; + + if (localeContents.includes(t)) { + const baseTabs = t.match('^\n?(?\t*)').groups.tabs + const newVal = formatMessage(t.val, { + locale, + add: program.add, + remove: program.remove, + newlines: program.newlines, + dedupe: program.dedupe, + trim: program.trim, + collapse: program.collapse, + + baseTabs: baseTabs.length, + key: t.key, + source, + target: t + }); + const valQuote = program.newlines && newVal.includes('\n') ? '`' : t.valQuote; + const valSpace = program.newlines && newVal.includes('\n') ? `\n${baseTabs}` : t.valSpace; + const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}${t.valQuote}${t.comma}${t.comment}`; + const noo = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${valSpace}${valQuote}${newVal}${valQuote}${t.comma}${t.comment}`; + + if (old !== noo) count += 1; + localeContents = localeContents.replace(old, noo); + } + }); + + await writeFile(absLocalesPath + locales[locale].file, localeContents); + } + }); + + const cliReport = `\n ${chalk.green('\u2714')} Formatted ${count} messages`; + console.log(cliReport); + + return; + } + + if (program.rename) { + let count = 0; + await Promise.all(Object.keys(locales).map(async locale => { + if (!allowedLocales || allowedLocales.includes(locale)) { + + const localeContents = locales[locale].contents; + const t = locales[locale].parsed[program.oldKey]; + + if (localeContents.includes(t)) { + const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; + const noo = `${t.keyQuote}${program.newKey}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; + + count += 1; + const newLocaleContents = localeContents.replace(old, noo); + + await writeFile(absLocalesPath + locales[locale].file, newLocaleContents); + console.log(`${chalk.green('\u2714')} ${locales[locale].file} - Renamed`); + } + else { + console.log(`${chalk.red('\u2716')} ${locales[locale].file} - Missing`); + } + } + })); + + const cliReport = `\n ${chalk.green('\u2714')} Renamed ${count} messages`; + console.log(cliReport); + + return; + } + + if (!sourceLocale) noSource(); + + const output = validateLocales({ locales, sourceLocale }); + const translatorOutput = {}; + + await Promise.all(output.map(async(locale, idx) => { + const localePath = `${absLocalesPath}${locales[locale.locale].file}`; + + if (!allowedLocales || allowedLocales.includes(locale.locale)) { + console.log((idx > 0 ? '\n' : '') + chalk.underline(localePath)); + if (program.issues) { + + locale.report.totals.ignored = 0; + + if (program.sort) { + const sorted = Object.values(locales[locale.locale].parsed) + .reduce((acc, val) => { + const block = !val.startsWith('\n\n') ? acc.pop() || [] : []; + block.push(val.replace('\n\n', '\n')); + acc.push(block); + return acc; + }, []) + .map(block => block.sort().join('')) + .sort() + .join('\n'); + + locales[locale.locale].contents = locales[locale.locale].contents.replace(Object.values(locales[locale.locale].parsed).join(''), sorted); + } + else { + + locale.issues.forEach(issue => { + if (program.removeExtraneous) { + if (issue.type === 'extraneous') { + locales[locale.locale].contents = locales[locale.locale].contents.replace(locales[locale.locale].parsed[issue.key], '') + console.log('Removed:', issue.key); + } + } + else if (program.addMissing) { + if (issue.type === 'missing') { + const keys = Object.keys(locales[sourceLocale].parsed); + const targetKeys = Object.keys(locales[locale.locale].parsed); + const keyIdx = keys.indexOf(issue.key); + const nextKey = keys[keyIdx + 1]; + const previousKey = keys[keyIdx - 1]; + const nextMessage = locales[locale.locale].parsed[nextKey]; + const siblingMessage = nextMessage || locales[locale.locale].parsed[previousKey] || locales[locale.locale].parsed[targetKeys[targetKeys.length - 1]]; + const contents = locales[locale.locale].contents; + const insertAt = contents.indexOf(siblingMessage) + Number(!nextMessage ? String(siblingMessage).length : 0); + const comma = !nextMessage && !siblingMessage.comma ? `,${siblingMessage.comment}` : ''; + const commaOffset = comma ? siblingMessage.comment.length : 0; + const sourceMessage = `${comma}${locales[sourceLocale].parsed[issue.key]}`; + locales[locale.locale].contents = [contents.slice(0, insertAt - commaOffset), sourceMessage, contents.slice(insertAt)].join(''); + console.log('Added:', issue.key); + locales[locale.locale].parsed[issue.key] = locales[sourceLocale].parsed[issue.key]; + } + } + else if (program.printMissing) { + if (['missing', 'untranslated'].includes(issue.type)) { + translatorOutput[issue.key] = issue.source; + } + } + else if (!program.ignoreIssueTypes || !program.ignoreIssueTypes + .replace(' ','') + .split(',') + .includes(issue.type) + ) { + console.log([ + ' ', chalk.grey(`${issue.line}:${issue.column}`), + ' ', chalk[issue.level == 'error' ? 'red' : 'yellow'](issue.level), + ' ', chalk.grey(issue.type), + ' ', chalk.cyan(issue.key), + ' ', chalk.white(issue.msg) + ].join('')); + } + else { + locale.report.totals.ignored += 1; + } + }); + } + + if (program.removeExtraneous || program.addMissing || program.sort) { + writeFile(localePath, locales[locale.locale].contents); + } + } + + if (program.printMissing) { + console.log(JSON.stringify(translatorOutput, null, 2)); + } + else if (program.removeExtraneous) { + const count = locale.report.errors ? locale.report.errors.extraneous || 0 : 0; + const cliReport = `\n ${chalk.green('\u2714')} Removed ${count} extraneous messages`; + console.log(cliReport); + } + else if (program.addMissing) { + const count = locale.report.errors ? locale.report.errors.missing || 0 : 0; + const cliReport = `\n ${chalk.green('\u2714')} Added ${count} missing messages`; + console.log(cliReport); + } + else if (program.sort) { + console.log('\nSorted'); + } + else if (locale.report.totals.errors || locale.report.totals.warnings) { + const color = locale.report.totals.errors ? 'red' : 'yellow'; + const total = locale.report.totals.errors + locale.report.totals.warnings; + const cliReport = chalk[color](`\n\u2716 ${total} issues (${locale.report.totals.errors} errors, ${locale.report.totals.warnings} warnings)${locale.report.totals.ignored ? chalk.grey(` - ${locale.report.totals.ignored} Ignored`) : ''}`); + console.log(cliReport); + return; + } + else { + const cliReport = `\n ${chalk.green('\u2714')} Passed`; + console.log(cliReport); + } + } + + locale.report = undefined; + + })); + + if (output.some(locale => locale.report?.totals.errors)) { + console.error('\nErrors were reported in at least one locale. See details above.'); + return 1; + } }); diff --git a/src/format.js b/src/format.js index adb9a68..8774b62 100644 --- a/src/format.js +++ b/src/format.js @@ -4,145 +4,145 @@ import { getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from '. import cldr from 'cldr'; function expandASTHashes(ast, parentValue) { - if (Array.isArray(ast)) { - ast.map(ast => expandASTHashes(ast, parentValue)); - } - - if (ast.type === 7) { // # - ast.type = 1; - ast.value = parentValue; - } - else if (ast.type === 6) { // plural, selectordinal - expandASTHashes(Object.values(ast.options).map(o => o.value), ast.value); - } + if (Array.isArray(ast)) { + ast.map(ast => expandASTHashes(ast, parentValue)); + } + + if (ast.type === 7) { // # + ast.type = 1; + ast.value = parentValue; + } + else if (ast.type === 6) { // plural, selectordinal + expandASTHashes(Object.values(ast.options).map(o => o.value), ast.value); + } } export function formatMessage(msg, options = {}) { - let ast; - try { - ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); - } catch(err) { - try { - const alteredMsg = msg.replace('\'{', '’{'); - ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); - msg = alteredMsg; - } catch(err2) { - if (err.location) { - console.log(`\nERROR: ${err.message}`); - console.log(`\tLocale: ${options.locale}`); - console.log(`\tKey: ${options.key}`); - console.log(`\tOriginal message: ${err.originalMessage}`); - console.log('\tAt or near:', msg.slice(err.location.start.offset, Math.max(err.location.end.offset, err.location.start.offset + 4))); - } else { - console.log(err); - console.log(`\tLocale: ${options.locale}`); - console.log(`\tKey: ${options.key}`); - } - return msg; - } - } - if (options.expandHashes) { - expandASTHashes(ast); - } - try { - ast = hoistSelectors(ast); - } catch(e) { - console.log(e); - } + let ast; + try { + ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); + } catch(err) { + try { + const alteredMsg = msg.replace('\'{', '’{'); + ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); + msg = alteredMsg; + } catch(err2) { + if (err.location) { + console.log(`\nERROR: ${err.message}`); + console.log(`\tLocale: ${options.locale}`); + console.log(`\tKey: ${options.key}`); + console.log(`\tOriginal message: ${err.originalMessage}`); + console.log('\tAt or near:', msg.slice(err.location.start.offset, Math.max(err.location.end.offset, err.location.start.offset + 4))); + } else { + console.log(err); + console.log(`\tLocale: ${options.locale}`); + console.log(`\tKey: ${options.key}`); + } + return msg; + } + } + if (options.expandHashes) { + expandASTHashes(ast); + } + try { + ast = hoistSelectors(ast); + } catch(e) { + console.log(e); + } return printAST(ast, { - useNewlines: options.newlines ?? msg.match(structureRegEx)?.join('').includes('\n'), - add: options.add ?? false, - remove: options.remove ?? false, - dedupe: options.dedupe ?? false, - trim: options.trim ?? false, - collapse: options.collapse ?? false, - - locale: options.locale, - args: options.source ? [...new Set(options.source.match(/(?<=[{<])[^,{}<>]+(?=[}>,])/g))] : [] - }, options.baseTabs); + useNewlines: options.newlines ?? msg.match(structureRegEx)?.join('').includes('\n'), + add: options.add ?? false, + remove: options.remove ?? false, + dedupe: options.dedupe ?? false, + trim: options.trim ?? false, + collapse: options.collapse ?? false, + + locale: options.locale, + args: options.source ? [...new Set(options.source.match(/(?<=[{<])[^,{}<>]+(?=[}>,])/g))] : [] + }, options.baseTabs); } function normalizeArgName(argName, availableArgs) { - if (!availableArgs.includes(argName)) { - if (availableArgs.length === 1) { - return availableArgs[0]; - } else { - return availableArgs.find(a => a.toLowerCase() === argName.toLowerCase()) ?? argName; - } - } - return argName; + if (!availableArgs.includes(argName)) { + if (availableArgs.length === 1) { + return availableArgs[0]; + } else { + return availableArgs.find(a => a.toLowerCase() === argName.toLowerCase()) ?? argName; + } + } + return argName; } function printAST(ast, options, level = 0) { - const { - locale, - swapOne = new Set(), - useNewlines = false, - add = false, - remove = false, - dedupe = false, - trim = false, - args = [] - } = options; - - const localeLower = locale.toLowerCase(); + const { + locale, + swapOne = new Set(), + useNewlines = false, + add = false, + remove = false, + dedupe = false, + trim = false, + args = [] + } = options; + + const localeLower = locale.toLowerCase(); if (Array.isArray(ast)) { - const swapOneClone = new Set(swapOne); - ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) - - const delimiters = (() => { - try { - return cldr.extractDelimiters(locale); - } catch(err) { - return cldr.extractDelimiters(locale.split('-')[0]); - } - })(); - - for (const k in delimiters) { - if (paddedQuoteLocales.includes(localeLower)) { - if (k.endsWith('Start')) { - delimiters[k] = delimiters[k].padEnd(2,'\u202f'); - } else if (k.endsWith('End')) { - delimiters[k] = delimiters[k].padStart(2,'\u202f'); - } - } - } - - let - quoteStart = delimiters.quotationStart, - quoteEnd = delimiters.quotationEnd, - singleQuoteStart = delimiters.alternateQuotationStart, - singleQuoteEnd = delimiters.alternateQuotationEnd; - - //if (1) { // todo: fromSource - if (localeLower.endsWith('-gb')) { - quoteStart = delimiters.alternateQuotationStart; - quoteEnd = delimiters.alternateQuotationEnd; - singleQuoteStart = delimiters.quotationStart; - singleQuoteEnd = delimiters.quotationEnd; - } - //} + const swapOneClone = new Set(swapOne); + ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) + + const delimiters = (() => { + try { + return cldr.extractDelimiters(locale); + } catch(err) { + return cldr.extractDelimiters(locale.split('-')[0]); + } + })(); + + for (const k in delimiters) { + if (paddedQuoteLocales.includes(localeLower)) { + if (k.endsWith('Start')) { + delimiters[k] = delimiters[k].padEnd(2,'\u202f'); + } else if (k.endsWith('End')) { + delimiters[k] = delimiters[k].padStart(2,'\u202f'); + } + } + } + + let + quoteStart = delimiters.quotationStart, + quoteEnd = delimiters.quotationEnd, + singleQuoteStart = delimiters.alternateQuotationStart, + singleQuoteEnd = delimiters.alternateQuotationEnd; + + //if (1) { // todo: fromSource + if (localeLower.endsWith('-gb')) { + quoteStart = delimiters.alternateQuotationStart; + quoteEnd = delimiters.alternateQuotationEnd; + singleQuoteStart = delimiters.quotationStart; + singleQuoteEnd = delimiters.quotationEnd; + } + //} return ast - .filter((i, idx) => !trim || i.type !== 0 || (idx !== 0 && idx !== ast.length - 1) || i.value.trim()) // filter out leading and trailing whitespace - .map((ast, idx, arr) => { - let trim = options.trim; - if (trim && ast.type === 0) { - if (arr.length === 1) { - trim = 'trim'; - } else if (!idx) { - trim = 'trimStart'; - } else if (idx === arr.length - 1) { - trim = 'trimEnd'; - } - } - return printAST(ast, { ...options, swapOne: swapOneClone, trim }, level); - }).join('') + .filter((i, idx) => !trim || i.type !== 0 || (idx !== 0 && idx !== ast.length - 1) || i.value.trim()) // filter out leading and trailing whitespace + .map((ast, idx, arr) => { + let trim = options.trim; + if (trim && ast.type === 0) { + if (arr.length === 1) { + trim = 'trim'; + } else if (!idx) { + trim = 'trimStart'; + } else if (idx === arr.length - 1) { + trim = 'trimEnd'; + } + } + return printAST(ast, { ...options, swapOne: swapOneClone, trim }, level); + }).join('') .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") .replace(/(?<=\s)\\?'|^\\?'/g, singleQuoteStart) // opening ' - .replace(/(?<=\S)'(?=\S)/g, '’') // apostrophe + .replace(/(?<=\S)'(?=\S)/g, '’') // apostrophe .replace(/\\?'/g, singleQuoteEnd) // closing ' .replace(/(?<=\s(\u0648)?)\\?"|^\\?"/g, quoteStart) // opening " .replace(/\\?"/g, quoteEnd) // closing " @@ -155,8 +155,8 @@ function printAST(ast, options, level = 0) { const type = ast.type; if (type === 0) { // straight text - const value = swapOne.size ? ast.value.replace(/1/g, `{${[...swapOne].join('|')}}`) : ast.value; - text += value[trim]?.() ?? value; + const value = swapOne.size ? ast.value.replace(/1/g, `{${[...swapOne].join('|')}}`) : ast.value; + text += value[trim]?.() ?? value; } else if (type === 1) { // simple arg text += `{${normalizeArgName(ast.value, args)}}`; @@ -167,8 +167,8 @@ function printAST(ast, options, level = 0) { if (typeof ast.style === 'string') return `, ${ast.style}`; else return `, ::${ast.style.pattern || ast.style.tokens.map(t => t.stem).join(' ')}`; } else { - return ''; - } + return ''; + } })(); const typesText = ['number', 'date', 'time']; @@ -188,44 +188,44 @@ function printAST(ast, options, level = 0) { else if (type === 6) { // plural, selectordinal const supportedCats = getPluralCats(locale, ast.pluralType); const unsupportedCats = [ ...Object.keys(ast.options).filter(o => !/^=\d+$/.test(o)) , ...sortedCats].filter(cat => !supportedCats.includes(cat)); - if (add) { + if (add) { supportedCats.forEach(cat => { - if (!/^(fr|pt)/.test(locale) && cat === 'one' && ast.options['=1']) return; // don't create orphaned `one` - // add missing supported categories + if (!/^(fr|pt)/.test(locale) && cat === 'one' && ast.options['=1']) return; // don't create orphaned `one` + // add missing supported categories ast.options[cat] ??= { ...(ast.options.other || ast.options.many || ast.options.few || Object.values(ast.options).at(-1)) }; }); - } - - if (ast.options.one) { - // `one` and `=1` are the same - if (ast.options['=1'] && JSON.stringify(ast.options['=1']) === JSON.stringify(ast.options['one'])) { - delete ast.options['=1']; - } - } else if (ast.options['=1'] && /(? { - if (k !== 'other' && printAST(v.value, { locale, swapOne, args }) === otherPrinted) { - delete ast.options[k]; - } - }); - } - - remove && unsupportedCats.forEach(cat => { - const currentKeys = Object.keys(ast.options); - if (currentKeys.includes(cat)) { - if (currentKeys.length === 1) { - ast.options.other = Object.assign({}, ast.options[cat]); - } - delete ast.options[cat]; - } - }); + } + + if (ast.options.one) { + // `one` and `=1` are the same + if (ast.options['=1'] && JSON.stringify(ast.options['=1']) === JSON.stringify(ast.options['one'])) { + delete ast.options['=1']; + } + } else if (ast.options['=1'] && /(? { + if (k !== 'other' && printAST(v.value, { locale, swapOne, args }) === otherPrinted) { + delete ast.options[k]; + } + }); + } + + remove && unsupportedCats.forEach(cat => { + const currentKeys = Object.keys(ast.options); + if (currentKeys.includes(cat)) { + if (currentKeys.length === 1) { + ast.options.other = Object.assign({}, ast.options[cat]); + } + delete ast.options[cat]; + } + }); const typeText = ast.pluralType === 'ordinal' ? 'selectordinal' : 'plural'; const offsetText = + ast.offset !== 0 ? ` offset:${ast.offset}` : ''; diff --git a/src/reporter.js b/src/reporter.js index b839997..f6e3b9b 100644 --- a/src/reporter.js +++ b/src/reporter.js @@ -1,100 +1,100 @@ export function Reporter(locale, fileContents = '') { - this._config = {}; - this._config.locale = locale; - this._config.fileContents = fileContents; - this.report = { totals: { errors: 0, warnings: 0 } }; - this.issues = []; + this._config = {}; + this._config.locale = locale; + this._config.fileContents = fileContents; + this.report = { totals: { errors: 0, warnings: 0 } }; + this.issues = []; } Reporter.prototype.config = function(targetMessage, sourceMessage, key) { - this._config.key = key || targetMessage.key; - if (typeof targetMessage !== "undefined") this._config.target = targetMessage; - if (typeof sourceMessage !== "undefined") this._config.source = sourceMessage; + this._config.key = key || targetMessage.key; + if (typeof targetMessage !== "undefined") this._config.target = targetMessage; + if (typeof sourceMessage !== "undefined") this._config.source = sourceMessage; }; Reporter.prototype.log = function(level, type, msg, column = 0, givenLine) { - const levels = level + 's'; - this.report.totals[levels]++; + const levels = level + 's'; + this.report.totals[levels]++; - this.report[levels] = this.report[levels] || {}; - this.report[levels][type] = this.report[levels][type] || 0; - this.report[levels][type]++; + this.report[levels] = this.report[levels] || {}; + this.report[levels][type] = this.report[levels][type] || 0; + this.report[levels][type]++; - const start = Math.max(this._config.fileContents.indexOf(this._config.target), 0) + column - const newlines = this._config.target.split(this._config.target.val)[0].match(/\n/)?.length || 0; - const line = givenLine || this._config.fileContents.substring(0, start).split('\n').length + newlines; + const start = Math.max(this._config.fileContents.indexOf(this._config.target), 0) + column + const newlines = this._config.target.split(this._config.target.val)[0].match(/\n/)?.length || 0; + const line = givenLine || this._config.fileContents.substring(0, start).split('\n').length + newlines; - const issue = { - locale: this._config.locale, - line, - column, - type, - level, - msg, - target: this._config.target?.val ?? this._config.target, - source: this._config.source?.val ?? this._config.source - }; + const issue = { + locale: this._config.locale, + line, + column, + type, + level, + msg, + target: this._config.target?.val ?? this._config.target, + source: this._config.source?.val ?? this._config.source + }; - if (this._config.key) issue.key = this._config.key; + if (this._config.key) issue.key = this._config.key; - this.issues.push(issue); + this.issues.push(issue); - return issue; + return issue; }; Reporter.prototype.warning = function(type, msg, details = {}) { - const relativeColumn = details.column || 0; + const relativeColumn = details.column || 0; - let column, keyPos, linePos, valPos; + let column, keyPos, linePos, valPos; - const cleanTarget = this._config.target - .replace(/\n/g, '\\n') - .replace(/"/g, '\\"'); + const cleanTarget = this._config.target + .replace(/\n/g, '\\n') + .replace(/"/g, '\\"'); - if (['split', 'newline'].includes(type)) { - column = 0; - } - else if (this._config.key) { - keyPos = this._config.fileContents.indexOf(`"${this._config.key}"`); - valPos = this._config.fileContents.indexOf(`"${cleanTarget}"`, keyPos); - linePos = this._config.fileContents.lastIndexOf(String.fromCharCode(10), keyPos); - column = (valPos + 1) - linePos + relativeColumn; + if (['split', 'newline'].includes(type)) { + column = 0; + } + else if (this._config.key) { + keyPos = this._config.fileContents.indexOf(`"${this._config.key}"`); + valPos = this._config.fileContents.indexOf(`"${cleanTarget}"`, keyPos); + linePos = this._config.fileContents.lastIndexOf(String.fromCharCode(10), keyPos); + column = (valPos + 1) - linePos + relativeColumn; - if (valPos === -1) { - // the target message likely contains a backslash that does not escape anything - column = 0; - } - } + if (valPos === -1) { + // the target message likely contains a backslash that does not escape anything + column = 0; + } + } - return this.log('warning', type, msg, column); + return this.log('warning', type, msg, column); }; Reporter.prototype.error = function(type, msg, details = {}) { - const relativeColumn = details.column || 0; - - let column = relativeColumn, - keyPos, line, linePos, valPos; - - if (type === 'missing') { - column = 0; - } - else if (this._config.key) { - // todo: this seems json-specific - const keyQuote = this._config.target.keyQuote; - const valQuote = this._config.target.valQuote; - keyPos = this._config.fileContents.indexOf(`${keyQuote}${this._config.key}${keyQuote}`); - valPos = this._config.fileContents.indexOf(`${valQuote}${this._config.target.val}${valQuote}`, keyPos); - linePos = this._config.fileContents.lastIndexOf(String.fromCharCode(10), keyPos); - column = (valPos + 1) - linePos + relativeColumn; - line = (this._config.fileContents.substring(0, linePos + column).match(/\n/g)?.length ?? -1) + 1; - column -= this._config.target.lastIndexOf('\n', column); - - if (valPos === -1) { - // the target message likely contains a backslash that does not escape anything - column = 0; - } - } - return this.log('error', type, msg, column, line); + const relativeColumn = details.column || 0; + + let column = relativeColumn, + keyPos, line, linePos, valPos; + + if (type === 'missing') { + column = 0; + } + else if (this._config.key) { + // todo: this seems json-specific + const keyQuote = this._config.target.keyQuote; + const valQuote = this._config.target.valQuote; + keyPos = this._config.fileContents.indexOf(`${keyQuote}${this._config.key}${keyQuote}`); + valPos = this._config.fileContents.indexOf(`${valQuote}${this._config.target.val}${valQuote}`, keyPos); + linePos = this._config.fileContents.lastIndexOf(String.fromCharCode(10), keyPos); + column = (valPos + 1) - linePos + relativeColumn; + line = (this._config.fileContents.substring(0, linePos + column).match(/\n/g)?.length ?? -1) + 1; + column -= this._config.target.lastIndexOf('\n', column); + + if (valPos === -1) { + // the target message likely contains a backslash that does not escape anything + column = 0; + } + } + return this.log('error', type, msg, column, line); }; diff --git a/src/utils.js b/src/utils.js index 21d3ba7..f7a723a 100644 --- a/src/utils.js +++ b/src/utils.js @@ -3,9 +3,9 @@ export const paddedQuoteLocales = ['fr', 'fr-ca', 'fr-fr', 'fr-on', 'vi-vn']; export const structureRegEx = /(?<=\s*){(.|\n)*?[{}]|\s*}(.|\n)*?[{}]|[{#]|(\s*)}/g; export function getPluralCats(locale, pluralType) { - return new Intl.PluralRules(locale, { type: pluralType }).resolvedOptions().pluralCategories; + return new Intl.PluralRules(locale, { type: pluralType }).resolvedOptions().pluralCategories; } export function formatList(arr, locale = 'en') { - return new Intl.ListFormat(locale).format(arr); + return new Intl.ListFormat(locale).format(arr); } diff --git a/src/validate.js b/src/validate.js index 5cc3d0b..fb0ccaf 100644 --- a/src/validate.js +++ b/src/validate.js @@ -10,283 +10,284 @@ let reporter; export function validateLocales({ locales, sourceLocale }, localesReporter) { - const sourceMessages = locales[sourceLocale].parsed; + const sourceMessages = locales[sourceLocale].parsed; - return Object.keys(locales).map((targetLocale) => { + return Object.keys(locales).map((targetLocale) => { - reporter = localesReporter ?? new Reporter(targetLocale, locales[targetLocale].contents); - const targetMessages = locales[targetLocale].parsed; - const checkedKeys = []; + reporter = localesReporter ?? new Reporter(targetLocale, locales[targetLocale].contents); + const targetMessages = locales[targetLocale].parsed; + const checkedKeys = []; - Object.keys(targetMessages).forEach(key => { + Object.keys(targetMessages).forEach(key => { - checkedKeys.push(key); - const targetMessage = targetMessages?.[key].val; - const sourceMessage = sourceMessages?.[key]?.val || ''; - const overrides = Array.from(targetMessages?.[key].comment.matchAll(/mfv-(?[a-z]+)/g)).map(m => m.groups.override) + checkedKeys.push(key); + const targetMessage = targetMessages?.[key].val; + const sourceMessage = sourceMessages?.[key]?.val || ''; + const overrides = Array.from(targetMessages?.[key].comment.matchAll(/mfv-(?[a-z]+)/g)).map(m => m.groups.override) - reporter.config(targetMessages[key], sourceMessages[key]); + reporter.config(targetMessages[key], sourceMessages[key]); - if (!sourceMessage) { - reporter.error('extraneous', 'Message does not exist in the source file.'); - } - else { - if (locales[targetLocale].duplicateKeys.has(key)) reporter.error('duplicate-keys', 'Key appears multiple times'); + if (!sourceMessage) { + reporter.error('extraneous', 'Message does not exist in the source file.'); + } + else { + if (locales[targetLocale].duplicateKeys.has(key)) reporter.error('duplicate-keys', 'Key appears multiple times'); - validateMessage({ - targetMessage, - targetLocale, - sourceMessage, - sourceLocale, - overrides - }, reporter); - } - }); + validateMessage({ + targetMessage, + targetLocale, + sourceMessage, + sourceLocale, + overrides + }, reporter); + } + }); - const missingKeys = Object.keys(sourceMessages).filter(arg => !checkedKeys.includes(arg)); + const missingKeys = Object.keys(sourceMessages).filter(arg => !checkedKeys.includes(arg)); - if (missingKeys.length) { - missingKeys.forEach((key) => { - reporter.config(sourceMessages[key], sourceMessages[key]); - reporter.error('missing', `Message missing from locale file.`) - }) - } + if (missingKeys.length) { + missingKeys.forEach((key) => { + reporter.config(sourceMessages[key], sourceMessages[key]); + reporter.error('missing', `Message missing from locale file.`) + }) + } - return { - locale: targetLocale, - issues: reporter.issues || [], - report: reporter.report, - parsed: true - } + return { + locale: targetLocale, + issues: reporter.issues || [], + report: reporter.report, + parsed: true + } - }); + }); } function checkNbsp(message, reporter) { - const structure = message.match(structureRegEx)?.join('') || ''; - const nbspPos = structure.indexOf(String.fromCharCode(160)); + const structure = message.match(structureRegEx)?.join('') || ''; + const nbspPos = structure.indexOf(String.fromCharCode(160)); - if (nbspPos > -1) { - reporter.error('nbsp', `Message contains invalid non-breaking space at position ${nbspPos}.`, { column: nbspPos }); - return true; - } + if (nbspPos > -1) { + reporter.error('nbsp', `Message contains invalid non-breaking space at position ${nbspPos}.`, { column: nbspPos }); + return true; + } } export function validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale, overrides }, msgReporter = reporter) { const re = /[\u2000-\u206F\u2E00-\u2E7F\n\r\\'!"#$%&()*+,\-.\/∕:;<=>?@\[\]^_`{|}~]/g; // eslint-disable-line - if (sourceLocale - && targetLocale.split('-')[0] !== sourceLocale.split('-')[0] - && targetMessage.replace(re,'') === sourceMessage.replace(re,'')) { - - if (!overrides?.includes('translated') - && sourceMessage - .replace(structureRegEx, '') - .replace(re,'') - .replace(/\s/g, '')) { - - msgReporter.warning('untranslated', `Message has not been translated.`); - } - } - - let parsedTarget; - try { - parsedTarget = Object.freeze(parse(targetMessage, { captureLocation: true, requiresOtherClause: false })); - } - catch(e) { - if ((targetMessage.match(/{/g) || 0).length !== (targetMessage.match(/}/g) || 0).length) { - msgReporter.error('brace', 'Mismatched braces', { column: e.location.start.column }); - } - else if (!checkNbsp(targetMessage, msgReporter)) { - if (e.message === 'EXPECT_SELECT_ARGUMENT_OPTIONS') { - msgReporter.error('parse', `Expected "," but "${e.originalMessage.substr(e.location.start.column, 1)}" found`, { column: e.location.start.column - 1 }); - } - else { - msgReporter.error('parse', e.message, { column: e.location.start.column - 1 }); - } - } - } - - if (parsedTarget) { - - const targetTokens = parsedTarget; - let sourceTokens; - - try { - sourceTokens = parse(sourceMessage, { requiresOtherClause: false }); - } - catch(e) { - msgReporter.error('source-error', 'Failed to parse source message.'); - return; - } - - const checkCases = (target, msg) => { - target?.forEach(part => { - if ([SELECT, PLURAL].includes(part.type)) { - if (!part.options.other) { - msgReporter.error('other', 'Missing "other" option'); - } - - if (part.type === PLURAL) { - const supportedCats = getPluralCats(targetLocale, part.pluralType); - const cats = Object.keys(part.options); - const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); - if (missingCats.length) msgReporter.warning('categories', `Missing categories ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); - const unsupportedCats = cats.filter(c => !/^=\d+$/.test(c) && !supportedCats.includes(c)); - - unsupportedCats.forEach(cat => { - const column = part.options[cat].location.start.offset; - msgReporter.error('categories', `Unsupported category "${cat}". Must be one of: "${supportedCats.join('", "')}", or explicit keys like "=0"`, { column }); - }); - } - - Object.values(part.options).forEach(o => { - checkCases(o.value, msg); - }); - } - }); - }; - - checkCases(parsedTarget, targetMessage); - - const targetMap = _map(targetTokens); - const sourceMap = _map(sourceTokens); - - const argDiff = Array.from(targetMap.arguments).filter(arg => !Array.from(sourceMap.arguments).includes(arg)); - - const badArgPos = targetMessage.indexOf(argDiff[0]); - if (argDiff.length) { - msgReporter.error('argument', `Unrecognized arguments: ${argDiff.join(', ')}. Must be one of: ${Array.from(sourceMap.arguments).join(', ')}`, { column: badArgPos }); - } - - checkNbsp(targetMessage, msgReporter); - - if (targetMap.cases.join(',') !== sourceMap.cases.join(',')) { - - const cleanTargetCases = targetMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); - const cleanSourceCases = sourceMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); - const optionDiff = cleanTargetCases.filter(arg => !cleanSourceCases.includes(arg)); - - optionDiff.forEach(o => { - msgReporter.error('option', `Unrecognized option "${o.replace(/.+\|5undefined\|/, '')}". Must be one of "${cleanSourceCases.map(o => o.replace(/.+\|5undefined\|/, '')).join('", "')}".`); - }); - - if (targetMap.nested && targetMap.cases.length === sourceMap.cases.length) { - // TODO: better identify case order vs nesting order - msgReporter.warning('nest-order', `Nesting order does not match source.`); - } - } - - const firstPlural = targetMap.cases.findIndex(i => i.match(/^.+\|(6(ordinal|cardinal))\|/)) + 1; - const lastSelect = targetMap.cases.findLastIndex(i => i.match(/^.+\|5undefined\|/)) + 1; - - if (targetMap.nested && firstPlural && lastSelect && firstPlural < lastSelect) { - msgReporter.warning('nest-ideal', '"plural" and "selectordinal" should always nest inside "select".'); - } - - if (targetTokens.length > 1) { - if (targetLocale == sourceLocale && targetTokens.find((token) => typeof token !== 'string' && [PLURAL, SELECT].includes(token.type))) { - msgReporter.warning('split','Message split by complex argument') - } - } - } + if (sourceLocale + && targetLocale.split('-')[0] !== sourceLocale.split('-')[0] + && targetMessage.replace(re,'') === sourceMessage.replace(re,'')) { + + if (!overrides?.includes('translated') + && sourceMessage + .replace(structureRegEx, '') + .replace(re,'') + .replace(/\s/g, '')) { + + msgReporter.warning('untranslated', `Message has not been translated.`); + } + } + + let parsedTarget; + try { + parsedTarget = Object.freeze(parse(targetMessage, { captureLocation: true, requiresOtherClause: false })); + } + catch(e) { + if ((targetMessage.match(/{/g) || 0).length !== (targetMessage.match(/}/g) || 0).length) { + msgReporter.error('brace', 'Mismatched braces', { column: e.location.start.column }); + } + else if (!checkNbsp(targetMessage, msgReporter)) { + if (e.message === 'EXPECT_SELECT_ARGUMENT_OPTIONS') { + msgReporter.error('parse', `Expected "," but "${e.originalMessage.substr(e.location.start.column, 1)}" found`, { column: e.location.start.column - 1 }); + } + else { + msgReporter.error('parse', e.message, { column: e.location.start.column - 1 }); + } + } + } + + if (parsedTarget) { + + const targetTokens = parsedTarget; + let sourceTokens; + + try { + sourceTokens = parse(sourceMessage, { requiresOtherClause: false }); + } + catch(e) { + msgReporter.error('source-error', 'Failed to parse source message.'); + return; + } + + const checkCases = (target, msg) => { + target?.forEach(part => { + if ([SELECT, PLURAL].includes(part.type)) { + if (!part.options.other) { + msgReporter.error('other', 'Missing "other" option'); + } + + if (part.type === PLURAL) { + const supportedCats = getPluralCats(targetLocale, part.pluralType); + const cats = Object.keys(part.options); + const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); + if (missingCats.length) msgReporter.warning('categories', `Missing categories ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); + const unsupportedCats = cats.filter(c => !/^=\d+$/.test(c) && !supportedCats.includes(c)); + + unsupportedCats.forEach(cat => { + const column = part.options[cat].location.start.offset; + msgReporter.error('categories', `Unsupported category "${cat}". Must be one of: "${supportedCats.join('", "')}", or explicit keys like "=0"`, { column }); + }); + } + + Object.values(part.options).forEach(o => { + checkCases(o.value, msg); + }); + } + }); + }; + + checkCases(parsedTarget, targetMessage); + + const targetMap = _map(targetTokens); + const sourceMap = _map(sourceTokens); + + const argDiff = Array.from(targetMap.arguments).filter(arg => !Array.from(sourceMap.arguments).includes(arg)); + + const badArgPos = targetMessage.indexOf(argDiff[0]); + if (argDiff.length) { + msgReporter.error('argument', `Unrecognized arguments: ${argDiff.join(', ')}. Must be one of: ${Array.from(sourceMap.arguments).join(', ')}`, { column: badArgPos }); + } + + checkNbsp(targetMessage, msgReporter); + + if (targetMap.cases.join(',') !== sourceMap.cases.join(',')) { + + const cleanTargetCases = targetMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); + const cleanSourceCases = sourceMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); + const optionDiff = cleanTargetCases.filter(arg => !cleanSourceCases.includes(arg)); + + optionDiff.forEach(o => { + msgReporter.error('option', `Unrecognized option "${o.replace(/.+\|5undefined\|/, '')}". Must be one of "${cleanSourceCases.map(o => o.replace(/.+\|5undefined\|/, '')).join('", "')}".`); + }); + + if (targetMap.nested && targetMap.cases.length === sourceMap.cases.length) { + // TODO: better identify case order vs nesting order + msgReporter.warning('nest-order', `Nesting order does not match source.`); + } + } + + const firstPlural = targetMap.cases.findIndex(i => i.match(/^.+\|(6(ordinal|cardinal))\|/)) + 1; + const lastSelect = targetMap.cases.findLastIndex(i => i.match(/^.+\|5undefined\|/)) + 1; + + if (targetMap.nested && firstPlural && lastSelect && firstPlural < lastSelect) { + msgReporter.warning('nest-ideal', '"plural" and "selectordinal" should always nest inside "select".'); + } + + if (targetTokens.length > 1) { + if (targetLocale == sourceLocale && targetTokens.find((token) => typeof token !== 'string' && [PLURAL, SELECT].includes(token.type))) { + msgReporter.warning('split','Message split by complex argument') + } + } + } } export function parseLocales(locales, useJSONObj) { - return locales.reduce((acc, { contents, file }) => { - const locale = file.split('.')[0]; - acc[locale] = { - contents, - duplicateKeys: new Set(), - parsed: {}, - file - }; - - const regex = useJSONObj - //[ ][ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] - ? /("(?.*)"(\s*):(\s*)\{)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/g - //[ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] - : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/g; - const matches = Array.from(contents.matchAll(regex)); - - let findContext = false; - let findValue = false; - - matches.forEach(match => { - - if (useJSONObj) { - if (findContext && match.groups.key === 'context') { - acc[locale].parsed[findContext].comment = match.groups.val; - findContext = false; - return; - } - - if (findValue && match.groups.key === 'translation') { - acc[locale].parsed[findValue].val = match.groups.val; - findValue = false; - return; - } - - if (match.groups.realKey) { - - if (match.groups.key === 'translation') { - findContext = match.groups.realKey; - } - if (match.groups.key === 'context') { - match.groups.comment = match.groups.val; - findValue = match.groups.realKey; - } - - match.groups.key = match.groups.realKey; - } - } - - if (!acc[locale].parsed[match.groups.key]) { - acc[locale].parsed[match.groups.key] = Object.assign(String(match[0]), match.groups); - } - else { - acc[locale].duplicateKeys.add(match.groups.key); - } - }); - return acc; - }, {}); + return locales.reduce((acc, { contents, file }) => { + const locale = file.split('.')[0]; + acc[locale] = { + contents, + duplicateKeys: new Set(), + parsed: {}, + file + }; + + const regex = useJSONObj + ? /("(?.*)"(\s*):(\s*)\{)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/g + //[ ][ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] + : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/g; + //[ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] + + const matches = Array.from(contents.matchAll(regex)); + + let findContext = false; + let findValue = false; + + matches.forEach(match => { + + if (useJSONObj) { + if (findContext && match.groups.key === 'context') { + acc[locale].parsed[findContext].comment = match.groups.val; + findContext = false; + return; + } + + if (findValue && match.groups.key === 'translation') { + acc[locale].parsed[findValue].val = match.groups.val; + findValue = false; + return; + } + + if (match.groups.realKey) { + + if (match.groups.key === 'translation') { + findContext = match.groups.realKey; + } + if (match.groups.key === 'context') { + match.groups.comment = match.groups.val; + findValue = match.groups.realKey; + } + + match.groups.key = match.groups.realKey; + } + } + + if (!acc[locale].parsed[match.groups.key]) { + acc[locale].parsed[match.groups.key] = Object.assign(String(match[0]), match.groups); + } + else { + acc[locale].duplicateKeys.add(match.groups.key); + } + }); + return acc; + }, {}); } function _map(ast, partsMap = { nested: false, arguments: new Set(), cases: [], messageTokens: [] }) { - ast.forEach(token => { + ast.forEach(token => { - if (typeof token !== 'string') { + if (typeof token !== 'string') { - if (token.type === ARGUMENT) { - partsMap.arguments.add(token.value); - } + if (token.type === ARGUMENT) { + partsMap.arguments.add(token.value); + } - if (token.options) { + if (token.options) { - if (partsMap.cases.length) { - partsMap.nested = true; - } + if (partsMap.cases.length) { + partsMap.nested = true; + } - Object.entries(token.options).forEach(([k, option]) => { - switch (token.type) { - case SELECT: - case PLURAL: - partsMap.cases.push(`${token.value}|${token.type}${token.pluralType}|${k}`); - break; - } + Object.entries(token.options).forEach(([k, option]) => { + switch (token.type) { + case SELECT: + case PLURAL: + partsMap.cases.push(`${token.value}|${token.type}${token.pluralType}|${k}`); + break; + } - _map(option.value, partsMap); - }); - } - } - else { - partsMap.messageTokens.push(token); - } + _map(option.value, partsMap); + }); + } + } + else { + partsMap.messageTokens.push(token); + } - }); + }); - return partsMap; + return partsMap; } diff --git a/test.js b/test.js index df25e8f..1967d0b 100644 --- a/test.js +++ b/test.js @@ -12,16 +12,16 @@ Object.keys(locales).forEach(key => { */ const locales = { - en: fs.readFileSync('./test/locales/json/bigfiles/en.json', 'utf8'), - fr: fs.readFileSync('./test/locales/json/bigfiles/fr.json', 'utf8'), - ar: fs.readFileSync('./test/locales/json/bigfiles/ar.json', 'utf8'), - es: fs.readFileSync('./test/locales/json/bigfiles/es.json', 'utf8') + String.fromCharCode(65279) + en: fs.readFileSync('./test/locales/json/bigfiles/en.json', 'utf8'), + fr: fs.readFileSync('./test/locales/json/bigfiles/fr.json', 'utf8'), + ar: fs.readFileSync('./test/locales/json/bigfiles/ar.json', 'utf8'), + es: fs.readFileSync('./test/locales/json/bigfiles/es.json', 'utf8') + String.fromCharCode(65279) } const issues = validateLocales({ - locales, - sourceLocale: 'en' + locales, + sourceLocale: 'en' }); console.log(JSON.stringify(issues, null, 2)); // eslint-disable-line no-console diff --git a/test/format.test.js b/test/format.test.js index 1405184..ac93c38 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -3,29 +3,29 @@ import { formatMessage } from '../src/format.js'; describe('formatMessage', () => { - let locale; - beforeEach(() => { - locale = 'en'; - }); - - [ - { locale: 'ar', expected: `This isn’t ”correct“` }, - { locale: 'cy', expected: `This isn’t “correct”` }, - { locale: 'de', expected: `This isn’t „correct“` }, - { locale: 'en', expected: `This isn’t “correct”` }, - { locale: 'en-gb', expected: `This isn’t ‘correct’` }, - { locale: 'fr', expected: `This isn’t «\u202fcorrect\u202f»` }, - { locale: 'sv', expected: `This isn’t ”correct”` }, - ].forEach(({ locale, expected }) => { - it(`should replace straight quotes with "${locale}" quotes with no options`, () => { - const message = `This isn't "correct"`; - const formatted = formatMessage(message, { locale }); - expect(formatted).to.equal(expected); - }); - }); - - [ - { locale: 'ar', expected: + let locale; + beforeEach(() => { + locale = 'en'; + }); + + [ + { locale: 'ar', expected: `This isn’t ”correct“` }, + { locale: 'cy', expected: `This isn’t “correct”` }, + { locale: 'de', expected: `This isn’t „correct“` }, + { locale: 'en', expected: `This isn’t “correct”` }, + { locale: 'en-gb', expected: `This isn’t ‘correct’` }, + { locale: 'fr', expected: `This isn’t «\u202fcorrect\u202f»` }, + { locale: 'sv', expected: `This isn’t ”correct”` }, + ].forEach(({ locale, expected }) => { + it(`should replace straight quotes with "${locale}" quotes with no options`, () => { + const message = `This isn't "correct"`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + }); + + [ + { locale: 'ar', expected: `{a, plural, one {{b, selectordinal, other {} @@ -35,7 +35,7 @@ describe('formatMessage', () => { many {} other {} }` }, - { locale: 'cy', expected: + { locale: 'cy', expected: `{a, plural, one {{b, selectordinal, one {} @@ -49,7 +49,7 @@ describe('formatMessage', () => { many {} other {} }` }, - { locale: 'es', expected: + { locale: 'es', expected: `{a, plural, one {{b, selectordinal, other {} @@ -57,7 +57,7 @@ describe('formatMessage', () => { many {} other {} }` }, - { locale: 'fr', expected: + { locale: 'fr', expected: `{a, plural, one {{b, selectordinal, one {} @@ -66,13 +66,13 @@ describe('formatMessage', () => { many {} other {} }` }, - { locale: 'ja', expected: + { locale: 'ja', expected: `{a, plural, other {} }` }, - ].forEach(({ locale, expected }) => { - it(`should remove plural and selectordinal categories that are unsupported in "${locale}" with the "remove" option`, () => { - const message = + ].forEach(({ locale, expected }) => { + it(`should remove plural and selectordinal categories that are unsupported in "${locale}" with the "remove" option`, () => { + const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}} two {} @@ -80,102 +80,102 @@ describe('formatMessage', () => { many {} other {} }`; - const formatted = formatMessage(message, { locale, remove: true }); - expect(formatted).to.equal(expected); - }); - }); - - it(`should not remove plural and selectordinal categories that are unsupported without the "remove" option`, () => { - const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; - const formatted = formatMessage(message, { locale }); - expect(formatted).to.equal(message); - }); - - it(`should insert newslines and tabs with the "newlines" option`, () => { - const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; - const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; - const formatted = formatMessage(message, { locale, newlines: true }); - expect(formatted).to.equal(expected); - }); - - it(`should insert newslines and tabs if the message structure already contains newlines with no option`, () => { - const message = `{a, plural,\none {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; - const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; - const formatted = formatMessage(message, { locale }); - expect(formatted).to.equal(expected); - }); - - it.skip(`should remove duplicate categories with no options`, () => { - const message = `{a, plural, one {value} other {value2}}`; - const expected = `{a, plural, one {value}}`; - const formatted = formatMessage(message, { locale }); - expect(formatted).to.equal(expected); - }); - - it(`should remove categories that are copies of a lower-precedence key with the "dedupe" option`, () => { - const message = `{a, plural, one {value} other {value}}`; - const expected = `{a, plural, other {value}}`; - const formatted = formatMessage(message, { locale, dedupe: true }); - expect(formatted).to.equal(expected); - }); - - it(`should convert "=1" keys to "one" when it contains a literal "1"`, () => { - const message = `{a, plural, =1 {value 1}}`; - const expected = `{a, plural, one {value {a}}}`; - const formatted = formatMessage(message, { locale, dedupe: true }); - expect(formatted).to.equal(expected); - }); - - it(`should remove "=1" cases when they can be converted to a duplicate case with the "dedupe" option`, () => { - const message = `{a, plural, =1 {value 1} other {value {a}}}`; - const expected = `{a, plural, other {value {a}}}`; - const formatted = formatMessage(message, { locale, dedupe: true }); - expect(formatted).to.equal(expected); - }); - - it(`should remove "=1" cases when they can be converted to unsupported "one" cases with the "remove" option`, () => { - locale = 'ja'; - const message = `{a, plural, =1 {value 1} other {value {a}}}`; - const expected = `{a, plural, other {value {a}}}`; - const formatted = formatMessage(message, { locale, remove: true }); - expect(formatted).to.equal(expected); - }); - - it(`should convert unsupported cases to "other" if there are no other cases`, () => { - locale = 'ja'; - const message = `{a, plural, two {value {a}}}`; - const expected = `{a, plural, other {value {a}}}`; - const formatted = formatMessage(message, { locale, remove: true }); - expect(formatted).to.equal(expected); - }); - - it(`should convert "=1" keys to "other" keys when they can be converted to unsupported "one" cases and there are no other keys with the "remove" option`, () => { - locale = 'ja'; - const message = `{a, plural, =1 {value 1}}}`; - const expected = `{a, plural, other {value {a}}}}`; - const formatted = formatMessage(message, { locale, remove: true }); - expect(formatted).to.equal(expected); - }); - - it('should hoist complex selectors to the outside and nest appropriately with no options', () => { - const message = `\t{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; - const expected = `{a, plural, =1 {{b, plural, =1 {\ta cat and a dog!} other {\ta cat and {b} dogs!}}} other {{b, plural, =1 {\t{a} cats and a dog!} other {\t{a} cats and {b} dogs!}}}}`; - const formatted = formatMessage(message, { locale }); - expect(formatted).to.equal(expected); - }); - - it(`should trim whitespace with the "trim" option`, () => { - const message = `\n{a, plural, other { value }}\t`; - const expected = `{a, plural, other {value}}`; - const formatted = formatMessage(message, { locale, trim: true }); - expect(formatted).to.equal(expected); - }); - - it(`should not trim internal whitespace with the "trim" option`, () => { - const message = `\n{a, plural, other { value {value2} value3 }}`; - const expected = `{a, plural, other {value {value2} value3}}`; - const formatted = formatMessage(message, { locale, trim: true }); - expect(formatted).to.equal(expected); - }); + const formatted = formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + }); + + it(`should not remove plural and selectordinal categories that are unsupported without the "remove" option`, () => { + const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(message); + }); + + it(`should insert newslines and tabs with the "newlines" option`, () => { + const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; + const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; + const formatted = formatMessage(message, { locale, newlines: true }); + expect(formatted).to.equal(expected); + }); + + it(`should insert newslines and tabs if the message structure already contains newlines with no option`, () => { + const message = `{a, plural,\none {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; + const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it.skip(`should remove duplicate categories with no options`, () => { + const message = `{a, plural, one {value} other {value2}}`; + const expected = `{a, plural, one {value}}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it(`should remove categories that are copies of a lower-precedence key with the "dedupe" option`, () => { + const message = `{a, plural, one {value} other {value}}`; + const expected = `{a, plural, other {value}}`; + const formatted = formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert "=1" keys to "one" when it contains a literal "1"`, () => { + const message = `{a, plural, =1 {value 1}}`; + const expected = `{a, plural, one {value {a}}}`; + const formatted = formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should remove "=1" cases when they can be converted to a duplicate case with the "dedupe" option`, () => { + const message = `{a, plural, =1 {value 1} other {value {a}}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should remove "=1" cases when they can be converted to unsupported "one" cases with the "remove" option`, () => { + locale = 'ja'; + const message = `{a, plural, =1 {value 1} other {value {a}}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert unsupported cases to "other" if there are no other cases`, () => { + locale = 'ja'; + const message = `{a, plural, two {value {a}}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + + it(`should convert "=1" keys to "other" keys when they can be converted to unsupported "one" cases and there are no other keys with the "remove" option`, () => { + locale = 'ja'; + const message = `{a, plural, =1 {value 1}}}`; + const expected = `{a, plural, other {value {a}}}}`; + const formatted = formatMessage(message, { locale, remove: true }); + expect(formatted).to.equal(expected); + }); + + it('should hoist complex selectors to the outside and nest appropriately with no options', () => { + const message = `\t{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; + const expected = `{a, plural, =1 {{b, plural, =1 {\ta cat and a dog!} other {\ta cat and {b} dogs!}}} other {{b, plural, =1 {\t{a} cats and a dog!} other {\t{a} cats and {b} dogs!}}}}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it(`should trim whitespace with the "trim" option`, () => { + const message = `\n{a, plural, other { value }}\t`; + const expected = `{a, plural, other {value}}`; + const formatted = formatMessage(message, { locale, trim: true }); + expect(formatted).to.equal(expected); + }); + + it(`should not trim internal whitespace with the "trim" option`, () => { + const message = `\n{a, plural, other { value {value2} value3 }}`; + const expected = `{a, plural, other {value {value2} value3}}`; + const formatted = formatMessage(message, { locale, trim: true }); + expect(formatted).to.equal(expected); + }); }); diff --git a/test/validate.test.js b/test/validate.test.js index e65c014..b453fa8 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -13,440 +13,329 @@ describe('validate', () => { describe('structureRegEx', () => { - [{ - name: 'simple argument', - message: 'abc{def}hij', - structure: '{def}' - }, - { - name: 'multiple simple arguments', - message: 'abc{def}hij {klm} nop {qrs}', - structure: '{def}{klm}{qrs}' - }].forEach(({ name, message, structure }) => { - it(`captures messageformat structure - ${name}`, () => { - expect(message.match(structureRegEx).join('')).to.equal(structure); - }); - }) - }) - describe('validateMessage', () => { - it('generates no issues with identical same-language messages', () => { - const sourceMessage = 'An {arg}'; - const targetMessage = 'An {arg}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(0); - }); - - it('generates an "untranslated" warning when messages are the same and languages are different', () => { - targetLocale = 'es-mx'; - const sourceMessage = 'An {arg}'; - const targetMessage = 'An {arg}'; - reporter.config(targetMessage, sourceMessage, 'key'); - reporter._config.locale = targetLocale; - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('untranslated'); - expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Message has not been translated.'); - }); - - it('generates an untranslated warning when messages are the same and languages are different', () => { + // untranslated + + it('generates no issues with identical same-language messages', () => { + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arg}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(0); + }); + + it('generates an "untranslated" warning when messages are the same and languages are different', () => { targetLocale = 'es-mx'; - const sourceString = 'An {arg}'; - const targetString = 'An {arg}'; - reporter.config(targetString, sourceString, 'key'); + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arg}'; + reporter.config(targetMessage, sourceMessage, 'key'); reporter._config.locale = targetLocale; - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('untranslated'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('String has not been translated.'); + expect(reporter.issues[0].msg).to.equal('Message has not been translated.'); + }); + + // categories + + it('generates a "categories" warning when a target message is missing supported plural categories', () => { + targetLocale = 'cy-gb'; + const sourceMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, one {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + reporter._config.locale = targetLocale; + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].level).to.equal('warning'); + expect(reporter.issues[0].msg).to.equal('Missing categories "zero", "two", "few", and "many"'); + }); + + it('generates "categories" errors when a target message uses unsupported plural categories', () => { + const sourceMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, =1 {} one {} two {} few {} many {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(3); + + expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Must be one of: "one", "other", or explicit keys like "=0"'); + + expect(reporter.issues[1].type).to.equal('categories'); + expect(reporter.issues[1].level).to.equal('error'); + expect(reporter.issues[1].msg).to.equal('Unsupported category "few". Must be one of: "one", "other", or explicit keys like "=0"'); + + expect(reporter.issues[2].type).to.equal('categories'); + expect(reporter.issues[2].level).to.equal('error'); + expect(reporter.issues[2].msg).to.equal('Unsupported category "many". Must be one of: "one", "other", or explicit keys like "=0"'); + }); + + it('generates "categories" errors when a target message uses nested unsupported plural categories', () => { + const sourceMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, one {{a, plural, one {} two {} other {}}} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Must be one of: "one", "other", or explicit keys like "=0"'); }); - it('generates a "categories" warning when a target message is missing supported plural categories', () => { - targetLocale = 'cy-gb'; - const sourceMessage = '{a, plural, one {} other {}}'; - const targetMessage = '{a, plural, one {} other {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - reporter._config.locale = targetLocale; - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('categories'); - expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Missing categories "zero", "two", "few", and "many"'); - }); - - it('generates "categories" errors when a target message uses unsupported plural categories', () => { - const sourceMessage = '{a, plural, one {} other {}}'; - const targetMessage = '{a, plural, =1 {} one {} two {} few {} many {} other {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - - expect(reporter.issues.length).to.equal(3); - - expect(reporter.issues[0].type).to.equal('categories'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Must be one of: "one", "other", or explicit keys like "=0"'); - - expect(reporter.issues[1].type).to.equal('categories'); - expect(reporter.issues[1].level).to.equal('error'); - expect(reporter.issues[1].msg).to.equal('Unsupported category "few". Must be one of: "one", "other", or explicit keys like "=0"'); - - expect(reporter.issues[2].type).to.equal('categories'); - expect(reporter.issues[2].level).to.equal('error'); - expect(reporter.issues[2].msg).to.equal('Unsupported category "many". Must be one of: "one", "other", or explicit keys like "=0"'); - }); - - it('generates "categories" errors when a target message uses nested unsupported plural categories', () => { - const sourceMessage = '{a, plural, one {} other {}}'; - const targetMessage = '{a, plural, one {{a, plural, one {} two {} other {}}} other {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('categories'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Must be one of: "one", "other", or explicit keys like "=0"'); - }); - - // split - - it('generates a "split" error when a source message is split by a complex argument', () => { - targetLocale = 'en'; - const sourceMessage = '{a, plural, one {} other {}} b'; - const targetMessage = '{a, plural, one {} other {}} b'; - reporter.config(targetMessage, sourceMessage, 'key'); - reporter._config.locale = targetLocale; - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('split'); - expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Message split by complex argument'); - }); - - it('generates a plural-key error when a source message is split by a complex arguemnt', () => { + // split + + it('generates a "split" error when a source message is split by a complex argument', () => { targetLocale = 'en'; - const sourceString = '{a, plural, one {} other {}} b'; - const targetString = '{a, plural, one {} other {}} b'; - reporter.config(targetString, sourceString, 'key'); + const sourceMessage = '{a, plural, one {} other {}} b'; + const targetMessage = '{a, plural, one {} other {}} b'; + reporter.config(targetMessage, sourceMessage, 'key'); reporter._config.locale = targetLocale; - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('split'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('String split by non-argument (e.g. select; plural).'); + expect(reporter.issues[0].msg).to.equal('Message split by complex argument'); }); - it('generates an "argument" error with unrecognized argument', () => { - const sourceMessage = 'An {arg}'; - const targetMessage = 'An {arG}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('argument'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized arguments: arG. Must be one of: arg'); - }); - - it('generates an argument error with unrecognized argument', () => { - const sourceString = 'An {arg}'; - const targetString = 'An {arG}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + // arg + + it('generates an "argument" error with unrecognized argument', () => { + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arG}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('argument'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized arguments ["arG"]'); + expect(reporter.issues[0].msg).to.equal('Unrecognized arguments: arG. Must be one of: arg'); + }); + + // brace + + it('does not generate a "brace" error with parseable mismatched braces', () => { + const sourceMessage = 'An {arg}'; + const targetMessage = 'An {arg}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(0); }); - it('does not generate a "brace" error with parseable mismatched braces', () => { - const sourceMessage = 'An {arg}'; - const targetMessage = 'An {arg}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(0); - }); - - it('generates a "brace" error with unparseable mismatched braces', () => { - const sourceMessage = '{a, plural, one {An {arg}} other {{a} args}}'; - const targetMessage = '{a, plural, one {An {arg} other {{a} args}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('brace'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Mismatched braces'); - }); - - it('does not generate a "brace" error with escaped mismatched braces', () => { - const sourceMessage = '{a, plural, one {An {arg}} other {}}'; - const targetMessage = '{a, plural, one {An {arg}\'}\'} other {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(0); - }); - - // option - - it('generates "option" errors with unrecognized cases in select arguments', () => { - const sourceMessage = '{a, select, b {} other {}}'; - const targetMessage = '{a, select, B {} C {} other {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - - expect(reporter.issues.length).to.equal(2); - - expect(reporter.issues[0].type).to.equal('option'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized option "B". Must be one of "b", "other".'); - - expect(reporter.issues[1].type).to.equal('option'); - expect(reporter.issues[1].level).to.equal('error'); - expect(reporter.issues[1].msg).to.equal('Unrecognized option "C". Must be one of "b", "other".'); - }); - - it.skip('generates a "option" error with missing cases in select arguments', () => { - const sourceMessage = '{a, select, b {} other {}}'; - const targetMessage = '{a, select, other {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('option'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Missing cases ["b"]'); - }); - - it.skip('generates a case error with missing cases in select arguments', () => { - const sourceString = '{a, select, b {} other {}}'; - const targetString = '{a, select, other {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it('generates a "brace" error with unparseable mismatched braces', () => { + const sourceMessage = '{a, plural, one {An {arg}} other {{a} args}}'; + const targetMessage = '{a, plural, one {An {arg} other {{a} args}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('case'); + expect(reporter.issues[0].type).to.equal('brace'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Missing cases ["b"]'); + expect(reporter.issues[0].msg).to.equal('Mismatched braces'); + }); + + it('does not generate a "brace" error with escaped mismatched braces', () => { + const sourceMessage = '{a, plural, one {An {arg}} other {}}'; + const targetMessage = '{a, plural, one {An {arg}\'}\'} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(0); }); - it('generates an "nbsp" error with non-breaking space in the messageformat structure', () => { - const sourceMessage = '{a, select, a {} other {}}'; - const targetMessage = '{a, select,\u00A0a {} other {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + // option + + it('generates "option" errors with unrecognized cases in select arguments', () => { + const sourceMessage = '{a, select, b {} other {}}'; + const targetMessage = '{a, select, B {} C {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(2); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('nbsp'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Message contains invalid non-breaking space at position 11.'); - }); + expect(reporter.issues[0].type).to.equal('option'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Unrecognized option "B". Must be one of "b", "other".'); - expect(reporter.issues[1].type).to.equal('case'); + expect(reporter.issues[1].type).to.equal('option'); expect(reporter.issues[1].level).to.equal('error'); - expect(reporter.issues[1].msg).to.equal('Unrecognized cases ["\u00A0other"]'); + expect(reporter.issues[1].msg).to.equal('Unrecognized option "C". Must be one of "b", "other".'); }); - it('generates a "nest-order" error with mismatched complex argument order', () => { - const sourceMessage = '{a, select, other {{b, select, other {}}}}'; - const targetMessage = '{b, select, other {{a, select, other {}}}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('nest-order'); - expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Nesting order does not match source.'); - }); - - it('generates a "nest-ideal" error with plural inside select', () => { - const sourceMessage = '{a, plural, one {} other {{b, select, other {}}}}'; - const targetMessage = '{a, plural, one {} other {{b, select, other {}}}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('nest-ideal'); - expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('"plural" and "selectordinal" should always nest inside "select".'); - }); - - it('generates a nest-ideal error with plural inside select', () => { - const sourceString = '{a, plural, one {} other {{b, select, other {}}}}'; - const targetString = '{a, plural, one {} other {{b, select, other {}}}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + it.skip('generates a "option" error with missing cases in select arguments', () => { + const sourceMessage = '{a, select, b {} other {}}'; + const targetMessage = '{a, select, other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('option'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Missing cases ["b"]'); + }); + + // nbsp + + it('generates an "nbsp" error with non-breaking space in the messageformat structure', () => { + const sourceMessage = '{a, select, a {} other {}}'; + const targetMessage = '{a, select,\u00A0a {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('nbsp'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Message contains invalid non-breaking space at position 11.'); + }); + + // nest + + it('generates a "nest-order" error with mismatched complex argument order', () => { + const sourceMessage = '{a, select, other {{b, select, other {}}}}'; + const targetMessage = '{b, select, other {{a, select, other {}}}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('nest-order'); + expect(reporter.issues[0].level).to.equal('warning'); + expect(reporter.issues[0].msg).to.equal('Nesting order does not match source.'); + }); + + it('generates a "nest-ideal" error with plural inside select', () => { + const sourceMessage = '{a, plural, one {} other {{b, select, other {}}}}'; + const targetMessage = '{a, plural, one {} other {{b, select, other {}}}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('nest-ideal'); expect(reporter.issues[0].level).to.equal('warning'); expect(reporter.issues[0].msg).to.equal('"plural" and "selectordinal" should always nest inside "select".'); }); - it('generates an "other" error with missing other case', () => { - const sourceMessage = '{a, select, b {}}'; - const targetMessage = '{a, select, b {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('other'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Missing "other" option'); - }); - - it('generates an "other" error with missing nested other case', () => { - const sourceMessage = '{a, select, other {{c, select, b {}}}}'; - const targetMessage = '{a, select, other {{c, select, b {}}}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('other'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Missing "other" option'); - }); - - // parse - - it('generates a "parse" error with an unparseable target message', () => { - const sourceMessage = '{a, select, b {}}'; - const targetMessage = '{a, select b {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('parse'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found'); - }); - - it('generates a parse error with an unparseable target message', () => { - const sourceString = '{a, select, b {}}'; - const targetString = '{a, select b {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + // other + + it('generates an "other" error with missing other case', () => { + const sourceMessage = '{a, select, b {}}'; + const targetMessage = '{a, select, b {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('other'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Missing "other" option'); + }); + + it('generates an "other" error with missing nested other case', () => { + const sourceMessage = '{a, select, other {{c, select, b {}}}}'; + const targetMessage = '{a, select, other {{c, select, b {}}}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('other'); + expect(reporter.issues[0].level).to.equal('error'); + expect(reporter.issues[0].msg).to.equal('Missing "other" option'); + }); + + // parse + + it('generates a "parse" error with an unparseable target message', () => { + const sourceMessage = '{a, select, b {}}'; + const targetMessage = '{a, select b {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('parse'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found.'); + expect(reporter.issues[0].msg).to.equal('Expected "," but "b" found'); }); - it('generates a "source-error" error an unparseable source message', () => { - const sourceMessage = '{a, select other {}}'; - const targetMessage = '{a, select, other {}}'; - reporter.config(targetMessage, sourceMessage, 'key'); - validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('source-error'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Failed to parse source message.'); - }); - - it('generates a source-error error an unparseable source message', () => { - const sourceString = '{a, select b {}}'; - const targetString = '{a, select, b {}}'; - reporter.config(targetString, sourceString, 'key'); - validateMessage({ targetString, targetLocale, sourceString, sourceLocale }, reporter); + // source + + it('generates a "source-error" error an unparseable source message', () => { + const sourceMessage = '{a, select other {}}'; + const targetMessage = '{a, select, other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('source-error'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Failed to parse source string.'); + expect(reporter.issues[0].msg).to.equal('Failed to parse source message.'); }); }); describe('validateLocales', () => { - it('generates an "extraneous" error with unexpected message in target locale', () => { - const sourceMessage = '{a, select, other {}}'; - const targetMessage = '{a, select, other {}}'; - const locales = parseLocales([{ - file: `${targetLocale}.json`, - contents: JSON.stringify({ - a: targetMessage, - b: targetMessage - }, null, '\t') - }, - { - file: `${sourceLocale}.json`, - contents: JSON.stringify({ - a: sourceMessage - }, null, '\t') - }]); - - validateLocales({ locales, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('extraneous'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Message does not exist in the source file.'); - }); + // extraneous + + it('generates an "extraneous" error with unexpected message in target locale', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, other {}}'; + const locales = parseLocales([{ + file: `${targetLocale}.json`, + contents: JSON.stringify({ + a: targetMessage, + b: targetMessage + }, null, '\t') + }, + { + file: `${sourceLocale}.json`, + contents: JSON.stringify({ + a: sourceMessage + }, null, '\t') + }]); validateLocales({ locales, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('extraneous'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('This string does not exist in the source file.'); + expect(reporter.issues[0].msg).to.equal('Message does not exist in the source file.'); }); - it('generates a "missing" error with missing message in the target locale', () => { - const sourceMessage = '{a, select, other {}}'; - const targetMessage = '{a, select, other {}}'; - const locales = parseLocales([{ - file: `${targetLocale}.json`, - contents: JSON.stringify({ - a: targetMessage - }, null, '\t') - }, - { - file: `${sourceLocale}.json`, - contents: JSON.stringify({ - a: sourceMessage, - b: sourceMessage - }, null, '\t') - }]); - - validateLocales({ locales, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('missing'); - expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Message missing from locale file.'); - }); + // missing + + it('generates a "missing" error with missing message in the target locale', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, other {}}'; + const locales = parseLocales([{ + file: `${targetLocale}.json`, + contents: JSON.stringify({ + a: targetMessage + }, null, '\t') + }, + { + file: `${sourceLocale}.json`, + contents: JSON.stringify({ + a: sourceMessage, + b: sourceMessage + }, null, '\t') + }]); validateLocales({ locales, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('missing'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('String missing from locale file.'); + expect(reporter.issues[0].msg).to.equal('Message missing from locale file.'); }); - it('generates a "duplicate-keys" error with duplicate messages in the target locale', () => { - const sourceMessage = '{a, select, other {}}'; - const targetMessage = '{a, select, other {}}'; - const locales = parseLocales([{ - file: `${targetLocale}.json`, - contents: `{ - "a": "${sourceMessage}", - "a": "${sourceMessage}" - }` - }, - { - file: `${sourceLocale}.json`, - contents: `{ - "a": "${targetMessage}" - }` - }]); + // duplicate-keys - it('generates a duplicate-keys error with duplicate messages in the target locale', () => { - const sourceString = '{a, select, other {}}'; - const targetString = '{a, select, other {}}'; + it('generates a "duplicate-keys" error with duplicate messages in the target locale', () => { + const sourceMessage = '{a, select, other {}}'; + const targetMessage = '{a, select, other {}}'; const locales = parseLocales([{ file: `${targetLocale}.json`, contents: `{ - "a": "${sourceString}", - "a": "${sourceString}" - }` + "a": "${sourceMessage}", + "a": "${sourceMessage}" + }` }, { file: `${sourceLocale}.json`, contents: `{ - "a": "${targetString}" - }` + "a": "${targetMessage}" + }` }]); validateLocales({ locales, sourceLocale }, reporter); From 84864670161e8757793a39e62c72a16fc78da2d8 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 19 Sep 2024 09:19:24 -0400 Subject: [PATCH 16/90] Prevent exit code when errors are ignored --- bin/cli.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 151e7dd..a9805e9 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -18,7 +18,7 @@ const { path, source: globalSource, locales: globalLocales, jsonObj: globalJsonO program .version(pkg.version) .option('--no-issues', 'Don\'t output issues') - .option('-i, --ignoreIssueTypes ', 'Ignore these comma-separated issue types') + .option('-i, --ignore ', 'Ignore these comma-separated issue types') .option('-l, --locales ', 'Process only these comma-separated locales') .option('-p, --path ', 'Path to a directory containing locale files') .option('-s, --source-locale ', 'The locale to use as the source') @@ -284,7 +284,7 @@ localesPaths.forEach(async localesPath => { console.log((idx > 0 ? '\n' : '') + chalk.underline(localePath)); if (program.issues) { - locale.report.totals.ignored = 0; + locale.report.totals.ignored = { warnings: 0, errors: 0 }; if (program.sort) { const sorted = Object.values(locales[locale.locale].parsed) @@ -333,7 +333,7 @@ localesPaths.forEach(async localesPath => { translatorOutput[issue.key] = issue.source; } } - else if (!program.ignoreIssueTypes || !program.ignoreIssueTypes + else if (!program.ignore || !program.ignore .replace(' ','') .split(',') .includes(issue.type) @@ -347,7 +347,7 @@ localesPaths.forEach(async localesPath => { ].join('')); } else { - locale.report.totals.ignored += 1; + locale.report.totals.ignored[`${issue.level}s`] += 1; } }); } @@ -376,7 +376,8 @@ localesPaths.forEach(async localesPath => { else if (locale.report.totals.errors || locale.report.totals.warnings) { const color = locale.report.totals.errors ? 'red' : 'yellow'; const total = locale.report.totals.errors + locale.report.totals.warnings; - const cliReport = chalk[color](`\n\u2716 ${total} issues (${locale.report.totals.errors} errors, ${locale.report.totals.warnings} warnings)${locale.report.totals.ignored ? chalk.grey(` - ${locale.report.totals.ignored} Ignored`) : ''}`); + const ignored = locale.report.totals.ignored.errors + locale.report.totals.ignored.warnings; + const cliReport = chalk[color](`\n\u2716 ${total} issues (${locale.report.totals.errors} errors, ${locale.report.totals.warnings} warnings)${ignored ? chalk.grey(` - ${ignored} Ignored`) : ''}`); console.log(cliReport); return; } @@ -390,7 +391,7 @@ localesPaths.forEach(async localesPath => { })); - if (output.some(locale => locale.report?.totals.errors)) { + if (output.some(locale => locale.report && locale.report.totals.errors - locale.report.totals.ignored.errors )) { console.error('\nErrors were reported in at least one locale. See details above.'); return 1; } From 6371a075de927b9fb32303224fcfc27b3bd0931b Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 19 Sep 2024 09:20:22 -0400 Subject: [PATCH 17/90] Tidy error names and messages --- src/format.js | 1 + src/validate.js | 31 ++++++++++++++++++------------- test/validate.test.js | 42 +++++++++++++++++++++--------------------- 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/format.js b/src/format.js index 8774b62..8dda357 100644 --- a/src/format.js +++ b/src/format.js @@ -19,6 +19,7 @@ function expandASTHashes(ast, parentValue) { export function formatMessage(msg, options = {}) { let ast; + try { ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); } catch(err) { diff --git a/src/validate.js b/src/validate.js index fb0ccaf..1310bec 100644 --- a/src/validate.js +++ b/src/validate.js @@ -29,7 +29,7 @@ export function validateLocales({ locales, sourceLocale }, localesReporter) { reporter.config(targetMessages[key], sourceMessages[key]); if (!sourceMessage) { - reporter.error('extraneous', 'Message does not exist in the source file.'); + reporter.error('extraneous', 'Message does not exist in the source file'); } else { if (locales[targetLocale].duplicateKeys.has(key)) reporter.error('duplicate-keys', 'Key appears multiple times'); @@ -49,7 +49,7 @@ export function validateLocales({ locales, sourceLocale }, localesReporter) { if (missingKeys.length) { missingKeys.forEach((key) => { reporter.config(sourceMessages[key], sourceMessages[key]); - reporter.error('missing', `Message missing from locale file.`) + reporter.error('missing', 'Message missing from locale file') }) } @@ -68,7 +68,7 @@ function checkNbsp(message, reporter) { const nbspPos = structure.indexOf(String.fromCharCode(160)); if (nbspPos > -1) { - reporter.error('nbsp', `Message contains invalid non-breaking space at position ${nbspPos}.`, { column: nbspPos }); + reporter.error('nbsp', `Message contains invalid non-breaking space at position ${nbspPos}`, { column: nbspPos }); return true; } } @@ -87,7 +87,7 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so .replace(re,'') .replace(/\s/g, '')) { - msgReporter.warning('untranslated', `Message has not been translated.`); + msgReporter.warning('untranslated', 'Message has not been translated'); } } @@ -118,7 +118,7 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so sourceTokens = parse(sourceMessage, { requiresOtherClause: false }); } catch(e) { - msgReporter.error('source-error', 'Failed to parse source message.'); + msgReporter.error('source-error', 'Failed to parse source message'); return; } @@ -133,12 +133,12 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const supportedCats = getPluralCats(targetLocale, part.pluralType); const cats = Object.keys(part.options); const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); - if (missingCats.length) msgReporter.warning('categories', `Missing categories ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); + if (missingCats.length) msgReporter.warning('categories-missing', `Missing ${missingCats.length === 1 ? 'category' : 'categories'} ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); const unsupportedCats = cats.filter(c => !/^=\d+$/.test(c) && !supportedCats.includes(c)); unsupportedCats.forEach(cat => { const column = part.options[cat].location.start.offset; - msgReporter.error('categories', `Unsupported category "${cat}". Must be one of: "${supportedCats.join('", "')}", or explicit keys like "=0"`, { column }); + msgReporter.error('categories', `Unsupported category "${cat}". Locale supports "${supportedCats.join('", "')}", and explicit keys like "=0".`, { column }); }); } @@ -158,7 +158,7 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const badArgPos = targetMessage.indexOf(argDiff[0]); if (argDiff.length) { - msgReporter.error('argument', `Unrecognized arguments: ${argDiff.join(', ')}. Must be one of: ${Array.from(sourceMap.arguments).join(', ')}`, { column: badArgPos }); + msgReporter.error('argument', `Unrecognized ${argDiff.length === 1 ? 'argument' : 'arguments'} ${formatList(argDiff.map(i => `"${i}"`))}. Source message uses ${formatList(Array.from(sourceMap.arguments).map(i => `"${i}"`))}.`, { column: badArgPos }); } checkNbsp(targetMessage, msgReporter); @@ -167,15 +167,20 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const cleanTargetCases = targetMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); const cleanSourceCases = sourceMap.cases.map(c => c.replace(/.+(?<=\|(6(ordinal|cardinal))\|).*/, '')); - const optionDiff = cleanTargetCases.filter(arg => !cleanSourceCases.includes(arg)); - optionDiff.forEach(o => { - msgReporter.error('option', `Unrecognized option "${o.replace(/.+\|5undefined\|/, '')}". Must be one of "${cleanSourceCases.map(o => o.replace(/.+\|5undefined\|/, '')).join('", "')}".`); + const extraOptions = cleanTargetCases.filter(arg => !cleanSourceCases.includes(arg)); + extraOptions.forEach(o => { + msgReporter.error('option', `Unrecognized option "${o.replace(/.+\|5undefined\|/, '')}". Argument uses ${formatList(cleanSourceCases.map(o => `"${o.replace(/.+\|5undefined\|/, '')}"`))}.`); + }); + + const missingOptions = cleanSourceCases.filter(arg => !cleanTargetCases.includes(arg)); + missingOptions.forEach(o => { + msgReporter.error('option-missing', `Missing option "${o.replace(/.+\|5undefined\|/, '')}"`); }); if (targetMap.nested && targetMap.cases.length === sourceMap.cases.length) { // TODO: better identify case order vs nesting order - msgReporter.warning('nest-order', `Nesting order does not match source.`); + msgReporter.warning('nest-order', 'Nesting order does not match source'); } } @@ -183,7 +188,7 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const lastSelect = targetMap.cases.findLastIndex(i => i.match(/^.+\|5undefined\|/)) + 1; if (targetMap.nested && firstPlural && lastSelect && firstPlural < lastSelect) { - msgReporter.warning('nest-ideal', '"plural" and "selectordinal" should always nest inside "select".'); + msgReporter.warning('nest-ideal', '"plural" and "selectordinal" should always nest inside "select"'); } if (targetTokens.length > 1) { diff --git a/test/validate.test.js b/test/validate.test.js index b453fa8..171d6a3 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -35,12 +35,12 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('untranslated'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Message has not been translated.'); + expect(reporter.issues[0].msg).to.equal('Message has not been translated'); }); // categories - it('generates a "categories" warning when a target message is missing supported plural categories', () => { + it('generates a "categories-missing" warning when a target message is missing supported plural categories', () => { targetLocale = 'cy-gb'; const sourceMessage = '{a, plural, one {} other {}}'; const targetMessage = '{a, plural, one {} other {}}'; @@ -48,7 +48,7 @@ describe('validate', () => { reporter._config.locale = targetLocale; validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].type).to.equal('categories-missing'); expect(reporter.issues[0].level).to.equal('warning'); expect(reporter.issues[0].msg).to.equal('Missing categories "zero", "two", "few", and "many"'); }); @@ -63,15 +63,15 @@ describe('validate', () => { expect(reporter.issues[0].type).to.equal('categories'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Must be one of: "one", "other", or explicit keys like "=0"'); + expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Locale supports "one", "other", and explicit keys like "=0".'); expect(reporter.issues[1].type).to.equal('categories'); expect(reporter.issues[1].level).to.equal('error'); - expect(reporter.issues[1].msg).to.equal('Unsupported category "few". Must be one of: "one", "other", or explicit keys like "=0"'); + expect(reporter.issues[1].msg).to.equal('Unsupported category "few". Locale supports "one", "other", and explicit keys like "=0".'); expect(reporter.issues[2].type).to.equal('categories'); expect(reporter.issues[2].level).to.equal('error'); - expect(reporter.issues[2].msg).to.equal('Unsupported category "many". Must be one of: "one", "other", or explicit keys like "=0"'); + expect(reporter.issues[2].msg).to.equal('Unsupported category "many". Locale supports "one", "other", and explicit keys like "=0".'); }); it('generates "categories" errors when a target message uses nested unsupported plural categories', () => { @@ -83,7 +83,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('categories'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Must be one of: "one", "other", or explicit keys like "=0"'); + expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Locale supports "one", "other", and explicit keys like "=0".'); }); // split @@ -111,7 +111,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('argument'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized arguments: arG. Must be one of: arg'); + expect(reporter.issues[0].msg).to.equal('Unrecognized argument "arG". Source message uses "arg".'); }); // brace @@ -145,9 +145,9 @@ describe('validate', () => { // option - it('generates "option" errors with unrecognized cases in select arguments', () => { + it('generates "option" errors with unrecognized options in select arguments', () => { const sourceMessage = '{a, select, b {} other {}}'; - const targetMessage = '{a, select, B {} C {} other {}}'; + const targetMessage = '{a, select, b {} c {} d {} other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); @@ -155,22 +155,22 @@ describe('validate', () => { expect(reporter.issues[0].type).to.equal('option'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized option "B". Must be one of "b", "other".'); + expect(reporter.issues[0].msg).to.equal('Unrecognized option "c". Argument uses "b" and "other".'); expect(reporter.issues[1].type).to.equal('option'); expect(reporter.issues[1].level).to.equal('error'); - expect(reporter.issues[1].msg).to.equal('Unrecognized option "C". Must be one of "b", "other".'); + expect(reporter.issues[1].msg).to.equal('Unrecognized option "d". Argument uses "b" and "other".'); }); - it.skip('generates a "option" error with missing cases in select arguments', () => { + it('generates an "option" error with missing options in select arguments', () => { const sourceMessage = '{a, select, b {} other {}}'; const targetMessage = '{a, select, other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('option'); + expect(reporter.issues[0].type).to.equal('option-missing'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Missing cases ["b"]'); + expect(reporter.issues[0].msg).to.equal('Missing option "b"'); }); // nbsp @@ -184,7 +184,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('nbsp'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Message contains invalid non-breaking space at position 11.'); + expect(reporter.issues[0].msg).to.equal('Message contains invalid non-breaking space at position 11'); }); // nest @@ -197,7 +197,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('nest-order'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Nesting order does not match source.'); + expect(reporter.issues[0].msg).to.equal('Nesting order does not match source'); }); it('generates a "nest-ideal" error with plural inside select', () => { @@ -208,7 +208,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('nest-ideal'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('"plural" and "selectordinal" should always nest inside "select".'); + expect(reporter.issues[0].msg).to.equal('"plural" and "selectordinal" should always nest inside "select"'); }); // other @@ -260,7 +260,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('source-error'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Failed to parse source message.'); + expect(reporter.issues[0].msg).to.equal('Failed to parse source message'); }); }); @@ -290,7 +290,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('extraneous'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Message does not exist in the source file.'); + expect(reporter.issues[0].msg).to.equal('Message does not exist in the source file'); }); // missing @@ -316,7 +316,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('missing'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Message missing from locale file.'); + expect(reporter.issues[0].msg).to.equal('Message missing from locale file'); }); // duplicate-keys From 2b72aca1dbe1a48522cc171313584be98a0d2e46 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 19 Sep 2024 10:55:38 -0400 Subject: [PATCH 18/90] Build required cldr data; cldr to devDependencies --- build-cldr-data.js | 26 +++++++ package-lock.json | 23 +++++- package.json | 5 +- src/cldr-data.js | 178 +++++++++++++++++++++++++++++++++++++++++++++ src/format.js | 12 +-- 5 files changed, 230 insertions(+), 14 deletions(-) create mode 100644 build-cldr-data.js create mode 100644 src/cldr-data.js diff --git a/build-cldr-data.js b/build-cldr-data.js new file mode 100644 index 0000000..a771c3b --- /dev/null +++ b/build-cldr-data.js @@ -0,0 +1,26 @@ +#!/usr/bin/env -S node --no-warnings --experimental-json-modules +import { writeFile } from 'node:fs/promises'; +import { env } from 'node:process'; +import cldr from 'cldr'; + +const defaultLocales = ['ar', 'cy', 'da', 'de', 'en', 'en-gb', 'es', 'es-es', 'fr', 'fr-ca', 'fr-fr', 'haw', 'hi', 'ja', 'ko', 'mi', 'nl', 'pt', 'sv', 'tr', 'zh-cn', 'zh-tw']; + +function getDelimiters(locale) { + try { + return cldr.extractDelimiters(locale); + } catch(err) { + return cldr.extractDelimiters(locale.split('-')[0]); + } +} + +const locales = env.MFV_LOCALES?.split(',') ?? defaultLocales; + +const data = {}; + +locales.forEach(locale => { + locale = locale.trim().toLowerCase(); + data[locale] = {}; + data[locale].delimiters = getDelimiters(locale); +}); + +await writeFile('./src/cldr-data.js', `export default ${JSON.stringify(data, null, '\t')}\n`); diff --git a/package-lock.json b/package-lock.json index f977982..f03edd5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,6 @@ "dependencies": { "@formatjs/icu-messageformat-parser": "^2.7.8", "chalk": "^4.1.0", - "cldr": "^7.5.0", "commander": "^6.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", @@ -27,6 +26,7 @@ "@eslint/eslintrc": "^3.1.0", "@eslint/js": "^9.10.0", "chai": "^5.1.1", + "cldr": "^7.5.0", "eslint": "^8.57.0", "globals": "^15.9.0", "mocha": "^10.7.3" @@ -718,6 +718,7 @@ "version": "0.8.10", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.10.tgz", "integrity": "sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==", + "dev": true, "engines": { "node": ">=10.0.0" } @@ -959,6 +960,7 @@ "version": "0.0.9", "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.0.9.tgz", "integrity": "sha512-nG8PYH+/4xB+8zkV4G844EtfvZ5tTiLFoX3dZ4nhF4t3OCKIb9UvaFyNmeZO2zOSmRWzBoTD+napN6hiL+EgcA==", + "dev": true, "dependencies": { "traverse": ">=0.3.0 <0.4" }, @@ -1018,6 +1020,7 @@ "version": "7.5.0", "resolved": "https://registry.npmjs.org/cldr/-/cldr-7.5.0.tgz", "integrity": "sha512-2qy3ASYFbNToTujNnk5Y8ak++B4TH/G+S8AEOrN1xUFZhxhmqWDPUGnOFGyId61vD2Trf+yE65wVzIcdE/bpPg==", + "dev": true, "dependencies": { "@xmldom/xmldom": "^0.8.0", "escodegen": "^2.0.0", @@ -1194,6 +1197,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, "dependencies": { "esprima": "^4.0.1", "estraverse": "^5.2.0", @@ -1441,6 +1445,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" @@ -1477,6 +1482,7 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, "engines": { "node": ">=4.0" } @@ -1485,6 +1491,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, "engines": { "node": ">=0.10.0" } @@ -1704,6 +1711,7 @@ "version": "0.0.4", "resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz", "integrity": "sha512-xyD4XgslstNAs72ENaoFvgMwtv8xhiDtC2AtzCG+8yF7W/Knxxm9BX+e2s25mm+HxMKh0rBmXVOEGF3zNImXvA==", + "dev": true, "dependencies": { "traverse": ">=0.2.4" }, @@ -1989,12 +1997,14 @@ "node_modules/lru-cache": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.0.tgz", - "integrity": "sha512-dVmQmXPBlTgFw77hm60ud//l2bCuDKkqC2on1EBoM7s9Urm9IQDrnujwZ93NFnAq0dVZ0HBXTS7PwEG+YE7+EQ==" + "integrity": "sha512-dVmQmXPBlTgFw77hm60ud//l2bCuDKkqC2on1EBoM7s9Urm9IQDrnujwZ93NFnAq0dVZ0HBXTS7PwEG+YE7+EQ==", + "dev": true }, "node_modules/memoizeasync": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/memoizeasync/-/memoizeasync-1.1.0.tgz", "integrity": "sha512-HMfzdLqClZo8HMyuM9B6TqnXCNhw82iVWRLqd2cAdXi063v2iJB4mQfWFeKVByN8VUwhmDZ8NMhryBwKrPRf8Q==", + "dev": true, "dependencies": { "lru-cache": "2.5.0", "passerror": "1.1.1" @@ -2209,6 +2219,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/passerror/-/passerror-1.1.1.tgz", "integrity": "sha512-PwrEQJBkJMxnxG+tdraz95vTstYnCRqiURNbGtg/vZHLgcAODc9hbiD5ZumGUoh3bpw0F0qKLje7Vd2Fd5Lx3g==", + "dev": true, "engines": { "node": "*" } @@ -2252,6 +2263,7 @@ "version": "0.10.0", "resolved": "https://registry.npmjs.org/pegjs/-/pegjs-0.10.0.tgz", "integrity": "sha512-qI5+oFNEGi3L5HAxDwN2LA4Gg7irF70Zs25edhjld9QemOgp0CbvMtbFcMvFtEo1OityPrcCzkQFB8JP/hxgow==", + "dev": true, "bin": { "pegjs": "bin/pegjs" }, @@ -2437,6 +2449,7 @@ "version": "0.3.5", "resolved": "https://registry.npmjs.org/seq/-/seq-0.3.5.tgz", "integrity": "sha512-sisY2Ln1fj43KBkRtXkesnRHYNdswIkIibvNe/0UKm2GZxjMbqmccpiatoKr/k2qX5VKiLU8xm+tz/74LAho4g==", + "dev": true, "dependencies": { "chainsaw": ">=0.0.7 <0.1", "hashish": ">=0.0.2 <0.1" @@ -2479,6 +2492,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, "optional": true, "engines": { "node": ">=0.10.0" @@ -2565,6 +2579,7 @@ "version": "0.3.9", "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "dev": true, "engines": { "node": "*" } @@ -2601,7 +2616,8 @@ "node_modules/unicoderegexp": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/unicoderegexp/-/unicoderegexp-0.4.1.tgz", - "integrity": "sha512-ydh8D5mdd2ldTS25GtZJEgLciuF0Qf2n3rwPhonELk3HioX201ClYGvZMc1bCmx6nblZiADQwbMWekeIqs51qw==" + "integrity": "sha512-ydh8D5mdd2ldTS25GtZJEgLciuF0Qf2n3rwPhonELk3HioX201ClYGvZMc1bCmx6nblZiADQwbMWekeIqs51qw==", + "dev": true }, "node_modules/update-browserslist-db": { "version": "1.1.0", @@ -2710,6 +2726,7 @@ "version": "0.0.33", "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.33.tgz", "integrity": "sha512-NNXnzrkDrAzalLhIUc01jO2mOzXGXh1JwPgkihcLLzw98c0WgYDmmjSh1Kl3wzaxSVWMuA+fe0WTWOBDWCBmNA==", + "dev": true, "engines": { "node": ">=0.6.0" } diff --git a/package.json b/package.json index f9e1c08..af241b0 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "lint:eslint": "eslint . --ext .js --ignore-path .gitignore", "test": "npm run lint && npm run test:unit", "test:unit": "mocha 'test/**/*.test.js'", - "prepublishOnly": "npm t" + "prepare": "npm run build && npm t", + "build": "node build-cldr-data.js" }, "bin": { "mfv": "bin/cli.js" @@ -29,7 +30,6 @@ "dependencies": { "@formatjs/icu-messageformat-parser": "^2.7.8", "chalk": "^4.1.0", - "cldr": "^7.5.0", "commander": "^6.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", @@ -41,6 +41,7 @@ "@eslint/compat": "^1.1.1", "@eslint/eslintrc": "^3.1.0", "@eslint/js": "^9.10.0", + "cldr": "^7.5.0", "chai": "^5.1.1", "eslint": "^8.57.0", "globals": "^15.9.0", diff --git a/src/cldr-data.js b/src/cldr-data.js new file mode 100644 index 0000000..950c9db --- /dev/null +++ b/src/cldr-data.js @@ -0,0 +1,178 @@ +export default { + "ar": { + "delimiters": { + "quotationStart": "”", + "quotationEnd": "“", + "alternateQuotationStart": "’", + "alternateQuotationEnd": "‘" + } + }, + "cy": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "da": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "de": { + "delimiters": { + "quotationStart": "„", + "quotationEnd": "“", + "alternateQuotationStart": "‚", + "alternateQuotationEnd": "‘" + } + }, + "en": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "en-gb": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "es": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "es-es": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "fr": { + "delimiters": { + "quotationStart": "«", + "quotationEnd": "»", + "alternateQuotationStart": "«", + "alternateQuotationEnd": "»" + } + }, + "fr-ca": { + "delimiters": { + "alternateQuotationStart": "”", + "alternateQuotationEnd": "“", + "quotationStart": "«", + "quotationEnd": "»" + } + }, + "fr-fr": { + "delimiters": { + "quotationStart": "«", + "quotationEnd": "»", + "alternateQuotationStart": "«", + "alternateQuotationEnd": "»" + } + }, + "haw": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "hi": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "ja": { + "delimiters": { + "quotationStart": "「", + "quotationEnd": "」", + "alternateQuotationStart": "『", + "alternateQuotationEnd": "』" + } + }, + "ko": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "mi": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "nl": { + "delimiters": { + "quotationStart": "‘", + "quotationEnd": "’", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "pt": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "sv": { + "delimiters": { + "quotationStart": "”", + "alternateQuotationStart": "’", + "quotationEnd": "”", + "alternateQuotationEnd": "’" + } + }, + "tr": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "zh-cn": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "zh-tw": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + } +} diff --git a/src/format.js b/src/format.js index 8dda357..2f71c61 100644 --- a/src/format.js +++ b/src/format.js @@ -1,7 +1,7 @@ import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.js'; import { parse } from '@formatjs/icu-messageformat-parser'; import { getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from './utils.js'; -import cldr from 'cldr'; +import cldrData from './cldr-data.js'; function expandASTHashes(ast, parentValue) { if (Array.isArray(ast)) { @@ -88,19 +88,13 @@ function printAST(ast, options, level = 0) { } = options; const localeLower = locale.toLowerCase(); + const localeData = cldrData[locale] ?? cldrData[locale.split('-')[0]]; + const delimiters = localeData.delimiters; if (Array.isArray(ast)) { const swapOneClone = new Set(swapOne); ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) - const delimiters = (() => { - try { - return cldr.extractDelimiters(locale); - } catch(err) { - return cldr.extractDelimiters(locale.split('-')[0]); - } - })(); - for (const k in delimiters) { if (paddedQuoteLocales.includes(localeLower)) { if (k.endsWith('Start')) { From 3eb729fc9e4932414fb7ed7818339ae9ea8edadf Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 20 Sep 2024 00:55:03 -0400 Subject: [PATCH 19/90] Move find-config to utils --- bin/cli.js | 21 ++++++++++++--------- src/utils.js | 9 +++++++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index a9805e9..29ae38c 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -5,15 +5,19 @@ import { parseLocales, validateLocales } from '../src/validate.js'; import { readFile, readdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; -import findConfig from 'find-config'; -import { formatMessage } from '../src/format.js' import glob from 'glob'; import pkg from '../package.json' with { type: 'json' }; import { program } from 'commander'; -import { structureRegEx } from '../src/utils.js'; +import { getConfig, structureRegEx } from '../src/utils.js'; -const configPath = findConfig('mfv.config.json'); -const { path, source: globalSource, locales: globalLocales, jsonObj: globalJsonObj } = configPath ? (await import(`file://${configPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; +let formatMessage; + +const { + path, + source: globalSource, + locales: globalLocales, + jsonObj: globalJsonObj +} = getConfig(); program .version(pkg.version) @@ -74,7 +78,8 @@ program .option('-d, --dedupe', 'Remove complex argument cases that duplicate the `other` case. Takes precedence over --add.') .option('-t, --trim', 'Trim whitespace from both ends of messages') .option('-c, --collapse', 'Collapse repeating whitepace') - .action(function() { + .action(async function() { + formatMessage = (await import('../src/format.js')).formatMessage; program.format = true; const opts = this.opts(); program.newlines = opts.newlines; @@ -110,9 +115,7 @@ localesPaths.forEach(async localesPath => { const absLocalesPath = `${process.cwd()}/${localesPath}`; - const subConfigPath = findConfig('mfv.config.json', { cwd: absLocalesPath }); - - const { source, locales: configLocales, jsonObj } = subConfigPath ? (await import(`file://${subConfigPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; /* eslint-disable-line global-require */ + const { source, locales: configLocales, jsonObj } = await getConfig(absLocalesPath); const files = await readdir(absLocalesPath).catch(err => { console.log(`Failed to read ${absLocalesPath}\n`); diff --git a/src/utils.js b/src/utils.js index f7a723a..48be62e 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,3 +1,12 @@ +import findConfig from 'find-config'; + +export async function getConfig(cwd) { + const configPath = findConfig('mfv.config.json', { cwd }); + const config = configPath ? (await import(`file://${configPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; + config.__configPath = configPath; + return config; +} + export const sortedCats = ['zero', 'one', 'two', 'few', 'many', 'other']; export const paddedQuoteLocales = ['fr', 'fr-ca', 'fr-fr', 'fr-on', 'vi-vn']; export const structureRegEx = /(?<=\s*){(.|\n)*?[{}]|\s*}(.|\n)*?[{}]|[{#]|(\s*)}/g; From be733d2a2468ff6e1eca935d22b4a563565f23e2 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 20 Sep 2024 00:59:39 -0400 Subject: [PATCH 20/90] Add build script --- bin/cli.js | 12 +++ build-cldr-data.js | 47 +++++++--- package-lock.json | 1 + package.json | 4 +- src/cldr-data-default.js | 154 +++++++++++++++++++++++++++++++++ src/cldr-data.js | 180 +-------------------------------------- 6 files changed, 206 insertions(+), 192 deletions(-) create mode 100644 src/cldr-data-default.js diff --git a/bin/cli.js b/bin/cli.js index 29ae38c..e3d097f 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -32,6 +32,13 @@ program program.validate = true; }); +program + .command('build [locales]') + .description('Build locale data for configured locales') + .action(() => { + program.build = true; + }); + program .command('print-missing') .description('Output JSON of all source messages that are missing or untranslated in the target') @@ -99,6 +106,11 @@ program program.parse(process.argv); +if (program.build) { + await import('../build-cldr-data.js'); + process.exit(); +} + const pathCombined = program.path || path; if (!pathCombined) { console.error('Must provide a path to the locale files using either the -p option or a config file.'); diff --git a/build-cldr-data.js b/build-cldr-data.js index a771c3b..9ce2cfa 100644 --- a/build-cldr-data.js +++ b/build-cldr-data.js @@ -1,10 +1,12 @@ -#!/usr/bin/env -S node --no-warnings --experimental-json-modules -import { writeFile } from 'node:fs/promises'; -import { env } from 'node:process'; -import cldr from 'cldr'; +import { readdir, writeFile } from 'node:fs/promises'; +import { env, stderr } from 'node:process'; +import { formatList, getConfig } from './src/utils.js'; +import { dirname, join, posix } from 'node:path'; const defaultLocales = ['ar', 'cy', 'da', 'de', 'en', 'en-gb', 'es', 'es-es', 'fr', 'fr-ca', 'fr-fr', 'haw', 'hi', 'ja', 'ko', 'mi', 'nl', 'pt', 'sv', 'tr', 'zh-cn', 'zh-tw']; +const SAVE_PATH = posix.join(dirname(import.meta.url), 'src/cldr-data.js').replace(/file:(\/c:)?/i, ''); + function getDelimiters(locale) { try { return cldr.extractDelimiters(locale); @@ -13,14 +15,33 @@ function getDelimiters(locale) { } } -const locales = env.MFV_LOCALES?.split(',') ?? defaultLocales; - -const data = {}; +let cldr; +await (async() => { + const config = await getConfig(); + let locales = env.MFV_LOCALES?.split(',') ?? config.locales; + locales ??= config.path && (await readdir(join(dirname(config.__configPath), config.path)).catch(() => {}))?.map(f => f.split('.')[0]); + locales ??= defaultLocales; + const nonDefaultLocales = locales?.filter(l => !defaultLocales.includes(l)); + if (nonDefaultLocales?.length) { + let cldrImport; + try { + cldrImport = await import('cldr'); + } catch(e) { + stderr.write(`\n\nSome configured locales (${formatList(nonDefaultLocales.map(l => `"${l}"`))}) require the 'cldr' package: npm i -D cldr\n\n`); + process.exitCode = 1; + return; + } + cldr = (cldrImport).default; + const data = {}; -locales.forEach(locale => { - locale = locale.trim().toLowerCase(); - data[locale] = {}; - data[locale].delimiters = getDelimiters(locale); -}); + locales.forEach(locale => { + locale = locale.trim().toLowerCase(); + data[locale] = {}; + data[locale].delimiters = getDelimiters(locale); + }); -await writeFile('./src/cldr-data.js', `export default ${JSON.stringify(data, null, '\t')}\n`); + await writeFile(SAVE_PATH, `export default ${JSON.stringify(data, null, '\t')}\n`); + } else { + await writeFile(SAVE_PATH, `import cldrData from './cldr-data-default.js';\nexport default cldrData;\n`); + } +})(); diff --git a/package-lock.json b/package-lock.json index f03edd5..7579cc4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "messageformat-validator", "version": "3.0.0-alpha.1", + "hasInstallScript": true, "license": "MIT", "dependencies": { "@formatjs/icu-messageformat-parser": "^2.7.8", diff --git a/package.json b/package.json index af241b0..5342fb8 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "test": "npm run lint && npm run test:unit", "test:unit": "mocha 'test/**/*.test.js'", "prepare": "npm run build && npm t", + "postinstall": "npm run build", "build": "node build-cldr-data.js" }, "bin": { @@ -23,7 +24,8 @@ ".": "./src/index.js" }, "files": [ - "/src" + "/src", + "/build-cldr-data.js" ], "author": "Danny Gleckler ", "license": "MIT", diff --git a/src/cldr-data-default.js b/src/cldr-data-default.js new file mode 100644 index 0000000..4b2321e --- /dev/null +++ b/src/cldr-data-default.js @@ -0,0 +1,154 @@ +export default { + "ar": { + "delimiters": { + "quotationStart": "”", + "quotationEnd": "“", + "alternateQuotationStart": "’", + "alternateQuotationEnd": "‘" + } + }, + "cy": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "da": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "de": { + "delimiters": { + "quotationStart": "„", + "quotationEnd": "“", + "alternateQuotationStart": "‚", + "alternateQuotationEnd": "‘" + } + }, + "en-gb": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "en": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "es-es": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "es": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "fr-fr": { + "delimiters": { + "quotationStart": "«", + "quotationEnd": "»", + "alternateQuotationStart": "«", + "alternateQuotationEnd": "»" + } + }, + "fr": { + "delimiters": { + "quotationStart": "«", + "quotationEnd": "»", + "alternateQuotationStart": "«", + "alternateQuotationEnd": "»" + } + }, + "hi": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "ja": { + "delimiters": { + "quotationStart": "「", + "quotationEnd": "」", + "alternateQuotationStart": "『", + "alternateQuotationEnd": "』" + } + }, + "ko": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "nl": { + "delimiters": { + "quotationStart": "‘", + "quotationEnd": "’", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "pt": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "sv": { + "delimiters": { + "quotationStart": "”", + "alternateQuotationStart": "’", + "quotationEnd": "”", + "alternateQuotationEnd": "’" + } + }, + "tr": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "zh-cn": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + }, + "zh-tw": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’" + } + } +} diff --git a/src/cldr-data.js b/src/cldr-data.js index 950c9db..c7af759 100644 --- a/src/cldr-data.js +++ b/src/cldr-data.js @@ -1,178 +1,2 @@ -export default { - "ar": { - "delimiters": { - "quotationStart": "”", - "quotationEnd": "“", - "alternateQuotationStart": "’", - "alternateQuotationEnd": "‘" - } - }, - "cy": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "da": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "de": { - "delimiters": { - "quotationStart": "„", - "quotationEnd": "“", - "alternateQuotationStart": "‚", - "alternateQuotationEnd": "‘" - } - }, - "en": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "en-gb": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "es": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "es-es": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "fr": { - "delimiters": { - "quotationStart": "«", - "quotationEnd": "»", - "alternateQuotationStart": "«", - "alternateQuotationEnd": "»" - } - }, - "fr-ca": { - "delimiters": { - "alternateQuotationStart": "”", - "alternateQuotationEnd": "“", - "quotationStart": "«", - "quotationEnd": "»" - } - }, - "fr-fr": { - "delimiters": { - "quotationStart": "«", - "quotationEnd": "»", - "alternateQuotationStart": "«", - "alternateQuotationEnd": "»" - } - }, - "haw": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "hi": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "ja": { - "delimiters": { - "quotationStart": "「", - "quotationEnd": "」", - "alternateQuotationStart": "『", - "alternateQuotationEnd": "』" - } - }, - "ko": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "mi": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "nl": { - "delimiters": { - "quotationStart": "‘", - "quotationEnd": "’", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "pt": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "sv": { - "delimiters": { - "quotationStart": "”", - "alternateQuotationStart": "’", - "quotationEnd": "”", - "alternateQuotationEnd": "’" - } - }, - "tr": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "zh-cn": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - }, - "zh-tw": { - "delimiters": { - "quotationStart": "“", - "quotationEnd": "”", - "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" - } - } -} +import cldrData from './cldr-data-default.js'; +export default cldrData; From 95715309b1bc58d91a7fcb431874f745bc1a2a91 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 20 Sep 2024 01:01:19 -0400 Subject: [PATCH 21/90] Locales config as array --- bin/cli.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index e3d097f..dd2edf5 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -135,8 +135,7 @@ localesPaths.forEach(async localesPath => { }); const sourceLocale = program.sourceLocale || source || globalSource; - const allowedLocalesString = program.locales || configLocales || globalLocales; - const allowedLocales = allowedLocalesString && allowedLocalesString.split(',').concat(sourceLocale); + const allowedLocales = (program.locales?.replace(/\s/g, '').split(',') || configLocales || globalLocales || []).concat(sourceLocale); const filteredFiles = !allowedLocales ? files.filter(file => !(/^\..*/g).test(file)) : files.filter(file => allowedLocales.includes(file.split('.')[0])); From d009584489be04bf60e0c81a7147b4b95262c54b Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 20 Sep 2024 08:35:01 -0400 Subject: [PATCH 22/90] Prevent double build on install --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5342fb8..8e10bc7 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "lint:eslint": "eslint . --ext .js --ignore-path .gitignore", "test": "npm run lint && npm run test:unit", "test:unit": "mocha 'test/**/*.test.js'", - "prepare": "npm run build && npm t", + "prepack": "npm run build && npm t", "postinstall": "npm run build", "build": "node build-cldr-data.js" }, From 09f07f6778159c16b9c0d57f693d434b54f930d1 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 20 Sep 2024 08:38:14 -0400 Subject: [PATCH 23/90] Cleanup --- build-cldr-data.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build-cldr-data.js b/build-cldr-data.js index 9ce2cfa..16cf941 100644 --- a/build-cldr-data.js +++ b/build-cldr-data.js @@ -17,11 +17,15 @@ function getDelimiters(locale) { let cldr; await (async() => { + let contents = `import cldrData from './cldr-data-default.js';\nexport default cldrData;\n`; + const config = await getConfig(); + let locales = env.MFV_LOCALES?.split(',') ?? config.locales; locales ??= config.path && (await readdir(join(dirname(config.__configPath), config.path)).catch(() => {}))?.map(f => f.split('.')[0]); locales ??= defaultLocales; const nonDefaultLocales = locales?.filter(l => !defaultLocales.includes(l)); + if (nonDefaultLocales?.length) { let cldrImport; try { @@ -40,8 +44,7 @@ await (async() => { data[locale].delimiters = getDelimiters(locale); }); - await writeFile(SAVE_PATH, `export default ${JSON.stringify(data, null, '\t')}\n`); - } else { - await writeFile(SAVE_PATH, `import cldrData from './cldr-data-default.js';\nexport default cldrData;\n`); + contents = `export default ${JSON.stringify(data, null, '\t')}\n`; } + await writeFile(SAVE_PATH, contents); })(); From 063d9ae0e24c466b408901c34c511066b613f9c3 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 20 Sep 2024 09:07:40 -0400 Subject: [PATCH 24/90] Rework duplicate error --- src/validate.js | 2 +- test/validate.test.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/validate.js b/src/validate.js index 1310bec..23b9b42 100644 --- a/src/validate.js +++ b/src/validate.js @@ -32,7 +32,7 @@ export function validateLocales({ locales, sourceLocale }, localesReporter) { reporter.error('extraneous', 'Message does not exist in the source file'); } else { - if (locales[targetLocale].duplicateKeys.has(key)) reporter.error('duplicate-keys', 'Key appears multiple times'); + if (locales[targetLocale].duplicateKeys.has(key)) reporter.error('duplicate', `Multiple messages named "${key}"`); validateMessage({ targetMessage, diff --git a/test/validate.test.js b/test/validate.test.js index 171d6a3..7295369 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -328,7 +328,7 @@ describe('validate', () => { file: `${targetLocale}.json`, contents: `{ "a": "${sourceMessage}", - "a": "${sourceMessage}" + a: "${sourceMessage}" }` }, { @@ -340,9 +340,9 @@ describe('validate', () => { validateLocales({ locales, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('duplicate-keys'); + expect(reporter.issues[0].type).to.equal('duplicate'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Key appears multiple times'); + expect(reporter.issues[0].msg).to.equal('Multiple messages named "a"'); }); }); From 1e263184a2892aa567b82e6061267ebffac1f30e Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 5 Dec 2024 16:33:14 -0500 Subject: [PATCH 25/90] Lots of stuff --- README.md | 68 +++++-- bin/cli.js | 97 ++++++--- build-cldr-data.js => build-locale-data.js | 11 +- package-lock.json | 10 +- package.json | 8 +- src/cldr-data.js | 2 - src/format.js | 184 ++++++++++++++---- ...data-default.js => locale-data-default.js} | 8 + src/utils.js | 6 + src/validate.js | 18 +- test/format.test.js | 31 ++- test/validate.test.js | 41 ++-- 12 files changed, 348 insertions(+), 136 deletions(-) rename build-cldr-data.js => build-locale-data.js (80%) delete mode 100644 src/cldr-data.js rename src/{cldr-data-default.js => locale-data-default.js} (95%) diff --git a/README.md b/README.md index e9268bc..54eef57 100644 --- a/README.md +++ b/README.md @@ -35,11 +35,9 @@ mfv -l es-es -p lang/ highlight myMessage `-V, --version` - output the version number -`-e, --throw-errors` - Throw an error if error issues are found - `--no-issues` - Don't output issues -`-i, --ignoreIssueTypes ` - Ignore these comma-separated issue types +`-i, --ignore ` - Ignore these comma-separated issue types `-l, --locales ` - Process only these comma-separated locales @@ -74,39 +72,73 @@ Some options can be configured with default values in `mfv.config.json` { "source": "en" "path": "lang/", - "locales": "ar,de,en,es,es-es,hi,tr", + "locales": ["ar", "de", "en", "es", "es-es", "hi", "tr"], "jsonObj": true } ``` ## Errors -`argument` - There are unrecognized arguments in the target message. +`argument` - Unrecognized argument + +`brace` - Mismatched braces + +`category` - Unsupported category + +`duplicate` - Multiple messages with the same name -`brace` - There are mismatched braces in the target message. +`extraneous` - Message does not exist in the source locale -`case` - There are unrecognized cases in the target message. +`missing` - Message missing from the target locale -`extraneous` - There is an extraneous message in the target locale. +`nbsp` - Message structure contains non-breaking space -`missing` - There is a message missing from the target locale. +`nest` - The nesting order of the target message does not match the source message -`nbsp` - There are invalid non-breaking spaces in the structure of the target message. +`option` - Unrecognized option -`nest` - The nesting order of the target message does not match the source message. +`option-missing` - Missing option used in the source -`other` - The target message is missing an `other` case +`other` - Missing "other" option -`parse` - The target message can not be parsed. +`parse` - Failed to parse message -`source` - There is an error in the source message. +`source` - Failed to parse source message ## Warnings -`categories` - The target message is missing plural categories used in the target locale +`category-missing` - Missing categories used by the target locale + +`nest-ideal` - A `select` is nested inside a `plural` or `selectordinal` + +`nest-order` - Nesting order does not match source + +`split` - Split by a complex argument + +`untranslated` - Message has not been translated + + +## Overrides + +You can mark individual messages as + +`mfv override fr option` + +A global list of overrides is pre-loaded: -`nest` - There is a `select` nested inside a `plural` or `selectordinal` in the target message. +v Expand Me -`split` - The target message is split by a non-argument. `plural`, `selectordinal`, and `select` cases should contain complete translations. +## v3 -`untranslated` - The target message has not been translated. +- Always throws on error. The `--throw-errors` option has been removed. +- The `locales` option now takes an array when in the config files +- New `format` subcommand rewrites messages to a standard format +- Issue types renamed: + - `case` -> `option` + - `nest` -> `nest-source` and `nest-ideal` + - `duplicate-keys` -> `duplicate` + - `plural-key` -> `category` + - `categories` -> `category-missing` + - `source-error` -> `source` +- New issue types + - `option-missing` diff --git a/bin/cli.js b/bin/cli.js index dd2edf5..93d3625 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -2,22 +2,24 @@ /* eslint-disable no-console */ +import { env } from 'node:process'; import { parseLocales, validateLocales } from '../src/validate.js'; import { readFile, readdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; import glob from 'glob'; import pkg from '../package.json' with { type: 'json' }; -import { program } from 'commander'; +import { program, Option } from 'commander'; import { getConfig, structureRegEx } from '../src/utils.js'; -let formatMessage; +let formatMessage, commandOpts; +const programArgs = {}; const { path, source: globalSource, locales: globalLocales, jsonObj: globalJsonObj -} = getConfig(); +} = await getConfig(env.PWD); program .version(pkg.version) @@ -33,7 +35,7 @@ program }); program - .command('build [locales]') + .command('build') .description('Build locale data for configured locales') .action(() => { program.build = true; @@ -72,8 +74,8 @@ program .description('Rename a message') .action((oldKey, newKey) => { program.rename = true; - program.oldKey = oldKey; - program.newKey = newKey; + programArgs.oldKey = oldKey; + programArgs.newKey = newKey; }); program @@ -84,17 +86,19 @@ program .option('-r, --remove', 'Remove cases for unsupported pural and selectordinal categories') .option('-d, --dedupe', 'Remove complex argument cases that duplicate the `other` case. Takes precedence over --add.') .option('-t, --trim', 'Trim whitespace from both ends of messages') - .option('-c, --collapse', 'Collapse repeating whitepace') + .addOption(new Option('-q, --quotes ', 'Replace quote characters with locale-appropriate characters').choices(['source', 'straight', 'both'])) .action(async function() { formatMessage = (await import('../src/format.js')).formatMessage; program.format = true; - const opts = this.opts(); + commandOpts = this.opts(); + /* program.newlines = opts.newlines; program.add = opts.add; program.remove = opts.remove; program.trim = opts.trim; - program.collapse = opts.collapse; + program.quotes = opts.quotes; program.dedupe = opts.dedupe + */ }); program @@ -104,14 +108,15 @@ program program.highlight = key; }); -program.parse(process.argv); +await program.parseAsync(process.argv); +const programOpts = program.opts(); if (program.build) { await import('../build-cldr-data.js'); process.exit(); } -const pathCombined = program.path || path; +const pathCombined = programOpts.path || path; if (!pathCombined) { console.error('Must provide a path to the locale files using either the -p option or a config file.'); process.exit(1); @@ -123,7 +128,7 @@ const noSource = () => { }; const localesPaths = glob.sync(pathCombined); -localesPaths.forEach(async localesPath => { +localesPaths.forEach(async (localesPath, idx) => { const absLocalesPath = `${process.cwd()}/${localesPath}`; @@ -134,11 +139,11 @@ localesPaths.forEach(async localesPath => { throw err; }); - const sourceLocale = program.sourceLocale || source || globalSource; - const allowedLocales = (program.locales?.replace(/\s/g, '').split(',') || configLocales || globalLocales || []).concat(sourceLocale); + const sourceLocale = programOpts.sourceLocale || source || globalSource; + const allowedLocales = programOpts.locales?.replace(/\s/g, '').split(',') || configLocales || globalLocales; const filteredFiles = !allowedLocales ? files.filter(file => !(/^\..*/g).test(file)) : - files.filter(file => allowedLocales.includes(file.split('.')[0])); + files.filter(file => allowedLocales.concat(sourceLocale).includes(file.split('.')[0])); const targetLocales = filteredFiles.map(file => file.split('.')[0]); if (program.removeExtraneous) { @@ -152,7 +157,7 @@ localesPaths.forEach(async localesPath => { } if (program.rename) { - console.log(`Renaming "${program.oldKey}" to "${program.newKey}" in:`, targetLocales.join(', ')); + console.log(`Renaming "${programArgs.oldKey}" to "${programArgs.newKey}" in:`, targetLocales.join(', ')); } if (program.format) { @@ -171,7 +176,7 @@ localesPaths.forEach(async localesPath => { }); if (!resources) return; - const useJSONObj = program.jsonObj || jsonObj || globalJsonObj; + const useJSONObj = programOpts.jsonObj || jsonObj || globalJsonObj; const locales = parseLocales(resources, useJSONObj); @@ -215,7 +220,6 @@ localesPaths.forEach(async localesPath => { if (!allowedLocales || allowedLocales.includes(locale)) { let localeContents = locales[locale].contents; - Object.values(locales[locale].parsed).forEach(t => { const source = sourceLocaleParsed[t.key]; @@ -224,20 +228,21 @@ localesPaths.forEach(async localesPath => { const baseTabs = t.match('^\n?(?\t*)').groups.tabs const newVal = formatMessage(t.val, { locale, - add: program.add, - remove: program.remove, - newlines: program.newlines, - dedupe: program.dedupe, - trim: program.trim, - collapse: program.collapse, + sourceLocale, + add: commandOpts.add, + remove: commandOpts.remove, + newlines: commandOpts.newlines, + dedupe: commandOpts.dedupe, + trim: commandOpts.trim, + collapse: commandOpts.collapse, baseTabs: baseTabs.length, key: t.key, source, target: t }); - const valQuote = program.newlines && newVal.includes('\n') ? '`' : t.valQuote; - const valSpace = program.newlines && newVal.includes('\n') ? `\n${baseTabs}` : t.valSpace; + const valQuote = commandOpts.newlines && newVal.includes('\n') ? '`' : t.valQuote; + const valSpace = commandOpts.newlines && newVal.includes('\n') ? `\n${baseTabs}` : t.valSpace; const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}${t.valQuote}${t.comma}${t.comment}`; const noo = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${valSpace}${valQuote}${newVal}${valQuote}${t.comma}${t.comment}`; @@ -262,11 +267,11 @@ localesPaths.forEach(async localesPath => { if (!allowedLocales || allowedLocales.includes(locale)) { const localeContents = locales[locale].contents; - const t = locales[locale].parsed[program.oldKey]; + const t = locales[locale].parsed[programArgs.oldKey]; if (localeContents.includes(t)) { const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; - const noo = `${t.keyQuote}${program.newKey}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; + const noo = `${t.keyQuote}${programArgs.newKey}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}`; count += 1; const newLocaleContents = localeContents.replace(old, noo); @@ -296,7 +301,7 @@ localesPaths.forEach(async localesPath => { if (!allowedLocales || allowedLocales.includes(locale.locale)) { console.log((idx > 0 ? '\n' : '') + chalk.underline(localePath)); - if (program.issues) { + if (programOpts.issues) { locale.report.totals.ignored = { warnings: 0, errors: 0 }; @@ -347,7 +352,7 @@ localesPaths.forEach(async localesPath => { translatorOutput[issue.key] = issue.source; } } - else if (!program.ignore || !program.ignore + else if (!programOpts.ignore || !programOpts.ignore .replace(' ','') .split(',') .includes(issue.type) @@ -405,7 +410,37 @@ localesPaths.forEach(async localesPath => { })); - if (output.some(locale => locale.report && locale.report.totals.errors - locale.report.totals.ignored.errors )) { + const totals = { + errors: 0, + warnings: 0, + ignored: 0 + }; + let error = false; + + output.forEach(locale => { + if (locale.report) { + totals.errors += locale.report.totals.errors + totals.warnings += locale.report.totals.warnings + totals.ignored += locale.report.totals.ignored + if (locale.report.totals.errors - locale.report.totals.ignored.errors) { + error = true; + } + } + }); + + if (totals.errors || totals.warnings) { + const color = totals.errors ? 'red' : 'yellow'; + const total = totals.errors + totals.warnings; + const ignored = totals.ignored.errors + totals.ignored.warnings; + const cliReport = chalk[color](`\u2716 ${total} issues (${totals.errors} errors, ${totals.warnings} warnings)${ignored ? chalk.grey(` - ${ignored} Ignored`) : ''}`); + console.log(`\n${chalk.bold('Totals')}`); + console.log(`\n${chalk.underline(localesPath)}`); + console.log(cliReport); + if (idx < localesPaths.length - 1) console.log(`\n${chalk.bold('---------------')}\n`); + return; + } + + if (error) { console.error('\nErrors were reported in at least one locale. See details above.'); return 1; } diff --git a/build-cldr-data.js b/build-locale-data.js similarity index 80% rename from build-cldr-data.js rename to build-locale-data.js index 16cf941..ff72dbd 100644 --- a/build-cldr-data.js +++ b/build-locale-data.js @@ -17,7 +17,7 @@ function getDelimiters(locale) { let cldr; await (async() => { - let contents = `import cldrData from './cldr-data-default.js';\nexport default cldrData;\n`; + let contents = `import localeData from './locale-data-default.js';\nexport default localeData;\n`; const config = await getConfig(); @@ -39,12 +39,17 @@ await (async() => { const data = {}; locales.forEach(locale => { - locale = locale.trim().toLowerCase(); + try { + locale = Intl.getCanonicalLocales(locale.trim().toLowerCase())[0]; + } catch(e) { + stderr.write(e.message); + process.exit(1); + } data[locale] = {}; data[locale].delimiters = getDelimiters(locale); }); - contents = `export default ${JSON.stringify(data, null, '\t')}\n`; + contents = `import defaultLocaleData from './locale-data-default.js';export default { ...${JSON.stringify(data, null, '\t')}\n`; } await writeFile(SAVE_PATH, contents); })(); diff --git a/package-lock.json b/package-lock.json index 7579cc4..b441a74 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@formatjs/icu-messageformat-parser": "^2.7.8", "chalk": "^4.1.0", - "commander": "^6.1.0", + "commander": "^12.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", "glob": "^7.1.6" @@ -1062,11 +1062,11 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/commander": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-6.1.0.tgz", - "integrity": "sha512-wl7PNrYWd2y5mp1OK/LhTlv8Ff4kQJQRXXAvF+uU/TPNiVJUxZLRYGj/B0y/lPGAVcSbJqH2Za/cvHmrPMC8mA==", + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", "engines": { - "node": ">= 6" + "node": ">=18" } }, "node_modules/concat-map": { diff --git a/package.json b/package.json index 8e10bc7..c1144e1 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "test:unit": "mocha 'test/**/*.test.js'", "prepack": "npm run build && npm t", "postinstall": "npm run build", - "build": "node build-cldr-data.js" + "build": "node build-locale-data.js" }, "bin": { "mfv": "bin/cli.js" @@ -25,14 +25,14 @@ }, "files": [ "/src", - "/build-cldr-data.js" + "/build-locale-data.js" ], "author": "Danny Gleckler ", "license": "MIT", "dependencies": { "@formatjs/icu-messageformat-parser": "^2.7.8", "chalk": "^4.1.0", - "commander": "^6.1.0", + "commander": "^12.1.0", "esm": "^3.2.25", "find-config": "^1.0.0", "glob": "^7.1.6" @@ -43,8 +43,8 @@ "@eslint/compat": "^1.1.1", "@eslint/eslintrc": "^3.1.0", "@eslint/js": "^9.10.0", - "cldr": "^7.5.0", "chai": "^5.1.1", + "cldr": "^7.5.0", "eslint": "^8.57.0", "globals": "^15.9.0", "mocha": "^10.7.3" diff --git a/src/cldr-data.js b/src/cldr-data.js deleted file mode 100644 index c7af759..0000000 --- a/src/cldr-data.js +++ /dev/null @@ -1,2 +0,0 @@ -import cldrData from './cldr-data-default.js'; -export default cldrData; diff --git a/src/format.js b/src/format.js index 2f71c61..5ef621d 100644 --- a/src/format.js +++ b/src/format.js @@ -1,7 +1,7 @@ import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.js'; import { parse } from '@formatjs/icu-messageformat-parser'; import { getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from './utils.js'; -import cldrData from './cldr-data.js'; +import localeData from './locale-data.js'; function expandASTHashes(ast, parentValue) { if (Array.isArray(ast)) { @@ -21,11 +21,13 @@ export function formatMessage(msg, options = {}) { let ast; try { - ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); + msg = options.quotes === 'straight' ? msg.replace(/'/g, "'''") : msg; + ast = parse(msg, { requiresOtherClause: false }); } catch(err) { try { + msg = options.quotes === 'straight' ? msg.replace(/'/g, "'''") : msg; const alteredMsg = msg.replace('\'{', '’{'); - ast = parse(msg.replace(/'/g, "'''"), { requiresOtherClause: false }); + ast = parse(alteredMsg, { requiresOtherClause: false }); msg = alteredMsg; } catch(err2) { if (err.location) { @@ -57,9 +59,10 @@ export function formatMessage(msg, options = {}) { remove: options.remove ?? false, dedupe: options.dedupe ?? false, trim: options.trim ?? false, - collapse: options.collapse ?? false, + quotes: options.quotes, locale: options.locale, + sourceLocale: options.sourceLocale, args: options.source ? [...new Set(options.source.match(/(?<=[{<])[^,{}<>]+(?=[}>,])/g))] : [] }, options.baseTabs); } @@ -78,49 +81,26 @@ function normalizeArgName(argName, availableArgs) { function printAST(ast, options, level = 0) { const { locale, + sourceLocale, + quotes, swapOne = new Set(), useNewlines = false, add = false, remove = false, dedupe = false, trim = false, - args = [] + args = [], } = options; const localeLower = locale.toLowerCase(); - const localeData = cldrData[locale] ?? cldrData[locale.split('-')[0]]; - const delimiters = localeData.delimiters; + const delimiters = (localeData[locale] ?? localeData[locale.split('-')[0]] ?? localeData['en'])?.delimiters; if (Array.isArray(ast)) { + //console.log(ast); const swapOneClone = new Set(swapOne); ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) - for (const k in delimiters) { - if (paddedQuoteLocales.includes(localeLower)) { - if (k.endsWith('Start')) { - delimiters[k] = delimiters[k].padEnd(2,'\u202f'); - } else if (k.endsWith('End')) { - delimiters[k] = delimiters[k].padStart(2,'\u202f'); - } - } - } - - let - quoteStart = delimiters.quotationStart, - quoteEnd = delimiters.quotationEnd, - singleQuoteStart = delimiters.alternateQuotationStart, - singleQuoteEnd = delimiters.alternateQuotationEnd; - - //if (1) { // todo: fromSource - if (localeLower.endsWith('-gb')) { - quoteStart = delimiters.alternateQuotationStart; - quoteEnd = delimiters.alternateQuotationEnd; - singleQuoteStart = delimiters.quotationStart; - singleQuoteEnd = delimiters.quotationEnd; - } - //} - - return ast + let msg = ast .filter((i, idx) => !trim || i.type !== 0 || (idx !== 0 && idx !== ast.length - 1) || i.value.trim()) // filter out leading and trailing whitespace .map((ast, idx, arr) => { let trim = options.trim; @@ -134,14 +114,101 @@ function printAST(ast, options, level = 0) { } } return printAST(ast, { ...options, swapOne: swapOneClone, trim }, level); - }).join('') - .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") - .replace(/(?<=\s)\\?'|^\\?'/g, singleQuoteStart) // opening ' - .replace(/(?<=\S)'(?=\S)/g, '’') // apostrophe - .replace(/\\?'/g, singleQuoteEnd) // closing ' - .replace(/(?<=\s(\u0648)?)\\?"|^\\?"/g, quoteStart) // opening " - .replace(/\\?"/g, quoteEnd) // closing " - .replace(/\|_escape_\|/g, "'"); + }).join(''); + + if (quotes) { + for (const k in delimiters) { + if (paddedQuoteLocales.includes(localeLower)) { + if (k.endsWith('Start')) { + delimiters[k] = delimiters[k].padEnd(2,'\u202f'); + } else if (k.endsWith('End')) { + delimiters[k] = delimiters[k].padStart(2,'\u202f'); + } + } + } + + let + quoteStart = delimiters.quotationStart, + quoteEnd = delimiters.quotationEnd, + altStart = delimiters.alternateQuotationStart, + altEnd = delimiters.alternateQuotationEnd; + + //if (1) { // todo: fromSource + if (localeLower.endsWith('-gb')) { + quoteStart = delimiters.alternateQuotationStart; + quoteEnd = delimiters.alternateQuotationEnd; + altStart = delimiters.quotationStart; + altEnd = delimiters.quotationEnd; + } + //} + + if (quotes === 'straight' || quotes === 'both') { + msg = msg + .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") + .replace(/(?<=\s)\\?'|^\\?'/g, altStart) // opening ' + .replace(/(?<=\S)'(?=\S)/g, '’') // apostrophe + .replace(/\\?'/g, altEnd) // closing ' + .replace(/(?<=\s(\u0648)?)\\?"|^\\?"/g, quoteStart) // opening " + .replace(/\\?"/g, quoteEnd) // closing " + .replace(/\|_escape_\|/g, "'"); + } + + if (quotes === 'source' || quotes === 'both') { + const { + quoteEnd: sourceQuoteEnd, + quoteStart: sourceQuoteStart, + altEnd: sourceAltEnd, + altStart: sourceAltStart + } = (locale => { + console.log(locale); + const delimiters = (localeData[locale] ?? localeData[locale.split('-')[0]] ?? localeData['en'])?.delimiters; + + for (const k in delimiters) { + if (paddedQuoteLocales.includes(locale.toLowerCase())) { + if (k.endsWith('Start')) { + delimiters[k] = delimiters[k].padEnd(2,'\u202f'); + } else if (k.endsWith('End')) { + delimiters[k] = delimiters[k].padStart(2,'\u202f'); + } + } + } + + let + quoteStart = delimiters.quotationStart, + quoteEnd = delimiters.quotationEnd, + altStart = delimiters.alternateQuotationStart, + altEnd = delimiters.alternateQuotationEnd; + + //if (1) { // todo: fromSource + if (locale.toLowerCase().endsWith('-gb')) { + quoteStart = delimiters.alternateQuotationStart; + quoteEnd = delimiters.alternateQuotationEnd; + altStart = delimiters.quotationStart; + altEnd = delimiters.quotationEnd; + } + + return { quoteStart, quoteEnd, altStart, altEnd }; + + })(sourceLocale); + + /* eslint-disable no-useless-escape */ + msg = msg + .replace(new RegExp(`''`, 'g'), '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") + .replace(new RegExp(`(?<=\s)\\\\?${sourceAltStart}|^\\\\?${sourceAltStart}`, 'g'), '|_altStart_|') // opening alt + .replace(new RegExp(`(?<=\\S)’(?=\\S)`, 'g'), '|_apostrophe_|') // apostrophe + .replace(new RegExp(`\\\\?${sourceAltEnd}`, 'g'), '|_altEnd_|') // closing alt + .replace(new RegExp(`(?<=\\s(\\u0648)?)\\\\?${sourceQuoteStart}|^\\\\?${sourceQuoteStart}`, 'g'), '|_quoteStart_|') // opening quote + .replace(new RegExp(`\\\\?${sourceQuoteEnd}`, 'g'), '|_quoteEnd_|') // closing quote + .replace(/\|_apostrophe_\|/g, "’") + .replace(/\|_escape_\|/g, "'") + .replace(/\|_quoteStart_\|/g, quoteStart) + .replace(/\|_quoteEnd_\|/g, quoteEnd) + .replace(/\|_altStart_\|/g, altStart) + .replace(/\|_altEnd_\|/g, altEnd); + /* eslint-enable no-useless-escape */ + } + } + return msg; } let text = ''; @@ -151,9 +218,20 @@ function printAST(ast, options, level = 0) { if (type === 0) { // straight text const value = swapOne.size ? ast.value.replace(/1/g, `{${[...swapOne].join('|')}}`) : ast.value; + if (swapOne.size) { + console.log('swapping...'); + console.log('was:', ast.value); + console.log('now:', value); + } + console.log(text); text += value[trim]?.() ?? value; } else if (type === 1) { // simple arg + if (ast.value.startsWith('disgw')) { + console.log(ast); + + console.log(args); + } text += `{${normalizeArgName(ast.value, args)}}`; } else if ([2, 3, 4].includes(type)) { // number, date, time @@ -170,6 +248,9 @@ function printAST(ast, options, level = 0) { text += `{${normalizeArgName(ast.value, args)}, ${typesText[type - 2]}${style}}`; } else if (type === 5) { // select + + + const optionsText = Object.entries(ast.options) .sort((a, b) => { return a[0] === 'other' ? 1 : (b[0] === 'other' ? -1 : 0); @@ -199,6 +280,10 @@ function printAST(ast, options, level = 0) { } else if (ast.options['=1'] && /(? !usedCats.includes(c)); + if (unusedCats.length === 1) { + const unrecognizedCats = usedCats.filter(c => !/^=\d+$/.test(c) && !supportedCats.includes(c)); + if (unrecognizedCats.length === 1) { + ast.options[unusedCats[0]] = ast.options[unrecognizedCats[0]]; + delete ast.options[unrecognizedCats[0]]; + } + } + remove && unsupportedCats.forEach(cat => { const currentKeys = Object.keys(ast.options); if (currentKeys.includes(cat)) { @@ -234,6 +330,12 @@ function printAST(ast, options, level = 0) { }).join('') + (useNewlines ? newline : ''); text += `{${normalizeArgName(ast.value, args)}, ${typeText},${offsetText}${optionsText}}`; + + if (swapOne.size) { + console.log('OPT TEXT'); + console.log(optionsText); + console.log(text); + } } else if (type === 7) { // # text += '#'; diff --git a/src/cldr-data-default.js b/src/locale-data-default.js similarity index 95% rename from src/cldr-data-default.js rename to src/locale-data-default.js index 4b2321e..2487a4d 100644 --- a/src/cldr-data-default.js +++ b/src/locale-data-default.js @@ -103,6 +103,14 @@ export default { "alternateQuotationEnd": "’" } }, + "mi": { + "delimiters": { + "quotationStart": "\"", + "quotationEnd": "\"", + "alternateQuotationStart": "\"", + "alternateQuotationEnd": "\"" + } + }, "nl": { "delimiters": { "quotationStart": "‘", diff --git a/src/utils.js b/src/utils.js index 48be62e..4c0fc98 100644 --- a/src/utils.js +++ b/src/utils.js @@ -4,6 +4,12 @@ export async function getConfig(cwd) { const configPath = findConfig('mfv.config.json', { cwd }); const config = configPath ? (await import(`file://${configPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; config.__configPath = configPath; + + if (config.locales !== undefined && !Array.isArray(config.locales)) { + console.error('locales config must be an array'); + process.exit(1); + } + return config; } diff --git a/src/validate.js b/src/validate.js index 23b9b42..5b03977 100644 --- a/src/validate.js +++ b/src/validate.js @@ -29,7 +29,7 @@ export function validateLocales({ locales, sourceLocale }, localesReporter) { reporter.config(targetMessages[key], sourceMessages[key]); if (!sourceMessage) { - reporter.error('extraneous', 'Message does not exist in the source file'); + reporter.error('extraneous', 'Message does not exist in the source locale'); } else { if (locales[targetLocale].duplicateKeys.has(key)) reporter.error('duplicate', `Multiple messages named "${key}"`); @@ -49,7 +49,7 @@ export function validateLocales({ locales, sourceLocale }, localesReporter) { if (missingKeys.length) { missingKeys.forEach((key) => { reporter.config(sourceMessages[key], sourceMessages[key]); - reporter.error('missing', 'Message missing from locale file') + reporter.error('missing', 'Message missing from the target locale') }) } @@ -68,7 +68,7 @@ function checkNbsp(message, reporter) { const nbspPos = structure.indexOf(String.fromCharCode(160)); if (nbspPos > -1) { - reporter.error('nbsp', `Message contains invalid non-breaking space at position ${nbspPos}`, { column: nbspPos }); + reporter.error('nbsp', `Message structure contains non-breaking space at position ${nbspPos}`, { column: nbspPos }); return true; } } @@ -118,7 +118,7 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so sourceTokens = parse(sourceMessage, { requiresOtherClause: false }); } catch(e) { - msgReporter.error('source-error', 'Failed to parse source message'); + msgReporter.error('source', 'Failed to parse source message'); return; } @@ -133,12 +133,12 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const supportedCats = getPluralCats(targetLocale, part.pluralType); const cats = Object.keys(part.options); const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); - if (missingCats.length) msgReporter.warning('categories-missing', `Missing ${missingCats.length === 1 ? 'category' : 'categories'} ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); + if (missingCats.length) msgReporter.warning('category-missing', `Missing ${missingCats.length === 1 ? 'category' : 'categories'} ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); const unsupportedCats = cats.filter(c => !/^=\d+$/.test(c) && !supportedCats.includes(c)); unsupportedCats.forEach(cat => { const column = part.options[cat].location.start.offset; - msgReporter.error('categories', `Unsupported category "${cat}". Locale supports "${supportedCats.join('", "')}", and explicit keys like "=0".`, { column }); + msgReporter.error('category', `Unsupported category "${cat}". Locale supports "${supportedCats.join('", "')}", and explicit keys like "=0".`, { column }); }); } @@ -175,7 +175,8 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const missingOptions = cleanSourceCases.filter(arg => !cleanTargetCases.includes(arg)); missingOptions.forEach(o => { - msgReporter.error('option-missing', `Missing option "${o.replace(/.+\|5undefined\|/, '')}"`); + o = o.replace(/.+\|5undefined\|/, ''); + o !== 'other' && msgReporter.error('option-missing', `Missing option "${o}"`); }); if (targetMap.nested && targetMap.cases.length === sourceMap.cases.length) { @@ -261,7 +262,8 @@ export function parseLocales(locales, useJSONObj) { } function _map(ast, partsMap = { nested: false, arguments: new Set(), cases: [], messageTokens: [] }) { - + //console.log(ast); + //console.log(JSON.stringify(partsMap, null, '\t')); ast.forEach(token => { if (typeof token !== 'string') { diff --git a/test/format.test.js b/test/format.test.js index ac93c38..1777fbb 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -17,9 +17,15 @@ describe('formatMessage', () => { { locale: 'fr', expected: `This isn’t «\u202fcorrect\u202f»` }, { locale: 'sv', expected: `This isn’t ”correct”` }, ].forEach(({ locale, expected }) => { - it(`should replace straight quotes with "${locale}" quotes with no options`, () => { + it(`should replace straight quotes with "${locale}" quotes when "quotes" option is "straight"`, () => { const message = `This isn't "correct"`; - const formatted = formatMessage(message, { locale }); + const formatted = formatMessage(message, { locale, quotes: 'straight' }); + expect(formatted).to.equal(expected); + }); + + it(`should replace source quotes with "${locale}" quotes when "quotes" option is "source"`, () => { + const message = `This isn’t “correct”`; + const formatted = formatMessage(message, { locale, sourceLocale: 'en', quotes: 'source' }); expect(formatted).to.equal(expected); }); }); @@ -105,13 +111,6 @@ describe('formatMessage', () => { expect(formatted).to.equal(expected); }); - it.skip(`should remove duplicate categories with no options`, () => { - const message = `{a, plural, one {value} other {value2}}`; - const expected = `{a, plural, one {value}}`; - const formatted = formatMessage(message, { locale }); - expect(formatted).to.equal(expected); - }); - it(`should remove categories that are copies of a lower-precedence key with the "dedupe" option`, () => { const message = `{a, plural, one {value} other {value}}`; const expected = `{a, plural, other {value}}`; @@ -178,4 +177,18 @@ describe('formatMessage', () => { expect(formatted).to.equal(expected); }); + it(`should replace bad plural categories with no options`, () => { + const message = `{a, plural, one {b} अन्य {च}}`; + const expected = `{a, plural, one {b} other {च}}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + + it(`should not replace bad plural categories when ambiguous`, () => { + const message = `{a, plural, अन्य {च}}`; + const expected = `{a, plural, अन्य {च}}`; + const formatted = formatMessage(message, { locale }); + expect(formatted).to.equal(expected); + }); + }); diff --git a/test/validate.test.js b/test/validate.test.js index 7295369..3adc015 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -38,9 +38,9 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Message has not been translated'); }); - // categories + // category - it('generates a "categories-missing" warning when a target message is missing supported plural categories', () => { + it('generates a "category-missing" warning when a target message is missing supported plural categories', () => { targetLocale = 'cy-gb'; const sourceMessage = '{a, plural, one {} other {}}'; const targetMessage = '{a, plural, one {} other {}}'; @@ -48,12 +48,12 @@ describe('validate', () => { reporter._config.locale = targetLocale; validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('categories-missing'); + expect(reporter.issues[0].type).to.equal('category-missing'); expect(reporter.issues[0].level).to.equal('warning'); expect(reporter.issues[0].msg).to.equal('Missing categories "zero", "two", "few", and "many"'); }); - it('generates "categories" errors when a target message uses unsupported plural categories', () => { + it('generates "category" errors when a target message uses unsupported plural categories', () => { const sourceMessage = '{a, plural, one {} other {}}'; const targetMessage = '{a, plural, =1 {} one {} two {} few {} many {} other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); @@ -61,27 +61,27 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(3); - expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].type).to.equal('category'); expect(reporter.issues[0].level).to.equal('error'); expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Locale supports "one", "other", and explicit keys like "=0".'); - expect(reporter.issues[1].type).to.equal('categories'); + expect(reporter.issues[1].type).to.equal('category'); expect(reporter.issues[1].level).to.equal('error'); expect(reporter.issues[1].msg).to.equal('Unsupported category "few". Locale supports "one", "other", and explicit keys like "=0".'); - expect(reporter.issues[2].type).to.equal('categories'); + expect(reporter.issues[2].type).to.equal('category'); expect(reporter.issues[2].level).to.equal('error'); expect(reporter.issues[2].msg).to.equal('Unsupported category "many". Locale supports "one", "other", and explicit keys like "=0".'); }); - it('generates "categories" errors when a target message uses nested unsupported plural categories', () => { + it('generates "category" errors when a target message uses nested unsupported plural categories', () => { const sourceMessage = '{a, plural, one {} other {}}'; const targetMessage = '{a, plural, one {{a, plural, one {} two {} other {}}} other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('categories'); + expect(reporter.issues[0].type).to.equal('category'); expect(reporter.issues[0].level).to.equal('error'); expect(reporter.issues[0].msg).to.equal('Unsupported category "two". Locale supports "one", "other", and explicit keys like "=0".'); }); @@ -162,7 +162,9 @@ describe('validate', () => { expect(reporter.issues[1].msg).to.equal('Unrecognized option "d". Argument uses "b" and "other".'); }); - it('generates an "option" error with missing options in select arguments', () => { + + + it('generates an "option-missing" error with missing options in select arguments', () => { const sourceMessage = '{a, select, b {} other {}}'; const targetMessage = '{a, select, other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); @@ -184,7 +186,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('nbsp'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Message contains invalid non-breaking space at position 11'); + expect(reporter.issues[0].msg).to.equal('Message structure contains non-breaking space at position 11'); }); // nest @@ -200,6 +202,15 @@ describe('validate', () => { expect(reporter.issues[0].msg).to.equal('Nesting order does not match source'); }); + it.skip('does not generate a "nest-order" error with non-nested messages', () => { + const sourceMessage = '{a, select, other {}} {b, select, other {}}'; + const targetMessage = '{b, select, other {}} {a, select, other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + console.log(reporter); + expect(reporter.issues.length).to.equal(0); + }); + it('generates a "nest-ideal" error with plural inside select', () => { const sourceMessage = '{a, plural, one {} other {{b, select, other {}}}}'; const targetMessage = '{a, plural, one {} other {{b, select, other {}}}}'; @@ -252,13 +263,13 @@ describe('validate', () => { // source - it('generates a "source-error" error an unparseable source message', () => { + it('generates a "source" error an unparseable source message', () => { const sourceMessage = '{a, select other {}}'; const targetMessage = '{a, select, other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); - expect(reporter.issues[0].type).to.equal('source-error'); + expect(reporter.issues[0].type).to.equal('source'); expect(reporter.issues[0].level).to.equal('error'); expect(reporter.issues[0].msg).to.equal('Failed to parse source message'); }); @@ -290,7 +301,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('extraneous'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Message does not exist in the source file'); + expect(reporter.issues[0].msg).to.equal('Message does not exist in the source locale'); }); // missing @@ -316,7 +327,7 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('missing'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Message missing from locale file'); + expect(reporter.issues[0].msg).to.equal('Message missing from the target locale'); }); // duplicate-keys From 59c4182a723c2a362b477dcc72a68b910d715791 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 5 Dec 2024 17:15:09 -0500 Subject: [PATCH 26/90] Handle fr-on data --- build-locale-data.js | 4 +++- src/format.js | 8 +++----- src/locale-data-default.js | 2 +- src/utils.js | 7 +++++++ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/build-locale-data.js b/build-locale-data.js index ff72dbd..7f29a42 100644 --- a/build-locale-data.js +++ b/build-locale-data.js @@ -4,6 +4,7 @@ import { formatList, getConfig } from './src/utils.js'; import { dirname, join, posix } from 'node:path'; const defaultLocales = ['ar', 'cy', 'da', 'de', 'en', 'en-gb', 'es', 'es-es', 'fr', 'fr-ca', 'fr-fr', 'haw', 'hi', 'ja', 'ko', 'mi', 'nl', 'pt', 'sv', 'tr', 'zh-cn', 'zh-tw']; +const defaultLocaleMap = { 'fr-on': 'fr-ca' }; const SAVE_PATH = posix.join(dirname(import.meta.url), 'src/cldr-data.js').replace(/file:(\/c:)?/i, ''); @@ -20,11 +21,12 @@ await (async() => { let contents = `import localeData from './locale-data-default.js';\nexport default localeData;\n`; const config = await getConfig(); + const localeMap = config.localeMap || defaultLocaleMap; let locales = env.MFV_LOCALES?.split(',') ?? config.locales; locales ??= config.path && (await readdir(join(dirname(config.__configPath), config.path)).catch(() => {}))?.map(f => f.split('.')[0]); locales ??= defaultLocales; - const nonDefaultLocales = locales?.filter(l => !defaultLocales.includes(l)); + const nonDefaultLocales = locales?.filter(l => !defaultLocales.includes(localeMap[l] ?? l)); if (nonDefaultLocales?.length) { let cldrImport; diff --git a/src/format.js b/src/format.js index 5ef621d..fc4280d 100644 --- a/src/format.js +++ b/src/format.js @@ -1,7 +1,6 @@ import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.js'; import { parse } from '@formatjs/icu-messageformat-parser'; -import { getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from './utils.js'; -import localeData from './locale-data.js'; +import { getLocaleData, getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from './utils.js'; function expandASTHashes(ast, parentValue) { if (Array.isArray(ast)) { @@ -93,7 +92,7 @@ function printAST(ast, options, level = 0) { } = options; const localeLower = locale.toLowerCase(); - const delimiters = (localeData[locale] ?? localeData[locale.split('-')[0]] ?? localeData['en'])?.delimiters; + const delimiters = getLocaleData(locale).delimiters; if (Array.isArray(ast)) { //console.log(ast); @@ -160,8 +159,7 @@ function printAST(ast, options, level = 0) { altEnd: sourceAltEnd, altStart: sourceAltStart } = (locale => { - console.log(locale); - const delimiters = (localeData[locale] ?? localeData[locale.split('-')[0]] ?? localeData['en'])?.delimiters; + const delimiters = getLocaleData(locale).delimiters; for (const k in delimiters) { if (paddedQuoteLocales.includes(locale.toLowerCase())) { diff --git a/src/locale-data-default.js b/src/locale-data-default.js index 2487a4d..1d6291a 100644 --- a/src/locale-data-default.js +++ b/src/locale-data-default.js @@ -63,7 +63,7 @@ export default { "alternateQuotationEnd": "’" } }, - "fr-fr": { + "fr-ca": { "delimiters": { "quotationStart": "«", "quotationEnd": "»", diff --git a/src/utils.js b/src/utils.js index 4c0fc98..e13ba50 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,4 +1,6 @@ +import { env } from 'node:process'; import findConfig from 'find-config'; +import localeData from './locale-data.js'; export async function getConfig(cwd) { const configPath = findConfig('mfv.config.json', { cwd }); @@ -17,6 +19,11 @@ export const sortedCats = ['zero', 'one', 'two', 'few', 'many', 'other']; export const paddedQuoteLocales = ['fr', 'fr-ca', 'fr-fr', 'fr-on', 'vi-vn']; export const structureRegEx = /(?<=\s*){(.|\n)*?[{}]|\s*}(.|\n)*?[{}]|[{#]|(\s*)}/g; +export async function getLocaleData(locale) { + locale = (await getConfig(env.PWD))?.localesMap[locale] || locale; + return (localeData[locale] ?? localeData[locale.split('-')[0]] ?? localeData['en']); +} + export function getPluralCats(locale, pluralType) { return new Intl.PluralRules(locale, { type: pluralType }).resolvedOptions().pluralCategories; } From ddbf1da416374080fceeb3bffc7730d269d99421 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 5 Dec 2024 20:16:15 -0500 Subject: [PATCH 27/90] Add starting locale-data file --- src/locale-data.js | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/locale-data.js diff --git a/src/locale-data.js b/src/locale-data.js new file mode 100644 index 0000000..bd023f7 --- /dev/null +++ b/src/locale-data.js @@ -0,0 +1,2 @@ +import cldrData from './locale-data-default.js'; +export default cldrData; From 3f28d0870c1f4933ebed83f9ba81a0765ce15ee8 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 5 Dec 2024 20:30:36 -0500 Subject: [PATCH 28/90] Fix build path --- bin/cli.js | 2 +- build-locale-data.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 93d3625..2e86403 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -112,7 +112,7 @@ await program.parseAsync(process.argv); const programOpts = program.opts(); if (program.build) { - await import('../build-cldr-data.js'); + await import('../build-locale-data.js'); process.exit(); } diff --git a/build-locale-data.js b/build-locale-data.js index 7f29a42..138bb1a 100644 --- a/build-locale-data.js +++ b/build-locale-data.js @@ -6,7 +6,7 @@ import { dirname, join, posix } from 'node:path'; const defaultLocales = ['ar', 'cy', 'da', 'de', 'en', 'en-gb', 'es', 'es-es', 'fr', 'fr-ca', 'fr-fr', 'haw', 'hi', 'ja', 'ko', 'mi', 'nl', 'pt', 'sv', 'tr', 'zh-cn', 'zh-tw']; const defaultLocaleMap = { 'fr-on': 'fr-ca' }; -const SAVE_PATH = posix.join(dirname(import.meta.url), 'src/cldr-data.js').replace(/file:(\/c:)?/i, ''); +const SAVE_PATH = posix.join(dirname(import.meta.url), 'src/locale-data.js').replace(/file:(\/c:)?/i, ''); function getDelimiters(locale) { try { From efa842d2ab59bde3d2cf3fb07969b9880b3beb80 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 9 Dec 2024 10:03:23 -0500 Subject: [PATCH 29/90] Fix locale data --- build-locale-data.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-locale-data.js b/build-locale-data.js index 138bb1a..860ef83 100644 --- a/build-locale-data.js +++ b/build-locale-data.js @@ -51,7 +51,7 @@ await (async() => { data[locale].delimiters = getDelimiters(locale); }); - contents = `import defaultLocaleData from './locale-data-default.js';export default { ...${JSON.stringify(data, null, '\t')}\n`; + contents = `import defaultLocaleData from './locale-data-default.js';\nexport default { ...defaultLocaleData, ...${JSON.stringify(data, null, '\t')}}\n`; } await writeFile(SAVE_PATH, contents); })(); From c667e8f57af6c069c90b71d4d9167e50b9a89386 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 9 Dec 2024 10:14:31 -0500 Subject: [PATCH 30/90] Fix localesMap check --- src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index e13ba50..a42225a 100644 --- a/src/utils.js +++ b/src/utils.js @@ -20,7 +20,7 @@ export const paddedQuoteLocales = ['fr', 'fr-ca', 'fr-fr', 'fr-on', 'vi-vn']; export const structureRegEx = /(?<=\s*){(.|\n)*?[{}]|\s*}(.|\n)*?[{}]|[{#]|(\s*)}/g; export async function getLocaleData(locale) { - locale = (await getConfig(env.PWD))?.localesMap[locale] || locale; + locale = (await getConfig(env.PWD))?.localesMap?.[locale] || locale; return (localeData[locale] ?? localeData[locale.split('-')[0]] ?? localeData['en']); } From 55a5d2a69f29734bd4bc3b8ec6181f67b8ad4466 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 11 Dec 2024 15:20:01 -0500 Subject: [PATCH 31/90] Clean up logging --- src/format.js | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/format.js b/src/format.js index fc4280d..c240ca5 100644 --- a/src/format.js +++ b/src/format.js @@ -95,7 +95,6 @@ function printAST(ast, options, level = 0) { const delimiters = getLocaleData(locale).delimiters; if (Array.isArray(ast)) { - //console.log(ast); const swapOneClone = new Set(swapOne); ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) @@ -216,20 +215,9 @@ function printAST(ast, options, level = 0) { if (type === 0) { // straight text const value = swapOne.size ? ast.value.replace(/1/g, `{${[...swapOne].join('|')}}`) : ast.value; - if (swapOne.size) { - console.log('swapping...'); - console.log('was:', ast.value); - console.log('now:', value); - } - console.log(text); text += value[trim]?.() ?? value; } else if (type === 1) { // simple arg - if (ast.value.startsWith('disgw')) { - console.log(ast); - - console.log(args); - } text += `{${normalizeArgName(ast.value, args)}}`; } else if ([2, 3, 4].includes(type)) { // number, date, time @@ -278,10 +266,6 @@ function printAST(ast, options, level = 0) { } else if (ast.options['=1'] && /(? Date: Thu, 12 Dec 2024 10:32:12 -0500 Subject: [PATCH 32/90] Add "near: ..." to parse errors --- src/validate.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/validate.js b/src/validate.js index 5b03977..f8d9915 100644 --- a/src/validate.js +++ b/src/validate.js @@ -104,7 +104,8 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so msgReporter.error('parse', `Expected "," but "${e.originalMessage.substr(e.location.start.column, 1)}" found`, { column: e.location.start.column - 1 }); } else { - msgReporter.error('parse', e.message, { column: e.location.start.column - 1 }); + const near = `at or near: ${targetMessage.slice(e.location.start.offset, Math.max(e.location.end.offset, e.location.start.offset + 4))}`; + msgReporter.error('parse', `${e.message} ${near}`, { column: e.location.start.column - 1 }); } } } From d1453a6512a928964ac6ad99954d315f484982ef Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 12 Dec 2024 13:52:59 -0500 Subject: [PATCH 33/90] Fix async looping issues --- bin/cli.js | 94 ++++++++++++++++++++++++++++++++++++--------------- src/format.js | 13 +++++-- 2 files changed, 77 insertions(+), 30 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 2e86403..5c5c335 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -128,7 +128,7 @@ const noSource = () => { }; const localesPaths = glob.sync(pathCombined); -localesPaths.forEach(async (localesPath, idx) => { +const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { const absLocalesPath = `${process.cwd()}/${localesPath}`; @@ -162,7 +162,7 @@ localesPaths.forEach(async (localesPath, idx) => { if (program.format) { if (!sourceLocale) noSource(); - console.log(`Formatting:`, targetLocales.join(', ')); + //console.log(`Formatting:`, targetLocales.join(', ')); } const resources = await Promise.all(filteredFiles.map(file => readFile(absLocalesPath + file, 'utf8'))) @@ -215,18 +215,22 @@ localesPaths.forEach(async (localesPath, idx) => { if (program.format) { let count = 0; + console.log(chalk.underline(localesPath)); + console.log(`Formatting:`, targetLocales.join(', ')); + const sourceLocaleParsed = locales[sourceLocale].parsed; - Object.keys(locales).forEach(async locale => { + + await Promise.all(Object.keys(locales).map(async locale => { if (!allowedLocales || allowedLocales.includes(locale)) { let localeContents = locales[locale].contents; - Object.values(locales[locale].parsed).forEach(t => { + await Promise.all(Object.values(locales[locale].parsed).map(async t => { const source = sourceLocaleParsed[t.key]; if (localeContents.includes(t)) { const baseTabs = t.match('^\n?(?\t*)').groups.tabs - const newVal = formatMessage(t.val, { + const newVal = await formatMessage(t.val, { locale, sourceLocale, add: commandOpts.add, @@ -235,6 +239,7 @@ localesPaths.forEach(async (localesPath, idx) => { dedupe: commandOpts.dedupe, trim: commandOpts.trim, collapse: commandOpts.collapse, + quotes: commandOpts.quotes, baseTabs: baseTabs.length, key: t.key, @@ -249,11 +254,11 @@ localesPaths.forEach(async (localesPath, idx) => { if (old !== noo) count += 1; localeContents = localeContents.replace(old, noo); } - }); + })); await writeFile(absLocalesPath + locales[locale].file, localeContents); } - }); + })); const cliReport = `\n ${chalk.green('\u2714')} Formatted ${count} messages`; console.log(cliReport); @@ -297,10 +302,19 @@ localesPaths.forEach(async (localesPath, idx) => { const translatorOutput = {}; await Promise.all(output.map(async(locale, idx) => { - const localePath = `${absLocalesPath}${locales[locale.locale].file}`; + const localeFile = `${locales[locale.locale].file}`; if (!allowedLocales || allowedLocales.includes(locale.locale)) { - console.log((idx > 0 ? '\n' : '') + chalk.underline(localePath)); + + if (program.sort || + program.removeExtraneous || + program.addMissing || + program.printMissing || + locale.report.totals.errors || + locale.report.totals.warnings) { + console.log((idx > 0 ? '\n' : '') + chalk.underline(`${localesPath}${localeFile}`)); + } + if (programOpts.issues) { locale.report.totals.ignored = { warnings: 0, errors: 0 }; @@ -372,7 +386,7 @@ localesPaths.forEach(async (localesPath, idx) => { } if (program.removeExtraneous || program.addMissing || program.sort) { - writeFile(localePath, locales[locale.locale].contents); + writeFile(localeFile, locales[locale.locale].contents); } } @@ -401,8 +415,7 @@ localesPaths.forEach(async (localesPath, idx) => { return; } else { - const cliReport = `\n ${chalk.green('\u2714')} Passed`; - console.log(cliReport); + // passed } } @@ -413,35 +426,62 @@ localesPaths.forEach(async (localesPath, idx) => { const totals = { errors: 0, warnings: 0, - ignored: 0 + ignored: { + errors: 0, + warnings: 0 + } }; - let error = false; output.forEach(locale => { if (locale.report) { - totals.errors += locale.report.totals.errors - totals.warnings += locale.report.totals.warnings - totals.ignored += locale.report.totals.ignored - if (locale.report.totals.errors - locale.report.totals.ignored.errors) { - error = true; - } + totals.errors += locale.report.totals.errors; + totals.warnings += locale.report.totals.warnings; + totals.ignored.errors += locale.report.totals.ignored.errors; + totals.ignored.warnings += locale.report.totals.ignored.warnings; } }); + if (idx < localesPaths.length - 1) console.log(`\n\n`); + + return totals; +})); + +if (results.filter(r => r).length) { + + const totals = { + errors: 0, + warnings: 0, + ignored: { + errors: 0, + warnings: 0 + } + }; + + results.forEach(result => { + if (result) { + totals.errors += result.errors; + totals.warnings += result.warnings; + totals.ignored.errors += result.ignored.errors; + totals.ignored.warnings += result.ignored.warnings; + } + }); + + console.log(chalk.bold(`\n\nTotal ${chalk.grey(chalk.grey(` ${pathCombined} `))}`)); + if (totals.errors || totals.warnings) { const color = totals.errors ? 'red' : 'yellow'; const total = totals.errors + totals.warnings; const ignored = totals.ignored.errors + totals.ignored.warnings; const cliReport = chalk[color](`\u2716 ${total} issues (${totals.errors} errors, ${totals.warnings} warnings)${ignored ? chalk.grey(` - ${ignored} Ignored`) : ''}`); - console.log(`\n${chalk.bold('Totals')}`); - console.log(`\n${chalk.underline(localesPath)}`); + console.log(chalk.bold(cliReport)); + } else { + const cliReport = `\n ${chalk.green('\u2714')} Passed`; console.log(cliReport); - if (idx < localesPaths.length - 1) console.log(`\n${chalk.bold('---------------')}\n`); - return; } - if (error) { + if (totals.errors - totals.ignored.errors) { console.error('\nErrors were reported in at least one locale. See details above.'); - return 1; } -}); + + process.exit(Number(Boolean(results.find(r => r === 1)))); +} diff --git a/src/format.js b/src/format.js index c240ca5..76b810f 100644 --- a/src/format.js +++ b/src/format.js @@ -16,7 +16,7 @@ function expandASTHashes(ast, parentValue) { } } -export function formatMessage(msg, options = {}) { +export async function formatMessage(msg, options = {}) { let ast; try { @@ -52,6 +52,11 @@ export function formatMessage(msg, options = {}) { console.log(e); } + const localeData = { + [options.locale]: await getLocaleData(options.locale), + [options.sourceLocale]: await getLocaleData(options.sourceLocale) + } + return printAST(ast, { useNewlines: options.newlines ?? msg.match(structureRegEx)?.join('').includes('\n'), add: options.add ?? false, @@ -62,6 +67,7 @@ export function formatMessage(msg, options = {}) { locale: options.locale, sourceLocale: options.sourceLocale, + localeData, args: options.source ? [...new Set(options.source.match(/(?<=[{<])[^,{}<>]+(?=[}>,])/g))] : [] }, options.baseTabs); } @@ -89,10 +95,11 @@ function printAST(ast, options, level = 0) { dedupe = false, trim = false, args = [], + localeData } = options; const localeLower = locale.toLowerCase(); - const delimiters = getLocaleData(locale).delimiters; + const { delimiters } = localeData[locale]; if (Array.isArray(ast)) { const swapOneClone = new Set(swapOne); @@ -158,7 +165,7 @@ function printAST(ast, options, level = 0) { altEnd: sourceAltEnd, altStart: sourceAltStart } = (locale => { - const delimiters = getLocaleData(locale).delimiters; + const { delimiters } = localeData[locale]; for (const k in delimiters) { if (paddedQuoteLocales.includes(locale.toLowerCase())) { From e1f588351fed9dda143983dcc292dff99cc0fbc2 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 13 Dec 2024 10:14:18 -0500 Subject: [PATCH 34/90] Fix escaping and quoting; Enable hash expansion --- bin/cli.js | 1 + src/format.js | 49 +++++++++++++++++++------- test/format.test.js | 85 +++++++++++++++++++++++++-------------------- 3 files changed, 86 insertions(+), 49 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 5c5c335..835cf51 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -240,6 +240,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { trim: commandOpts.trim, collapse: commandOpts.collapse, quotes: commandOpts.quotes, + expandHashes: true, baseTabs: baseTabs.length, key: t.key, diff --git a/src/format.js b/src/format.js index 76b810f..be8404a 100644 --- a/src/format.js +++ b/src/format.js @@ -52,10 +52,8 @@ export async function formatMessage(msg, options = {}) { console.log(e); } - const localeData = { - [options.locale]: await getLocaleData(options.locale), - [options.sourceLocale]: await getLocaleData(options.sourceLocale) - } + const localeData = { [options.locale]: await getLocaleData(options.locale) }; + if (options.sourceLocale) localeData[options.sourceLocale] = await getLocaleData(options.sourceLocale); return printAST(ast, { useNewlines: options.newlines ?? msg.match(structureRegEx)?.join('').includes('\n'), @@ -83,7 +81,7 @@ function normalizeArgName(argName, availableArgs) { return argName; } -function printAST(ast, options, level = 0) { +function printAST(ast, options, level = 0, parentValue) { const { locale, sourceLocale, @@ -95,12 +93,13 @@ function printAST(ast, options, level = 0) { dedupe = false, trim = false, args = [], - localeData + localeData, + isFirst = true, + isLast = true } = options; const localeLower = locale.toLowerCase(); - const { delimiters } = localeData[locale]; - + //console.log(ast); if (Array.isArray(ast)) { const swapOneClone = new Set(swapOne); ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) @@ -118,10 +117,18 @@ function printAST(ast, options, level = 0) { trim = 'trimEnd'; } } - return printAST(ast, { ...options, swapOne: swapOneClone, trim }, level); + return printAST(ast, { ...options, + isFirst: !idx, + isLast: idx === arr.length - 1, + swapOne: swapOneClone, + trim + }, + level); }).join(''); if (quotes) { + const { delimiters } = localeData[locale]; + for (const k in delimiters) { if (paddedQuoteLocales.includes(localeLower)) { if (k.endsWith('Start')) { @@ -148,6 +155,7 @@ function printAST(ast, options, level = 0) { //} if (quotes === 'straight' || quotes === 'both') { + //console.log(msg); msg = msg .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") .replace(/(?<=\s)\\?'|^\\?'/g, altStart) // opening ' @@ -156,6 +164,7 @@ function printAST(ast, options, level = 0) { .replace(/(?<=\s(\u0648)?)\\?"|^\\?"/g, quoteStart) // opening " .replace(/\\?"/g, quoteEnd) // closing " .replace(/\|_escape_\|/g, "'"); + //console.log(msg); } if (quotes === 'source' || quotes === 'both') { @@ -221,8 +230,24 @@ function printAST(ast, options, level = 0) { const type = ast.type; if (type === 0) { // straight text - const value = swapOne.size ? ast.value.replace(/1/g, `{${[...swapOne].join('|')}}`) : ast.value; - text += value[trim]?.() ?? value; + let escaped = ast.value; + //console.log(escaped); + // If this literal starts with a ' and its not the 1st node, this means the node before it is non-literal + // and the `'` needs to be unescaped + if (!isFirst && escaped[0] === "'") { + escaped = "''".concat(escaped.slice(1)); + } + // Same logic but for last el + if (!isLast && escaped[escaped.length - 1] === "'") { + escaped = "".concat(escaped.slice(0, escaped.length - 1), "''"); + } + //console.log(escaped); + escaped = escaped.replace(/([{}](?:.*[{}])?)/su, "'$1'"); + //console.log(escaped); + escaped = parentValue ? escaped.replace('#', "'#'") : escaped; + + escaped = swapOne.size ? escaped.replace(/1/g, `{${[...swapOne].join('|')}}`) : escaped; + text += escaped[trim]?.() ?? escaped; } else if (type === 1) { // simple arg text += `{${normalizeArgName(ast.value, args)}}`; @@ -315,7 +340,7 @@ function printAST(ast, options, level = 0) { } return sortedCats.indexOf(a[0]) > sortedCats.indexOf(b[0]) ? 1 : -1; }).map(([opt, { value }]) => { - return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, swapOne }, level + 1)}}`; + return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, swapOne }, level + 1, ast.value)}}`; }).join('') + (useNewlines ? newline : ''); text += `{${normalizeArgName(ast.value, args)}, ${typeText},${offsetText}${optionsText}}`; diff --git a/test/format.test.js b/test/format.test.js index 1777fbb..a8ff1c7 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { formatMessage } from '../src/format.js'; -describe('formatMessage', () => { +describe('await formatMessage', () => { let locale; beforeEach(() => { @@ -17,19 +17,30 @@ describe('formatMessage', () => { { locale: 'fr', expected: `This isn’t «\u202fcorrect\u202f»` }, { locale: 'sv', expected: `This isn’t ”correct”` }, ].forEach(({ locale, expected }) => { - it(`should replace straight quotes with "${locale}" quotes when "quotes" option is "straight"`, () => { + it(`should replace straight quotes with "${locale}" quotes when "quotes" option is "straight"`, async() => { const message = `This isn't "correct"`; - const formatted = formatMessage(message, { locale, quotes: 'straight' }); + const formatted = await formatMessage(message, { locale, sourceLocale: 'en', quotes: 'straight' }); expect(formatted).to.equal(expected); }); - it(`should replace source quotes with "${locale}" quotes when "quotes" option is "source"`, () => { + it(`should replace source quotes with "${locale}" quotes when "quotes" option is "source"`, async() => { const message = `This isn’t “correct”`; - const formatted = formatMessage(message, { locale, sourceLocale: 'en', quotes: 'source' }); + const formatted = await formatMessage(message, { locale, sourceLocale: 'en', quotes: 'source' }); expect(formatted).to.equal(expected); }); }); + [ + { condition: 'no options are set', options: {} }, + { condition: 'the "quotes" option is "straight"', options: { quotes: 'straight' } } + ].forEach(({ condition, options }) => { + it(`should preserve escapes when ${condition}`, async() => { + const message = `An '{escaped}' argument`; + const formatted = await formatMessage(message, { locale: 'en', sourceLocale: 'en', ...options }); + expect(formatted).to.equal(message); + }); + }); + [ { locale: 'ar', expected: `{a, plural, @@ -77,7 +88,7 @@ describe('formatMessage', () => { other {} }` }, ].forEach(({ locale, expected }) => { - it(`should remove plural and selectordinal categories that are unsupported in "${locale}" with the "remove" option`, () => { + it(`should remove plural and selectordinal categories that are unsupported in "${locale}" with the "remove" option`, async() => { const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}} @@ -86,108 +97,108 @@ describe('formatMessage', () => { many {} other {} }`; - const formatted = formatMessage(message, { locale, remove: true }); + const formatted = await formatMessage(message, { locale, remove: true }); expect(formatted).to.equal(expected); }); }); - it(`should not remove plural and selectordinal categories that are unsupported without the "remove" option`, () => { + it(`should not remove plural and selectordinal categories that are unsupported without the "remove" option`, async() => { const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; - const formatted = formatMessage(message, { locale }); + const formatted = await formatMessage(message, { locale }); expect(formatted).to.equal(message); }); - it(`should insert newslines and tabs with the "newlines" option`, () => { + it(`should insert newslines and tabs with the "newlines" option`, async() => { const message = `{a, plural, one {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; - const formatted = formatMessage(message, { locale, newlines: true }); + const formatted = await formatMessage(message, { locale, newlines: true }); expect(formatted).to.equal(expected); }); - it(`should insert newslines and tabs if the message structure already contains newlines with no option`, () => { + it(`should insert newslines and tabs if the message structure already contains newlines with no option`, async() => { const message = `{a, plural,\none {{b, selectordinal, one {} two {} few {} many {} other {}}}}`; const expected = `{a, plural,\n\tone {{b, selectordinal,\n\t\tone {}\n\t\ttwo {}\n\t\tfew {}\n\t\tmany {}\n\t\tother {}\n\t}}\n}`; - const formatted = formatMessage(message, { locale }); + const formatted = await formatMessage(message, { locale }); expect(formatted).to.equal(expected); }); - it(`should remove categories that are copies of a lower-precedence key with the "dedupe" option`, () => { + it(`should remove categories that are copies of a lower-precedence key with the "dedupe" option`, async() => { const message = `{a, plural, one {value} other {value}}`; const expected = `{a, plural, other {value}}`; - const formatted = formatMessage(message, { locale, dedupe: true }); + const formatted = await formatMessage(message, { locale, dedupe: true }); expect(formatted).to.equal(expected); }); - it(`should convert "=1" keys to "one" when it contains a literal "1"`, () => { + it(`should convert "=1" keys to "one" when it contains a literal "1"`, async() => { const message = `{a, plural, =1 {value 1}}`; const expected = `{a, plural, one {value {a}}}`; - const formatted = formatMessage(message, { locale, dedupe: true }); + const formatted = await formatMessage(message, { locale, dedupe: true }); expect(formatted).to.equal(expected); }); - it(`should remove "=1" cases when they can be converted to a duplicate case with the "dedupe" option`, () => { + it(`should remove "=1" cases when they can be converted to a duplicate case with the "dedupe" option`, async() => { const message = `{a, plural, =1 {value 1} other {value {a}}}`; const expected = `{a, plural, other {value {a}}}`; - const formatted = formatMessage(message, { locale, dedupe: true }); + const formatted = await formatMessage(message, { locale, dedupe: true }); expect(formatted).to.equal(expected); }); - it(`should remove "=1" cases when they can be converted to unsupported "one" cases with the "remove" option`, () => { + it(`should remove "=1" cases when they can be converted to unsupported "one" cases with the "remove" option`, async() => { locale = 'ja'; const message = `{a, plural, =1 {value 1} other {value {a}}}`; const expected = `{a, plural, other {value {a}}}`; - const formatted = formatMessage(message, { locale, remove: true }); + const formatted = await formatMessage(message, { locale, remove: true }); expect(formatted).to.equal(expected); }); - it(`should convert unsupported cases to "other" if there are no other cases`, () => { + it(`should convert unsupported cases to "other" if there are no other cases`, async() => { locale = 'ja'; const message = `{a, plural, two {value {a}}}`; const expected = `{a, plural, other {value {a}}}`; - const formatted = formatMessage(message, { locale, remove: true }); + const formatted = await formatMessage(message, { locale, remove: true }); expect(formatted).to.equal(expected); }); - it(`should convert "=1" keys to "other" keys when they can be converted to unsupported "one" cases and there are no other keys with the "remove" option`, () => { + it(`should convert "=1" keys to "other" keys when they can be converted to unsupported "one" cases and there are no other keys with the "remove" option`, async() => { locale = 'ja'; - const message = `{a, plural, =1 {value 1}}}`; - const expected = `{a, plural, other {value {a}}}}`; - const formatted = formatMessage(message, { locale, remove: true }); + const message = `{a, plural, =1 {value 1}}`; + const expected = `{a, plural, other {value {a}}}`; + const formatted = await formatMessage(message, { locale, remove: true }); expect(formatted).to.equal(expected); }); - it('should hoist complex selectors to the outside and nest appropriately with no options', () => { + it('should hoist complex selectors to the outside and nest appropriately with no options', async() => { const message = `\t{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; const expected = `{a, plural, =1 {{b, plural, =1 {\ta cat and a dog!} other {\ta cat and {b} dogs!}}} other {{b, plural, =1 {\t{a} cats and a dog!} other {\t{a} cats and {b} dogs!}}}}`; - const formatted = formatMessage(message, { locale }); + const formatted = await formatMessage(message, { locale }); expect(formatted).to.equal(expected); }); - it(`should trim whitespace with the "trim" option`, () => { + it(`should trim whitespace with the "trim" option`, async() => { const message = `\n{a, plural, other { value }}\t`; const expected = `{a, plural, other {value}}`; - const formatted = formatMessage(message, { locale, trim: true }); + const formatted = await formatMessage(message, { locale, trim: true }); expect(formatted).to.equal(expected); }); - it(`should not trim internal whitespace with the "trim" option`, () => { + it(`should not trim internal whitespace with the "trim" option`, async() => { const message = `\n{a, plural, other { value {value2} value3 }}`; const expected = `{a, plural, other {value {value2} value3}}`; - const formatted = formatMessage(message, { locale, trim: true }); + const formatted = await formatMessage(message, { locale, trim: true }); expect(formatted).to.equal(expected); }); - it(`should replace bad plural categories with no options`, () => { + it(`should replace bad plural categories with no options`, async() => { const message = `{a, plural, one {b} अन्य {च}}`; const expected = `{a, plural, one {b} other {च}}`; - const formatted = formatMessage(message, { locale }); + const formatted = await formatMessage(message, { locale }); expect(formatted).to.equal(expected); }); - it(`should not replace bad plural categories when ambiguous`, () => { + it(`should not replace bad plural categories when ambiguous`, async() => { const message = `{a, plural, अन्य {च}}`; const expected = `{a, plural, अन्य {च}}`; - const formatted = formatMessage(message, { locale }); + const formatted = await formatMessage(message, { locale }); expect(formatted).to.equal(expected); }); From 044e75b6364f4380ed9590b4aae3b82e90b59d8b Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 13 Dec 2024 11:15:35 -0500 Subject: [PATCH 35/90] Actually fix escaping + cleanup --- src/format.js | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/format.js b/src/format.js index be8404a..89d4212 100644 --- a/src/format.js +++ b/src/format.js @@ -99,7 +99,7 @@ function printAST(ast, options, level = 0, parentValue) { } = options; const localeLower = locale.toLowerCase(); - //console.log(ast); + if (Array.isArray(ast)) { const swapOneClone = new Set(swapOne); ast.forEach(a => a.type === 1 && swapOneClone.delete(a.value)) @@ -155,16 +155,13 @@ function printAST(ast, options, level = 0, parentValue) { //} if (quotes === 'straight' || quotes === 'both') { - //console.log(msg); msg = msg .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") .replace(/(?<=\s)\\?'|^\\?'/g, altStart) // opening ' .replace(/(?<=\S)'(?=\S)/g, '’') // apostrophe .replace(/\\?'/g, altEnd) // closing ' .replace(/(?<=\s(\u0648)?)\\?"|^\\?"/g, quoteStart) // opening " - .replace(/\\?"/g, quoteEnd) // closing " - .replace(/\|_escape_\|/g, "'"); - //console.log(msg); + .replace(/\\?"/g, quoteEnd); // closing " } if (quotes === 'source' || quotes === 'both') { @@ -213,7 +210,6 @@ function printAST(ast, options, level = 0, parentValue) { .replace(new RegExp(`(?<=\\s(\\u0648)?)\\\\?${sourceQuoteStart}|^\\\\?${sourceQuoteStart}`, 'g'), '|_quoteStart_|') // opening quote .replace(new RegExp(`\\\\?${sourceQuoteEnd}`, 'g'), '|_quoteEnd_|') // closing quote .replace(/\|_apostrophe_\|/g, "’") - .replace(/\|_escape_\|/g, "'") .replace(/\|_quoteStart_\|/g, quoteStart) .replace(/\|_quoteEnd_\|/g, quoteEnd) .replace(/\|_altStart_\|/g, altStart) @@ -221,7 +217,7 @@ function printAST(ast, options, level = 0, parentValue) { /* eslint-enable no-useless-escape */ } } - return msg; + return msg.replace(/\|_escape_\|/g, "'"); } let text = ''; @@ -231,7 +227,6 @@ function printAST(ast, options, level = 0, parentValue) { if (type === 0) { // straight text let escaped = ast.value; - //console.log(escaped); // If this literal starts with a ' and its not the 1st node, this means the node before it is non-literal // and the `'` needs to be unescaped if (!isFirst && escaped[0] === "'") { @@ -241,10 +236,8 @@ function printAST(ast, options, level = 0, parentValue) { if (!isLast && escaped[escaped.length - 1] === "'") { escaped = "".concat(escaped.slice(0, escaped.length - 1), "''"); } - //console.log(escaped); - escaped = escaped.replace(/([{}](?:.*[{}])?)/su, "'$1'"); - //console.log(escaped); - escaped = parentValue ? escaped.replace('#', "'#'") : escaped; + escaped = escaped.replace(/'([{}](?:.*[{}])?)'/gsu, "|_escape_|$1|_escape_|"); + escaped = parentValue ? escaped.replace("'#'", "|_escape_|#|_escape_|") : escaped; escaped = swapOne.size ? escaped.replace(/1/g, `{${[...swapOne].join('|')}}`) : escaped; text += escaped[trim]?.() ?? escaped; From cc4f52ed52af64ef302b534c906940ad2d73f224 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 13 Dec 2024 12:02:53 -0500 Subject: [PATCH 36/90] Improve arg finding --- src/format.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/format.js b/src/format.js index 89d4212..1692a13 100644 --- a/src/format.js +++ b/src/format.js @@ -66,7 +66,7 @@ export async function formatMessage(msg, options = {}) { locale: options.locale, sourceLocale: options.sourceLocale, localeData, - args: options.source ? [...new Set(options.source.match(/(?<=[{<])[^,{}<>]+(?=[}>,])/g))] : [] + args: options.source ? [...new Set(options.source.match(/(?<=[{<]\s*)[^,{}<>\s]+(?=\s*[}>,])/g))] : [] }, options.baseTabs); } From f624e9c9b4cfae31f4a6f004eefc93c6fccd6101 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 13 Dec 2024 12:06:02 -0500 Subject: [PATCH 37/90] Final escaping fix (hopefully) --- src/format.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/format.js b/src/format.js index 1692a13..3b07c37 100644 --- a/src/format.js +++ b/src/format.js @@ -16,15 +16,18 @@ function expandASTHashes(ast, parentValue) { } } +function escape(msg) { + return msg.replace(/'([{}](?:.*[{}])?)'/gsu, "'''$1'''"); +} + export async function formatMessage(msg, options = {}) { let ast; try { - msg = options.quotes === 'straight' ? msg.replace(/'/g, "'''") : msg; + msg = options.quotes === 'straight' ? msg.replace(/'/g, "'''") : escape(msg); ast = parse(msg, { requiresOtherClause: false }); } catch(err) { try { - msg = options.quotes === 'straight' ? msg.replace(/'/g, "'''") : msg; const alteredMsg = msg.replace('\'{', '’{'); ast = parse(alteredMsg, { requiresOtherClause: false }); msg = alteredMsg; From fac81e053f98403842458637be38c28df92b7f46 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 13 Dec 2024 14:19:25 -0500 Subject: [PATCH 38/90] Improve argument correction --- src/format.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/format.js b/src/format.js index 3b07c37..9288ee2 100644 --- a/src/format.js +++ b/src/format.js @@ -69,7 +69,7 @@ export async function formatMessage(msg, options = {}) { locale: options.locale, sourceLocale: options.sourceLocale, localeData, - args: options.source ? [...new Set(options.source.match(/(?<=[{<]\s*)[^,{}<>\s]+(?=\s*[}>,])/g))] : [] + args: options.source ? [...options.source.match(/(?<=[{<]\s*)[^,{}<>\s]+(?=\s*[}>,])/g)] : [] }, options.baseTabs); } @@ -78,9 +78,15 @@ function normalizeArgName(argName, availableArgs) { if (availableArgs.length === 1) { return availableArgs[0]; } else { - return availableArgs.find(a => a.toLowerCase() === argName.toLowerCase()) ?? argName; + const caseMatch = availableArgs.find(a => a.toLowerCase() === argName.toLowerCase()); + if (caseMatch) { + availableArgs.splice(availableArgs.indexOf(caseMatch), 1); + return caseMatch; + } + return availableArgs.join('|'); } } + availableArgs.splice(availableArgs.indexOf(argName), 1); return argName; } @@ -263,7 +269,7 @@ function printAST(ast, options, level = 0, parentValue) { } else if (type === 5) { // select - + const argName = normalizeArgName(ast.value, args); const optionsText = Object.entries(ast.options) .sort((a, b) => { @@ -273,9 +279,10 @@ function printAST(ast, options, level = 0, parentValue) { return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, options, level + 1)}}`; }).join('') + (useNewlines ? newline : ''); - text += `{${normalizeArgName(ast.value, args)}, select,${optionsText}}`; + text += `{${argName}, select,${optionsText}}`; } else if (type === 6) { // plural, selectordinal + const argName = normalizeArgName(ast.value, args); const supportedCats = getPluralCats(locale, ast.pluralType); const unsupportedCats = [ ...Object.keys(ast.options).filter(o => !/^=\d+$/.test(o)) , ...sortedCats].filter(cat => !supportedCats.includes(cat)); if (add) { @@ -339,7 +346,7 @@ function printAST(ast, options, level = 0, parentValue) { return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, swapOne }, level + 1, ast.value)}}`; }).join('') + (useNewlines ? newline : ''); - text += `{${normalizeArgName(ast.value, args)}, ${typeText},${offsetText}${optionsText}}`; + text += `{${argName}, ${typeText},${offsetText}${optionsText}}`; } else if (type === 7) { // # text += '#'; From 534b26d0dd662310019634ad35ceada585c09abf Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 13 Dec 2024 14:46:55 -0500 Subject: [PATCH 39/90] Add arg correction option --- bin/cli.js | 3 ++- src/format.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 835cf51..9215ed7 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -86,6 +86,7 @@ program .option('-r, --remove', 'Remove cases for unsupported pural and selectordinal categories') .option('-d, --dedupe', 'Remove complex argument cases that duplicate the `other` case. Takes precedence over --add.') .option('-t, --trim', 'Trim whitespace from both ends of messages') + .option('-c, --correct', 'Attempt to correct argument names. A source locale must be provided.') .addOption(new Option('-q, --quotes ', 'Replace quote characters with locale-appropriate characters').choices(['source', 'straight', 'both'])) .action(async function() { formatMessage = (await import('../src/format.js')).formatMessage; @@ -226,7 +227,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { let localeContents = locales[locale].contents; await Promise.all(Object.values(locales[locale].parsed).map(async t => { - const source = sourceLocaleParsed[t.key]; + const source = commandOpts.correct ? sourceLocaleParsed[t.key] : null; if (localeContents.includes(t)) { const baseTabs = t.match('^\n?(?\t*)').groups.tabs diff --git a/src/format.js b/src/format.js index 9288ee2..2e0958a 100644 --- a/src/format.js +++ b/src/format.js @@ -74,7 +74,7 @@ export async function formatMessage(msg, options = {}) { } function normalizeArgName(argName, availableArgs) { - if (!availableArgs.includes(argName)) { + if (availableArgs.length && !availableArgs.includes(argName)) { if (availableArgs.length === 1) { return availableArgs[0]; } else { From 88838dfb6d5b73dc77fe598a125514fdc43401e6 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 13 Dec 2024 15:06:35 -0500 Subject: [PATCH 40/90] Indicate path glob support --- bin/cli.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cli.js b/bin/cli.js index 9215ed7..33b160b 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -26,7 +26,7 @@ program .option('--no-issues', 'Don\'t output issues') .option('-i, --ignore ', 'Ignore these comma-separated issue types') .option('-l, --locales ', 'Process only these comma-separated locales') - .option('-p, --path ', 'Path to a directory containing locale files') + .option('-p, --path ', 'Glob path to a directory containing locale files') .option('-s, --source-locale ', 'The locale to use as the source') .option('--json-obj', 'Indicate that the files to be parsed are JSON files with keys that have objects for values') .command('validate', { isDefault: true, hidden: true }) From 1ff2a65601fa75bfc7c42eb2cf069b554e70f90e Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 13 Dec 2024 19:01:55 -0500 Subject: [PATCH 41/90] Fix escaped arg regex --- src/format.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/format.js b/src/format.js index 2e0958a..23c837a 100644 --- a/src/format.js +++ b/src/format.js @@ -17,7 +17,7 @@ function expandASTHashes(ast, parentValue) { } function escape(msg) { - return msg.replace(/'([{}](?:.*[{}])?)'/gsu, "'''$1'''"); + return msg.replace(/'([{}](?:.*?[{}])?)'/gsu, "'''$1'''"); } export async function formatMessage(msg, options = {}) { From 28f332fad080c4e0527925102836f7755bc2b239 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 28 Jan 2025 12:42:44 -0500 Subject: [PATCH 42/90] Improve argument correction --- src/format.js | 62 ++++++++++++++++++++++++++++++++-------------- src/locale-data.js | 4 +-- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/format.js b/src/format.js index 23c837a..0987349 100644 --- a/src/format.js +++ b/src/format.js @@ -2,6 +2,15 @@ import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.j import { parse } from '@formatjs/icu-messageformat-parser'; import { getLocaleData, getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from './utils.js'; +function getArgs(asts, types = [1,2,3,4,5,6,8], args = []) { + asts.forEach(ast => { + if (types.includes(ast.type)) args.push([ast.value, ast.type]); + if (ast.options) Object.values(ast.options).map(({ value: asts }) => getArgs(asts, types, args)); + if (ast.children) args.concat(getArgs(ast.children, types, args)); + }) + return args; +} + function expandASTHashes(ast, parentValue) { if (Array.isArray(ast)) { ast.map(ast => expandASTHashes(ast, parentValue)); @@ -58,6 +67,15 @@ export async function formatMessage(msg, options = {}) { const localeData = { [options.locale]: await getLocaleData(options.locale) }; if (options.sourceLocale) localeData[options.sourceLocale] = await getLocaleData(options.sourceLocale); + let args = []; + if (options.source) { + try { + args = getArgs(parse(options.source, { requiresOtherClause: false })); + } catch(e) { + console.warn('WARN Source error:', options.key); + } + } + return printAST(ast, { useNewlines: options.newlines ?? msg.match(structureRegEx)?.join('').includes('\n'), add: options.add ?? false, @@ -69,25 +87,31 @@ export async function formatMessage(msg, options = {}) { locale: options.locale, sourceLocale: options.sourceLocale, localeData, - args: options.source ? [...options.source.match(/(?<=[{<]\s*)[^,{}<>\s]+(?=\s*[}>,])/g)] : [] + args }, options.baseTabs); } -function normalizeArgName(argName, availableArgs) { - if (availableArgs.length && !availableArgs.includes(argName)) { - if (availableArgs.length === 1) { - return availableArgs[0]; +function normalizeArgName(name, type, availableArgs) { + if (availableArgs.length) { + const idx = availableArgs.findIndex(([n, t]) => n === name && t === type); + if (idx > -1) { + availableArgs.splice(idx, 1); + return name; + } + else if (availableArgs.length === 1) { + return availableArgs.pop()[0]; } else { - const caseMatch = availableArgs.find(a => a.toLowerCase() === argName.toLowerCase()); - if (caseMatch) { - availableArgs.splice(availableArgs.indexOf(caseMatch), 1); + const idx = availableArgs.findIndex(([n, t]) => t === type && n.toLowerCase() === name.toLowerCase()); + if (idx > -1) { + const caseMatch = availableArgs.splice(idx, 1)[0]; return caseMatch; } - return availableArgs.join('|'); + return [...new Set(availableArgs + .reduce((acc, [n, t]) => t === type && acc.push(n) && acc || acc, []) + )].join('|'); } } - availableArgs.splice(availableArgs.indexOf(argName), 1); - return argName; + return name; } function printAST(ast, options, level = 0, parentValue) { @@ -252,7 +276,7 @@ function printAST(ast, options, level = 0, parentValue) { text += escaped[trim]?.() ?? escaped; } else if (type === 1) { // simple arg - text += `{${normalizeArgName(ast.value, args)}}`; + text += `{${normalizeArgName(ast.value, type, args)}}`; } else if ([2, 3, 4].includes(type)) { // number, date, time const style = (() => { @@ -265,24 +289,22 @@ function printAST(ast, options, level = 0, parentValue) { })(); const typesText = ['number', 'date', 'time']; - text += `{${normalizeArgName(ast.value, args)}, ${typesText[type - 2]}${style}}`; + text += `{${normalizeArgName(ast.value, type, args)}, ${typesText[type - 2]}${style}}`; } else if (type === 5) { // select - const argName = normalizeArgName(ast.value, args); - + const argName = normalizeArgName(ast.value, type, args); const optionsText = Object.entries(ast.options) .sort((a, b) => { return a[0] === 'other' ? 1 : (b[0] === 'other' ? -1 : 0); }) .map(([opt, { value }]) => { - return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, options, level + 1)}}`; + return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, args }, level + 1)}}`; }).join('') + (useNewlines ? newline : ''); text += `{${argName}, select,${optionsText}}`; } else if (type === 6) { // plural, selectordinal - const argName = normalizeArgName(ast.value, args); const supportedCats = getPluralCats(locale, ast.pluralType); const unsupportedCats = [ ...Object.keys(ast.options).filter(o => !/^=\d+$/.test(o)) , ...sortedCats].filter(cat => !supportedCats.includes(cat)); if (add) { @@ -335,6 +357,8 @@ function printAST(ast, options, level = 0, parentValue) { } }); + const argName = normalizeArgName(ast.value, type, args); + const typeText = ast.pluralType === 'ordinal' ? 'selectordinal' : 'plural'; const offsetText = + ast.offset !== 0 ? ` offset:${ast.offset}` : ''; const optionsText = Object.entries(ast.options).sort((a, b) => { @@ -343,7 +367,7 @@ function printAST(ast, options, level = 0, parentValue) { } return sortedCats.indexOf(a[0]) > sortedCats.indexOf(b[0]) ? 1 : -1; }).map(([opt, { value }]) => { - return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, swapOne }, level + 1, ast.value)}}`; + return `${newline}${useNewlines ? '\t' : ''}${opt} {${printAST(value, { ...options, swapOne, args: [...args] }, level + 1, ast.value)}}`; }).join('') + (useNewlines ? newline : ''); text += `{${argName}, ${typeText},${offsetText}${optionsText}}`; @@ -352,7 +376,7 @@ function printAST(ast, options, level = 0, parentValue) { text += '#'; } else if (type === 8) { // tag - text += `<${normalizeArgName(ast.value, args)}>${printAST(ast.children, options, level + 1)}`; + text += `<${normalizeArgName(ast.value, type, args)}>${printAST(ast.children, options, level + 1)}`; } else { // unhandled console.warn('unhandled type:', type); diff --git a/src/locale-data.js b/src/locale-data.js index bd023f7..f7204c4 100644 --- a/src/locale-data.js +++ b/src/locale-data.js @@ -1,2 +1,2 @@ -import cldrData from './locale-data-default.js'; -export default cldrData; +import localeData from './locale-data-default.js'; +export default localeData; From 2516a0de891aeba9d6a322a10b959d757825a832 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 29 Jan 2025 16:56:42 -0500 Subject: [PATCH 43/90] 3.0.0-beta.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b441a74..9fe274a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-alpha.1", + "version": "3.0.0-beta.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-alpha.1", + "version": "3.0.0-beta.0", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index c1144e1..37eb562 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-alpha.1", + "version": "3.0.0-beta.0", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 27bec454e5c83fd5bf7448e90a1ac50482dffd03 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 3 Feb 2025 12:41:31 -0500 Subject: [PATCH 44/90] 3.0.0-beta.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9fe274a..765528c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.0", + "version": "3.0.0-beta.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.0", + "version": "3.0.0-beta.1", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 37eb562..f04758e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.0", + "version": "3.0.0-beta.1", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 20af6d6b1e6289c08f141ad15fdc134246e829d3 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 4 Feb 2025 13:47:41 -0500 Subject: [PATCH 45/90] Indent multiline templates beyond their keys --- bin/cli.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 33b160b..81542bf 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -243,13 +243,13 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { quotes: commandOpts.quotes, expandHashes: true, - baseTabs: baseTabs.length, + baseTabs: baseTabs.length + (commandOpts.newlines ? 1 : 0), key: t.key, source, target: t }); const valQuote = commandOpts.newlines && newVal.includes('\n') ? '`' : t.valQuote; - const valSpace = commandOpts.newlines && newVal.includes('\n') ? `\n${baseTabs}` : t.valSpace; + const valSpace = commandOpts.newlines && newVal.includes('\n') ? `\n\t${baseTabs}` : t.valSpace; const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}${t.valQuote}${t.comma}${t.comment}`; const noo = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${valSpace}${valQuote}${newVal}${valQuote}${t.comma}${t.comment}`; From df90294ec763de5c49de143caddd22d52e1ca729 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 4 Feb 2025 13:48:16 -0500 Subject: [PATCH 46/90] Add apostophe character to delimiters --- build-locale-data.js | 17 ++++++++-- src/format.js | 17 ++++++---- src/locale-data-default.js | 69 +++++++++++++++++++++++++++----------- 3 files changed, 74 insertions(+), 29 deletions(-) diff --git a/build-locale-data.js b/build-locale-data.js index 860ef83..ef98a55 100644 --- a/build-locale-data.js +++ b/build-locale-data.js @@ -9,11 +9,24 @@ const defaultLocaleMap = { 'fr-on': 'fr-ca' }; const SAVE_PATH = posix.join(dirname(import.meta.url), 'src/locale-data.js').replace(/file:(\/c:)?/i, ''); function getDelimiters(locale) { + let delimiters; + const lang = locale.split('-')[0]; try { - return cldr.extractDelimiters(locale); + delimiters = cldr.extractDelimiters(locale); } catch(err) { - return cldr.extractDelimiters(locale.split('-')[0]); + delimiters = cldr.extractDelimiters(lang); } + + switch (lang) { + case 'haw': + delimiters.apostrophe = "'"; + break; + default: + delimiters.apostrophe = '’'; + break; + } + + return delimiters } let cldr; diff --git a/src/format.js b/src/format.js index 0987349..fb2e8df 100644 --- a/src/format.js +++ b/src/format.js @@ -176,7 +176,8 @@ function printAST(ast, options, level = 0, parentValue) { quoteStart = delimiters.quotationStart, quoteEnd = delimiters.quotationEnd, altStart = delimiters.alternateQuotationStart, - altEnd = delimiters.alternateQuotationEnd; + altEnd = delimiters.alternateQuotationEnd, + apostrophe = delimiters.apostrophe; //if (1) { // todo: fromSource if (localeLower.endsWith('-gb')) { @@ -191,7 +192,7 @@ function printAST(ast, options, level = 0, parentValue) { msg = msg .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") .replace(/(?<=\s)\\?'|^\\?'/g, altStart) // opening ' - .replace(/(?<=\S)'(?=\S)/g, '’') // apostrophe + .replace(/(?<=\S)'(?=\S)/g, apostrophe) // apostrophe .replace(/\\?'/g, altEnd) // closing ' .replace(/(?<=\s(\u0648)?)\\?"|^\\?"/g, quoteStart) // opening " .replace(/\\?"/g, quoteEnd); // closing " @@ -202,7 +203,8 @@ function printAST(ast, options, level = 0, parentValue) { quoteEnd: sourceQuoteEnd, quoteStart: sourceQuoteStart, altEnd: sourceAltEnd, - altStart: sourceAltStart + altStart: sourceAltStart, + apostrophe: sourceApostrophe } = (locale => { const { delimiters } = localeData[locale]; @@ -220,7 +222,8 @@ function printAST(ast, options, level = 0, parentValue) { quoteStart = delimiters.quotationStart, quoteEnd = delimiters.quotationEnd, altStart = delimiters.alternateQuotationStart, - altEnd = delimiters.alternateQuotationEnd; + altEnd = delimiters.alternateQuotationEnd, + apostrophe = delimiters.apostrophe; //if (1) { // todo: fromSource if (locale.toLowerCase().endsWith('-gb')) { @@ -230,7 +233,7 @@ function printAST(ast, options, level = 0, parentValue) { altEnd = delimiters.quotationEnd; } - return { quoteStart, quoteEnd, altStart, altEnd }; + return { quoteStart, quoteEnd, altStart, altEnd, apostrophe }; })(sourceLocale); @@ -238,11 +241,11 @@ function printAST(ast, options, level = 0, parentValue) { msg = msg .replace(new RegExp(`''`, 'g'), '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") .replace(new RegExp(`(?<=\s)\\\\?${sourceAltStart}|^\\\\?${sourceAltStart}`, 'g'), '|_altStart_|') // opening alt - .replace(new RegExp(`(?<=\\S)’(?=\\S)`, 'g'), '|_apostrophe_|') // apostrophe + .replace(new RegExp(`(?<=\\S)${sourceApostrophe}(?=\\S)`, 'g'), '|_apostrophe_|') // apostrophe .replace(new RegExp(`\\\\?${sourceAltEnd}`, 'g'), '|_altEnd_|') // closing alt .replace(new RegExp(`(?<=\\s(\\u0648)?)\\\\?${sourceQuoteStart}|^\\\\?${sourceQuoteStart}`, 'g'), '|_quoteStart_|') // opening quote .replace(new RegExp(`\\\\?${sourceQuoteEnd}`, 'g'), '|_quoteEnd_|') // closing quote - .replace(/\|_apostrophe_\|/g, "’") + .replace(/\|_apostrophe_\|/g, apostrophe) .replace(/\|_quoteStart_\|/g, quoteStart) .replace(/\|_quoteEnd_\|/g, quoteEnd) .replace(/\|_altStart_\|/g, altStart) diff --git a/src/locale-data-default.js b/src/locale-data-default.js index 1d6291a..84478bd 100644 --- a/src/locale-data-default.js +++ b/src/locale-data-default.js @@ -4,7 +4,8 @@ export default { "quotationStart": "”", "quotationEnd": "“", "alternateQuotationStart": "’", - "alternateQuotationEnd": "‘" + "alternateQuotationEnd": "‘", + "apostrophe": "’" } }, "cy": { @@ -12,7 +13,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "da": { @@ -20,7 +22,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "de": { @@ -28,7 +31,8 @@ export default { "quotationStart": "„", "quotationEnd": "“", "alternateQuotationStart": "‚", - "alternateQuotationEnd": "‘" + "alternateQuotationEnd": "‘", + "apostrophe": "’" } }, "en-gb": { @@ -36,7 +40,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "en": { @@ -44,7 +49,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "es-es": { @@ -52,7 +58,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "es": { @@ -60,7 +67,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "fr-ca": { @@ -68,7 +76,8 @@ export default { "quotationStart": "«", "quotationEnd": "»", "alternateQuotationStart": "«", - "alternateQuotationEnd": "»" + "alternateQuotationEnd": "»", + "apostrophe": "’" } }, "fr": { @@ -76,7 +85,17 @@ export default { "quotationStart": "«", "quotationEnd": "»", "alternateQuotationStart": "«", - "alternateQuotationEnd": "»" + "alternateQuotationEnd": "»", + "apostrophe": "’" + } + }, + "haw": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "'" } }, "hi": { @@ -84,7 +103,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "ja": { @@ -92,7 +112,8 @@ export default { "quotationStart": "「", "quotationEnd": "」", "alternateQuotationStart": "『", - "alternateQuotationEnd": "』" + "alternateQuotationEnd": "』", + "apostrophe": "’" } }, "ko": { @@ -100,7 +121,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "mi": { @@ -108,7 +130,8 @@ export default { "quotationStart": "\"", "quotationEnd": "\"", "alternateQuotationStart": "\"", - "alternateQuotationEnd": "\"" + "alternateQuotationEnd": "\"", + "apostrophe": "’" } }, "nl": { @@ -116,7 +139,8 @@ export default { "quotationStart": "‘", "quotationEnd": "’", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "pt": { @@ -124,7 +148,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "sv": { @@ -132,7 +157,8 @@ export default { "quotationStart": "”", "alternateQuotationStart": "’", "quotationEnd": "”", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "tr": { @@ -140,7 +166,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "zh-cn": { @@ -148,7 +175,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } }, "zh-tw": { @@ -156,7 +184,8 @@ export default { "quotationStart": "“", "quotationEnd": "”", "alternateQuotationStart": "‘", - "alternateQuotationEnd": "’" + "alternateQuotationEnd": "’", + "apostrophe": "’" } } } From fd2aa10798dfdb0ab79bc76544d4fcf45f13ba75 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 4 Feb 2025 14:06:35 -0500 Subject: [PATCH 47/90] Handle apostophes with pure cldr data --- src/format.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/format.js b/src/format.js index fb2e8df..9fe08bb 100644 --- a/src/format.js +++ b/src/format.js @@ -177,7 +177,7 @@ function printAST(ast, options, level = 0, parentValue) { quoteEnd = delimiters.quotationEnd, altStart = delimiters.alternateQuotationStart, altEnd = delimiters.alternateQuotationEnd, - apostrophe = delimiters.apostrophe; + apostrophe = delimiters.apostrophe ?? '’'; //if (1) { // todo: fromSource if (localeLower.endsWith('-gb')) { @@ -223,7 +223,7 @@ function printAST(ast, options, level = 0, parentValue) { quoteEnd = delimiters.quotationEnd, altStart = delimiters.alternateQuotationStart, altEnd = delimiters.alternateQuotationEnd, - apostrophe = delimiters.apostrophe; + apostrophe = delimiters.apostrophe ?? '’'; //if (1) { // todo: fromSource if (locale.toLowerCase().endsWith('-gb')) { From 76c14e6a777fe7caf77fc67149af4ce04a3533ba Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 4 Feb 2025 14:29:50 -0500 Subject: [PATCH 48/90] Add haw quote test --- src/format.js | 3 ++- test/format.test.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/format.js b/src/format.js index 9fe08bb..882da40 100644 --- a/src/format.js +++ b/src/format.js @@ -192,8 +192,9 @@ function printAST(ast, options, level = 0, parentValue) { msg = msg .replace(/''/g, '|_single_|').replace(/'/g, '|_escape_|').replace(/\|_single_\|/g, "'") .replace(/(?<=\s)\\?'|^\\?'/g, altStart) // opening ' - .replace(/(?<=\S)'(?=\S)/g, apostrophe) // apostrophe + .replace(/(?<=\S)'(?=\S)/g, '|_apostrophe_|') // apostrophe .replace(/\\?'/g, altEnd) // closing ' + .replace(/\|_apostrophe_\|/g, apostrophe) .replace(/(?<=\s(\u0648)?)\\?"|^\\?"/g, quoteStart) // opening " .replace(/\\?"/g, quoteEnd); // closing " } diff --git a/test/format.test.js b/test/format.test.js index a8ff1c7..c33118a 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { formatMessage } from '../src/format.js'; -describe('await formatMessage', () => { +describe('formatMessage', () => { let locale; beforeEach(() => { @@ -15,6 +15,7 @@ describe('await formatMessage', () => { { locale: 'en', expected: `This isn’t “correct”` }, { locale: 'en-gb', expected: `This isn’t ‘correct’` }, { locale: 'fr', expected: `This isn’t «\u202fcorrect\u202f»` }, + { locale: 'haw', expected: `This isn't “correct”` }, { locale: 'sv', expected: `This isn’t ”correct”` }, ].forEach(({ locale, expected }) => { it(`should replace straight quotes with "${locale}" quotes when "quotes" option is "straight"`, async() => { From ab88455b2d7fd243d51032f6dab9ce53689a55a1 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 4 Feb 2025 14:30:57 -0500 Subject: [PATCH 49/90] 3.0.0-beta.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 765528c..8a8d0da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.1", + "version": "3.0.0-beta.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.1", + "version": "3.0.0-beta.2", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index f04758e..4d402e0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.1", + "version": "3.0.0-beta.2", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 7e95b2493e4171b05d221c3c0849a13771060246 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 21 Mar 2025 23:26:35 -0400 Subject: [PATCH 50/90] Fix case match argument correction --- src/format.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/format.js b/src/format.js index 882da40..bed2e31 100644 --- a/src/format.js +++ b/src/format.js @@ -103,8 +103,8 @@ function normalizeArgName(name, type, availableArgs) { } else { const idx = availableArgs.findIndex(([n, t]) => t === type && n.toLowerCase() === name.toLowerCase()); if (idx > -1) { - const caseMatch = availableArgs.splice(idx, 1)[0]; - return caseMatch; + // case match + return availableArgs.splice(idx, 1)[0][0]; } return [...new Set(availableArgs .reduce((acc, [n, t]) => t === type && acc.push(n) && acc || acc, []) From 8adfe4cc754815988963bba632917282f4c83f14 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 21 Mar 2025 23:26:45 -0400 Subject: [PATCH 51/90] 3.0.0-beta.3 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8a8d0da..ecc4184 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.2", + "version": "3.0.0-beta.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.2", + "version": "3.0.0-beta.3", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 4d402e0..6ce7618 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.2", + "version": "3.0.0-beta.3", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From d0aa7464787f562ad1ea5cb9ad102a04eebe9e4f Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 23 May 2025 10:23:29 -0400 Subject: [PATCH 52/90] Escape quotes as necessary --- bin/cli.js | 5 ++++- src/format.js | 5 ++++- src/validate.js | 9 +++++++++ test/validate.test.js | 17 +++++++++++++++-- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 81542bf..805de18 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -231,7 +231,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { if (localeContents.includes(t)) { const baseTabs = t.match('^\n?(?\t*)').groups.tabs - const newVal = await formatMessage(t.val, { + let newVal = await formatMessage(t.val, { locale, sourceLocale, add: commandOpts.add, @@ -250,6 +250,9 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { }); const valQuote = commandOpts.newlines && newVal.includes('\n') ? '`' : t.valQuote; const valSpace = commandOpts.newlines && newVal.includes('\n') ? `\n\t${baseTabs}` : t.valSpace; + + newVal = newVal.replaceAll(valQuote, `\\${valQuote}`); + const old = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${t.valSpace}${t.valQuote}${t.val}${t.valQuote}${t.comma}${t.comment}`; const noo = `${t.keyQuote}${t.key}${t.keyQuote}${t.keySpace}:${valSpace}${valQuote}${newVal}${valQuote}${t.comma}${t.comment}`; diff --git a/src/format.js b/src/format.js index bed2e31..b229e13 100644 --- a/src/format.js +++ b/src/format.js @@ -4,7 +4,10 @@ import { getLocaleData, getPluralCats, paddedQuoteLocales, sortedCats, structure function getArgs(asts, types = [1,2,3,4,5,6,8], args = []) { asts.forEach(ast => { - if (types.includes(ast.type)) args.push([ast.value, ast.type]); + if (types.includes(ast.type)) { + if (ast.type === 8) args.push([ast.value, ast.type]); // account for closing tag + args.push([ast.value, ast.type]); + } if (ast.options) Object.values(ast.options).map(({ value: asts }) => getArgs(asts, types, args)); if (ast.children) args.concat(getArgs(ast.children, types, args)); }) diff --git a/src/validate.js b/src/validate.js index f8d9915..8521fb2 100644 --- a/src/validate.js +++ b/src/validate.js @@ -134,6 +134,15 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const supportedCats = getPluralCats(targetLocale, part.pluralType); const cats = Object.keys(part.options); const missingCats = supportedCats.filter(c => c !== 'other' && !cats.includes(c)); + + if (missingCats.includes('one') && cats.includes('=1')) { + // locales where `one` can match something other than 1 + const oneIsNot1Regex = /^(fr|da|hi|pt(?!-pt))(-?|$)/; + if (!oneIsNot1Regex.test(targetLocale)) { + missingCats.splice(missingCats.indexOf('one'), 1); + } + } + if (missingCats.length) msgReporter.warning('category-missing', `Missing ${missingCats.length === 1 ? 'category' : 'categories'} ${formatList(sortedCats.filter(c => missingCats.includes(c)).map(i => `"${i}"`))}`); const unsupportedCats = cats.filter(c => !/^=\d+$/.test(c) && !supportedCats.includes(c)); diff --git a/test/validate.test.js b/test/validate.test.js index 3adc015..0fff76e 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -43,14 +43,27 @@ describe('validate', () => { it('generates a "category-missing" warning when a target message is missing supported plural categories', () => { targetLocale = 'cy-gb'; const sourceMessage = '{a, plural, one {} other {}}'; - const targetMessage = '{a, plural, one {} other {}}'; + const targetMessage = '{a, plural, other {}}'; reporter.config(targetMessage, sourceMessage, 'key'); reporter._config.locale = targetLocale; validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); expect(reporter.issues[0].type).to.equal('category-missing'); expect(reporter.issues[0].level).to.equal('warning'); - expect(reporter.issues[0].msg).to.equal('Missing categories "zero", "two", "few", and "many"'); + expect(reporter.issues[0].msg).to.equal('Missing categories "zero", "one", "two", "few", and "many"'); + }); + + it('does not generate a "category-missing" warning for `one` in a target locale where `one` can only match 1 and `=1` exists', () => { + targetLocale = 'es'; + const sourceMessage = '{a, plural, =1 {} other {}}'; + const targetMessage = '{a, plural, =1 {} other {}}'; + reporter.config(targetMessage, sourceMessage, 'key'); + reporter._config.locale = targetLocale; + validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); + expect(reporter.issues.length).to.equal(1); + expect(reporter.issues[0].type).to.equal('category-missing'); + expect(reporter.issues[0].level).to.equal('warning'); + expect(reporter.issues[0].msg).to.equal('Missing category "many"'); }); it('generates "category" errors when a target message uses unsupported plural categories', () => { From 961a1b1f89c1306ce4c9f4cf7164aada77990a4d Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 23 May 2025 10:24:19 -0400 Subject: [PATCH 53/90] 3.0.0-beta.4 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index ecc4184..8a206de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.3", + "version": "3.0.0-beta.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.3", + "version": "3.0.0-beta.4", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 6ce7618..5b926b0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.3", + "version": "3.0.0-beta.4", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 1f7c195807b3dc356697e5040e5613c0ae87f735 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 23 May 2025 11:39:06 -0400 Subject: [PATCH 54/90] Unescape quotes before formatting --- bin/cli.js | 5 +++-- src/format.js | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 805de18..8cbfd5c 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -230,8 +230,9 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { const source = commandOpts.correct ? sourceLocaleParsed[t.key] : null; if (localeContents.includes(t)) { - const baseTabs = t.match('^\n?(?\t*)').groups.tabs - let newVal = await formatMessage(t.val, { + const baseTabs = t.match('^\n?(?\t*)').groups.tabs; + const unescapedValue = t.val.replaceAll(`\\${t.valQuote}`, t.valQuote); + let newVal = await formatMessage(unescapedValue, { locale, sourceLocale, add: commandOpts.add, diff --git a/src/format.js b/src/format.js index b229e13..9e5623f 100644 --- a/src/format.js +++ b/src/format.js @@ -28,7 +28,7 @@ function expandASTHashes(ast, parentValue) { } } -function escape(msg) { +function mfEscape(msg) { return msg.replace(/'([{}](?:.*?[{}])?)'/gsu, "'''$1'''"); } @@ -36,7 +36,7 @@ export async function formatMessage(msg, options = {}) { let ast; try { - msg = options.quotes === 'straight' ? msg.replace(/'/g, "'''") : escape(msg); + msg = options.quotes === 'straight' ? msg.replace(/'/g, "'''") : mfEscape(msg); ast = parse(msg, { requiresOtherClause: false }); } catch(err) { try { From f2d1d57cb16a44bc2ee5fcaf318190ee2e7e7da9 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 23 May 2025 11:39:21 -0400 Subject: [PATCH 55/90] 3.0.0-beta.5 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8a206de..a2b3bde 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.4", + "version": "3.0.0-beta.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.4", + "version": "3.0.0-beta.5", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 5b926b0..9db06f2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.4", + "version": "3.0.0-beta.5", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 876daeaf75af8ce6c7e885b65d3c6283ca4b4cd3 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 3 Jun 2025 14:36:16 -0400 Subject: [PATCH 56/90] Export structureRegEx and formatMessage --- src/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index f8cfe95..5639098 100644 --- a/src/index.js +++ b/src/index.js @@ -1,2 +1,4 @@ export { Reporter } from './reporter.js'; -export { structureRegEx, validateMessage } from './validate.js'; +export { validateMessage } from './validate.js'; +export { structureRegEx } from './utils.js'; +export { formatMessage } from './format.js'; From af0c2e403a76d3241c4a93ced3f99257dbb36a47 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 4 Jun 2025 10:04:17 -0400 Subject: [PATCH 57/90] Improve structureRegEx to handle escapes better (but not perfectly) --- src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index a42225a..f840011 100644 --- a/src/utils.js +++ b/src/utils.js @@ -17,7 +17,7 @@ export async function getConfig(cwd) { export const sortedCats = ['zero', 'one', 'two', 'few', 'many', 'other']; export const paddedQuoteLocales = ['fr', 'fr-ca', 'fr-fr', 'fr-on', 'vi-vn']; -export const structureRegEx = /(?<=\s*){(.|\n)*?[{}]|\s*}(.|\n)*?[{}]|[{#]|(\s*)}/g; +export const structureRegEx = /(?<=\s*)(?}](?:[^']*?[{<>}])?))')(''|([{<]([^']|\n)*?[{<>}])|\s*}([^']|\n)*?[{}]|#)/g export async function getLocaleData(locale) { locale = (await getConfig(env.PWD))?.localesMap?.[locale] || locale; From c9f6218754c067bcfd8f6409ec91beead3b040c4 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 4 Jun 2025 20:36:23 -0400 Subject: [PATCH 58/90] Fix indentation when not using -n --- bin/cli.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cli.js b/bin/cli.js index 8cbfd5c..6b5b876 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -230,7 +230,10 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { const source = commandOpts.correct ? sourceLocaleParsed[t.key] : null; if (localeContents.includes(t)) { - const baseTabs = t.match('^\n?(?\t*)').groups.tabs; + const baseTabs = t.valSpace.includes('\n') && !commandOpts.newlines + ? t.valSpace.match(/^\n?(?\t*)/).groups.tabs + : t.match(/^\n?(?\t*)/).groups.tabs; + const unescapedValue = t.val.replaceAll(`\\${t.valQuote}`, t.valQuote); let newVal = await formatMessage(unescapedValue, { locale, From fe126b8d7941b29cf8b60bad31a087a1926cf2ba Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 4 Jun 2025 20:37:33 -0400 Subject: [PATCH 59/90] 3.0.0-beta.6 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index a2b3bde..7d63f4c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.5", + "version": "3.0.0-beta.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.5", + "version": "3.0.0-beta.6", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 9db06f2..38ca935 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.5", + "version": "3.0.0-beta.6", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From f414ffa55a6636dc9c8d0c46795c17ded089ff20 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 4 Jun 2025 20:37:53 -0400 Subject: [PATCH 60/90] 3.0.0-beta.7 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d63f4c..3d30433 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.6", + "version": "3.0.0-beta.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.6", + "version": "3.0.0-beta.7", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 38ca935..ce6b4ad 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.6", + "version": "3.0.0-beta.7", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 13afe41ac16fa1eacf906bd6b108d34b6e4a956f Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 6 Jun 2025 15:45:41 -0400 Subject: [PATCH 61/90] Fix unicode regex --- src/validate.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/validate.js b/src/validate.js index 8521fb2..ff57b5c 100644 --- a/src/validate.js +++ b/src/validate.js @@ -221,9 +221,9 @@ export function parseLocales(locales, useJSONObj) { }; const regex = useJSONObj - ? /("(?.*)"(\s*):(\s*)\{)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/g + ? /("(?.*)"(\s*):(\s*)\{)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/gv //[ ][ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] - : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/g; + : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/gv; //[ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] const matches = Array.from(contents.matchAll(regex)); From 997157aadcf75e8e12f6adb19719e18051eea6bb Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 6 Jun 2025 16:08:23 -0400 Subject: [PATCH 62/90] Fix file writing --- bin/cli.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 6b5b876..b77bc40 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -312,7 +312,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { await Promise.all(output.map(async(locale, idx) => { const localeFile = `${locales[locale.locale].file}`; - + const localeFilePath = `${localesPath}${localeFile}`; if (!allowedLocales || allowedLocales.includes(locale.locale)) { if (program.sort || @@ -321,7 +321,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { program.printMissing || locale.report.totals.errors || locale.report.totals.warnings) { - console.log((idx > 0 ? '\n' : '') + chalk.underline(`${localesPath}${localeFile}`)); + console.log((idx > 0 ? '\n' : '') + chalk.underline(localeFilePath)); } if (programOpts.issues) { @@ -376,7 +376,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { } } else if (!programOpts.ignore || !programOpts.ignore - .replace(' ','') + .replace(' ', '') .split(',') .includes(issue.type) ) { @@ -395,7 +395,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { } if (program.removeExtraneous || program.addMissing || program.sort) { - writeFile(localeFile, locales[locale.locale].contents); + await writeFile(localeFilePath, locales[locale.locale].contents); } } From 69dacb2a5d64174d15406ff6bd9fa58c0db0f115 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Sun, 8 Jun 2025 11:25:10 -0400 Subject: [PATCH 63/90] Sort . above everything else --- bin/cli.js | 6 +++--- src/utils.js | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index b77bc40..8e77a22 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -9,7 +9,7 @@ import chalk from 'chalk'; import glob from 'glob'; import pkg from '../package.json' with { type: 'json' }; import { program, Option } from 'commander'; -import { getConfig, structureRegEx } from '../src/utils.js'; +import { getConfig, sortFn, structureRegEx } from '../src/utils.js'; let formatMessage, commandOpts; const programArgs = {}; @@ -336,8 +336,8 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { acc.push(block); return acc; }, []) - .map(block => block.sort().join('')) - .sort() + .map(block => block.sort(sortFn).join('')) + .sort(sortFn) .join('\n'); locales[locale.locale].contents = locales[locale.locale].contents.replace(Object.values(locales[locale.locale].parsed).join(''), sorted); diff --git a/src/utils.js b/src/utils.js index f840011..f97914a 100644 --- a/src/utils.js +++ b/src/utils.js @@ -31,3 +31,6 @@ export function getPluralCats(locale, pluralType) { export function formatList(arr, locale = 'en') { return new Intl.ListFormat(locale).format(arr); } + +// sort '.' above '-' +export const sortFn = (a, b) => a.replace(/\./g, '\u0000') >= b.replace(/\./g, '\u0000') ? 1 : -1; From 38cae66e2a0a7ce5b069030e8d48bb08632d2407 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 9 Jun 2025 07:53:16 -0400 Subject: [PATCH 64/90] Fix exit code --- bin/cli.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cli.js b/bin/cli.js index 8e77a22..265d2a9 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -457,6 +457,8 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { if (results.filter(r => r).length) { + let exitCode = 0; + const totals = { errors: 0, warnings: 0, @@ -490,7 +492,8 @@ if (results.filter(r => r).length) { if (totals.errors - totals.ignored.errors) { console.error('\nErrors were reported in at least one locale. See details above.'); + exitCode = 1; } - process.exit(Number(Boolean(results.find(r => r === 1)))); + process.exit(exitCode); } From 11d3f7a7f3f01d8c25bf96e0e6dc2cdd763e53e5 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 9 Jun 2025 08:05:06 -0400 Subject: [PATCH 65/90] Do not hoist with trim enabled --- bin/cli.js | 4 +++- src/format.js | 11 +++++++---- test/format.test.js | 9 ++++++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 265d2a9..7669b71 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -85,8 +85,9 @@ program .option('-a, --add', 'Add cases for missing supported pural and selectordinal categories') .option('-r, --remove', 'Remove cases for unsupported pural and selectordinal categories') .option('-d, --dedupe', 'Remove complex argument cases that duplicate the `other` case. Takes precedence over --add.') - .option('-t, --trim', 'Trim whitespace from both ends of messages') + .option('-t, --trim', 'Trim whitespace from both ends of messages. Disables selector hoisting.') .option('-c, --correct', 'Attempt to correct argument names. A source locale must be provided.') + .addOption(new Option('--no-hoist', 'Do not hoist selectors').hideHelp()) .addOption(new Option('-q, --quotes ', 'Replace quote characters with locale-appropriate characters').choices(['source', 'straight', 'both'])) .action(async function() { formatMessage = (await import('../src/format.js')).formatMessage; @@ -246,6 +247,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { collapse: commandOpts.collapse, quotes: commandOpts.quotes, expandHashes: true, + hoist: commandOpts.hoist, baseTabs: baseTabs.length + (commandOpts.newlines ? 1 : 0), key: t.key, diff --git a/src/format.js b/src/format.js index 9e5623f..d3c1dc5 100644 --- a/src/format.js +++ b/src/format.js @@ -61,10 +61,13 @@ export async function formatMessage(msg, options = {}) { if (options.expandHashes) { expandASTHashes(ast); } - try { - ast = hoistSelectors(ast); - } catch(e) { - console.log(e); + + if ((options.hoist ?? true) && !options.trim) { + try { + ast = hoistSelectors(ast); + } catch(e) { + console.log(e); + } } const localeData = { [options.locale]: await getLocaleData(options.locale) }; diff --git a/test/format.test.js b/test/format.test.js index c33118a..951d5d6 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -168,13 +168,20 @@ describe('formatMessage', () => { expect(formatted).to.equal(expected); }); - it('should hoist complex selectors to the outside and nest appropriately with no options', async() => { + it('should hoist selectors to the outside and nest appropriately with no options', async() => { const message = `\t{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; const expected = `{a, plural, =1 {{b, plural, =1 {\ta cat and a dog!} other {\ta cat and {b} dogs!}}} other {{b, plural, =1 {\t{a} cats and a dog!} other {\t{a} cats and {b} dogs!}}}}`; const formatted = await formatMessage(message, { locale }); expect(formatted).to.equal(expected); }); + it('should not hoist selectors with the "trim" option', async() => { + const message = `\t{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; + const expected = `{a, plural, =1 {a cat} other {{a} cats}} and {b, plural, =1 {a dog} other {{b} dogs}}!`; + const formatted = await formatMessage(message, { locale, trim: true }); + expect(formatted).to.equal(expected); + }); + it(`should trim whitespace with the "trim" option`, async() => { const message = `\n{a, plural, other { value }}\t`; const expected = `{a, plural, other {value}}`; From d131d54e81cdbdd9015e4b81aea53a69d8f85fd7 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 9 Jun 2025 08:08:25 -0400 Subject: [PATCH 66/90] 3.0.0-beta.8 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3d30433..f27112f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.7", + "version": "3.0.0-beta.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.7", + "version": "3.0.0-beta.8", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index ce6b4ad..0477f25 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.7", + "version": "3.0.0-beta.8", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 2392dcf677df893516b74acd4e909b371434c03e Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 9 Jun 2025 08:27:08 -0400 Subject: [PATCH 67/90] Add Thai and Vietnamese as default locales --- build-locale-data.js | 4 ++-- src/locale-data-default.js | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/build-locale-data.js b/build-locale-data.js index ef98a55..3216c97 100644 --- a/build-locale-data.js +++ b/build-locale-data.js @@ -3,7 +3,7 @@ import { env, stderr } from 'node:process'; import { formatList, getConfig } from './src/utils.js'; import { dirname, join, posix } from 'node:path'; -const defaultLocales = ['ar', 'cy', 'da', 'de', 'en', 'en-gb', 'es', 'es-es', 'fr', 'fr-ca', 'fr-fr', 'haw', 'hi', 'ja', 'ko', 'mi', 'nl', 'pt', 'sv', 'tr', 'zh-cn', 'zh-tw']; +const defaultLocales = ['ar', 'cy', 'da', 'de', 'en', 'en-gb', 'es', 'es-es', 'fr', 'fr-ca', 'fr-fr', 'haw', 'hi', 'ja', 'ko', 'mi', 'nl', 'pt', 'sv', 'th', 'tr', 'vi', 'zh-cn', 'zh-tw']; const defaultLocaleMap = { 'fr-on': 'fr-ca' }; const SAVE_PATH = posix.join(dirname(import.meta.url), 'src/locale-data.js').replace(/file:(\/c:)?/i, ''); @@ -53,7 +53,7 @@ await (async() => { cldr = (cldrImport).default; const data = {}; - locales.forEach(locale => { + nonDefaultLocales.forEach(locale => { try { locale = Intl.getCanonicalLocales(locale.trim().toLowerCase())[0]; } catch(e) { diff --git a/src/locale-data-default.js b/src/locale-data-default.js index 84478bd..f9a8c48 100644 --- a/src/locale-data-default.js +++ b/src/locale-data-default.js @@ -161,7 +161,25 @@ export default { "apostrophe": "’" } }, + "th": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’" + } + }, "tr": { + "delimiters": { + "quotationStart": "“", + "quotationEnd": "”", + "alternateQuotationStart": "‘", + "alternateQuotationEnd": "’", + "apostrophe": "’", + } + }, + "vi": { "delimiters": { "quotationStart": "“", "quotationEnd": "”", From 5f949657c9c645817517cddfc7e6c8937d0f4e77 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 9 Jun 2025 14:32:45 -0400 Subject: [PATCH 68/90] Fix writes more --- bin/cli.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/cli.js b/bin/cli.js index 7669b71..5c8278b 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -129,6 +129,7 @@ const noSource = () => { process.exit(1); }; +const writes = []; const localesPaths = glob.sync(pathCombined); const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { @@ -397,7 +398,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { } if (program.removeExtraneous || program.addMissing || program.sort) { - await writeFile(localeFilePath, locales[locale.locale].contents); + writes.push([localeFilePath, locales[locale.locale].contents]); } } @@ -457,6 +458,10 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { return totals; })); +for (const [path, contents] of writes) { + await writeFile(path, contents); +} + if (results.filter(r => r).length) { let exitCode = 0; From dd3df2547027c3995b49a41111c9b7eb6e9f5658 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 9 Jun 2025 14:45:35 -0400 Subject: [PATCH 69/90] Fix print-missing --- bin/cli.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index 5c8278b..cd96148 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -311,9 +311,9 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { if (!sourceLocale) noSource(); const output = validateLocales({ locales, sourceLocale }); - const translatorOutput = {}; await Promise.all(output.map(async(locale, idx) => { + const printMissing = {}; const localeFile = `${locales[locale.locale].file}`; const localeFilePath = `${localesPath}${localeFile}`; if (!allowedLocales || allowedLocales.includes(locale.locale)) { @@ -375,7 +375,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { } else if (program.printMissing) { if (['missing', 'untranslated'].includes(issue.type)) { - translatorOutput[issue.key] = issue.source; + printMissing[issue.key] = issue.source; } } else if (!programOpts.ignore || !programOpts.ignore @@ -403,7 +403,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { } if (program.printMissing) { - console.log(JSON.stringify(translatorOutput, null, 2)); + console.log(JSON.stringify(printMissing, null, 2)); } else if (program.removeExtraneous) { const count = locale.report.errors ? locale.report.errors.extraneous || 0 : 0; From 1b8e7afa078fe77fb338c275f62511eabdb6af47 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 9 Jun 2025 14:56:25 -0400 Subject: [PATCH 70/90] 3.0.0-beta.9 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f27112f..2489a96 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.8", + "version": "3.0.0-beta.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.8", + "version": "3.0.0-beta.9", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 0477f25..8d68da8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.8", + "version": "3.0.0-beta.9", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 968c80b36f1d717c44f12e4f47dbf1657d696ea2 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 10 Jun 2025 07:57:31 -0400 Subject: [PATCH 71/90] Fix source quotes --- src/format.js | 6 +++--- test/format.test.js | 28 +++++++++++++++++----------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/format.js b/src/format.js index d3c1dc5..80eed30 100644 --- a/src/format.js +++ b/src/format.js @@ -205,7 +205,7 @@ function printAST(ast, options, level = 0, parentValue) { .replace(/\\?"/g, quoteEnd); // closing " } - if (quotes === 'source' || quotes === 'both') { + if (quotes === 'source' || quotes === 'both' && !locale.endsWith('-gb')) { const { quoteEnd: sourceQuoteEnd, quoteStart: sourceQuoteStart, @@ -250,8 +250,8 @@ function printAST(ast, options, level = 0, parentValue) { .replace(new RegExp(`(?<=\s)\\\\?${sourceAltStart}|^\\\\?${sourceAltStart}`, 'g'), '|_altStart_|') // opening alt .replace(new RegExp(`(?<=\\S)${sourceApostrophe}(?=\\S)`, 'g'), '|_apostrophe_|') // apostrophe .replace(new RegExp(`\\\\?${sourceAltEnd}`, 'g'), '|_altEnd_|') // closing alt - .replace(new RegExp(`(?<=\\s(\\u0648)?)\\\\?${sourceQuoteStart}|^\\\\?${sourceQuoteStart}`, 'g'), '|_quoteStart_|') // opening quote - .replace(new RegExp(`\\\\?${sourceQuoteEnd}`, 'g'), '|_quoteEnd_|') // closing quote + .replace(new RegExp(`\\\\?${sourceQuoteStart}(\\b|(?=\\p{Sc}|\\p{P}))`, 'g'), '|_quoteStart_|') // opening quote + .replace(new RegExp(`\\b\\\\?${sourceQuoteEnd}`, 'g'), '|_quoteEnd_|') // closing quote .replace(/\|_apostrophe_\|/g, apostrophe) .replace(/\|_quoteStart_\|/g, quoteStart) .replace(/\|_quoteEnd_\|/g, quoteEnd) diff --git a/test/format.test.js b/test/format.test.js index 951d5d6..6b5167d 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -9,26 +9,32 @@ describe('formatMessage', () => { }); [ - { locale: 'ar', expected: `This isn’t ”correct“` }, - { locale: 'cy', expected: `This isn’t “correct”` }, - { locale: 'de', expected: `This isn’t „correct“` }, - { locale: 'en', expected: `This isn’t “correct”` }, - { locale: 'en-gb', expected: `This isn’t ‘correct’` }, - { locale: 'fr', expected: `This isn’t «\u202fcorrect\u202f»` }, - { locale: 'haw', expected: `This isn't “correct”` }, - { locale: 'sv', expected: `This isn’t ”correct”` }, + { locale: 'ar', expected: `This isn’t ”correct“.` }, + { locale: 'cy', expected: `This isn’t “correct”.` }, + { locale: 'de', expected: `This isn’t „correct“.` }, + { locale: 'en', expected: `This isn’t “correct”.` }, + { locale: 'en-gb', expected: `This isn’t ‘correct’.` }, + { locale: 'fr', expected: `This isn’t «\u202fcorrect\u202f».` }, + { locale: 'haw', expected: `This isn't “correct”.` }, + { locale: 'sv', expected: `This isn’t ”correct”.` }, + { locale: 'mi', expected: `This isn’t "correct".` }, ].forEach(({ locale, expected }) => { it(`should replace straight quotes with "${locale}" quotes when "quotes" option is "straight"`, async() => { - const message = `This isn't "correct"`; + const message = `This isn't "correct".`; const formatted = await formatMessage(message, { locale, sourceLocale: 'en', quotes: 'straight' }); expect(formatted).to.equal(expected); }); - it(`should replace source quotes with "${locale}" quotes when "quotes" option is "source"`, async() => { - const message = `This isn’t “correct”`; + it(`should replace source quotes with "${locale}" quotes when "quotes" option is "source"`, async () => { + const message = `This isn’t “correct”.`; const formatted = await formatMessage(message, { locale, sourceLocale: 'en', quotes: 'source' }); expect(formatted).to.equal(expected); }); + + it(`should not alter own "${locale}" quotes when "quotes" option is "source"`, async() => { + const formattedSelf = await formatMessage(expected, { locale, sourceLocale: 'en', quotes: 'source' }); + expect(formattedSelf).to.equal(expected); + }); }); [ From c47d73c8450a8b79c707257c54b09a1ff9faec81 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 10 Jun 2025 07:58:53 -0400 Subject: [PATCH 72/90] 3.0.0-beta.10 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2489a96..37d2987 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.9", + "version": "3.0.0-beta.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.9", + "version": "3.0.0-beta.10", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 8d68da8..927bd9e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.9", + "version": "3.0.0-beta.10", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 0e85064147217ca898a7e3839dedd7601687a634 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 19 Jun 2025 16:02:33 -0400 Subject: [PATCH 73/90] Fix hash replacements --- src/format.js | 4 ++-- test/format.test.js | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/format.js b/src/format.js index 80eed30..1d50a31 100644 --- a/src/format.js +++ b/src/format.js @@ -19,11 +19,11 @@ function expandASTHashes(ast, parentValue) { ast.map(ast => expandASTHashes(ast, parentValue)); } - if (ast.type === 7) { // # + if (ast.type === 7 && parentValue) { // # ast.type = 1; ast.value = parentValue; } - else if (ast.type === 6) { // plural, selectordinal + else if (ast.type === 6 && !ast.offset) { // plural, selectordinal expandASTHashes(Object.values(ast.options).map(o => o.value), ast.value); } } diff --git a/test/format.test.js b/test/format.test.js index 6b5167d..a213a2b 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -216,4 +216,18 @@ describe('formatMessage', () => { expect(formatted).to.equal(expected); }); + it(`should replace hashes`, async() => { + const message = `{a, plural, one {There is # thing}}`; + const expected = `{a, plural, one {There is {a} thing}}`; + const formatted = await formatMessage(message, { locale, expandHashes: true }); + expect(formatted).to.equal(expected); + }); + + it(`should not replace hashes if there is an offset`, async() => { + const message = `{a, plural, offset:1 one {There is # thing}}`; + const expected = `{a, plural, offset:1 one {There is # thing}}`; + const formatted = await formatMessage(message, { locale, expandHashes: true }); + expect(formatted).to.equal(expected); + }); + }); From dfbc96bd60f26315e94b0c0cdee88aaf4d664981 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 19 Jun 2025 17:10:04 -0400 Subject: [PATCH 74/90] 3.0.0-beta.11 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 37d2987..4f415c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.10", + "version": "3.0.0-beta.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.10", + "version": "3.0.0-beta.11", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 927bd9e..d671d74 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.10", + "version": "3.0.0-beta.11", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From f23142b07a2bc4a4cf88da6292f1c4e95a2fbb2e Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 19 Jun 2025 23:08:43 -0400 Subject: [PATCH 75/90] Fix sorting comma handling --- bin/cli.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index cd96148..e5c3996 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -332,18 +332,25 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { locale.report.totals.ignored = { warnings: 0, errors: 0 }; if (program.sort) { + let lastComma = ','; const sorted = Object.values(locales[locale.locale].parsed) .reduce((acc, val) => { const block = !val.startsWith('\n\n') ? acc.pop() || [] : []; + if (!val.comma) { + lastComma = ''; + val.comma = ','; + val = `${val.match(/\s*/)[0]}${val.keyQuote}${val.key}${val.keyQuote}${val.keySpace}:${val.valSpace}${val.valQuote}${val.val}${val.valQuote}${val.comma}${val.comment}`; + } block.push(val.replace('\n\n', '\n')); acc.push(block); return acc; }, []) .map(block => block.sort(sortFn).join('')) .sort(sortFn) - .join('\n'); - locales[locale.locale].contents = locales[locale.locale].contents.replace(Object.values(locales[locale.locale].parsed).join(''), sorted); + const last = sorted.pop(); + sorted.push(last.replace(/,(\s*($|\/\/[^\n]*))/, `${lastComma}$1`)); + locales[locale.locale].contents = locales[locale.locale].contents.replace(Object.values(locales[locale.locale].parsed).join(''), sorted.join('\n')); } else { From d56f349782845caa51660a9f7046dc27bacf8672 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 19 Jun 2025 23:08:54 -0400 Subject: [PATCH 76/90] 3.0.0-beta.12 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4f415c9..2568950 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.11", + "version": "3.0.0-beta.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.11", + "version": "3.0.0-beta.12", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index d671d74..eac5d0c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.11", + "version": "3.0.0-beta.12", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 3896ace3fa3b99f47d898f4d9de8cab6b0b11f9f Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 20 Jun 2025 00:25:06 -0400 Subject: [PATCH 77/90] Handle comments properly when checking comma for sort --- bin/cli.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cli.js b/bin/cli.js index e5c3996..6736a6b 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -349,7 +349,7 @@ const results = await Promise.all(localesPaths.map(async (localesPath, idx) => { .sort(sortFn) const last = sorted.pop(); - sorted.push(last.replace(/,(\s*($|\/\/[^\n]*))/, `${lastComma}$1`)); + sorted.push(last.replace(/,(\s*(\/\/[^\n]*)?$)/, `${lastComma}$1`)); locales[locale.locale].contents = locales[locale.locale].contents.replace(Object.values(locales[locale.locale].parsed).join(''), sorted.join('\n')); } else { From 608a148f1e4578792102546db0390ab3193bf726 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Fri, 20 Jun 2025 00:25:16 -0400 Subject: [PATCH 78/90] 3.0.0-beta.13 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2568950..5a0f6b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.12", + "version": "3.0.0-beta.13", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.12", + "version": "3.0.0-beta.13", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index eac5d0c..8a8b285 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.12", + "version": "3.0.0-beta.13", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 0e56cb04327332450f284e4d1b8967833e1971e7 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 25 Jun 2025 16:23:20 -0400 Subject: [PATCH 79/90] Convert English source "=1" cases to "one" if only basic plural differences from "other" are found --- src/format.js | 20 ++++++++++++---- src/validate.js | 2 +- test/format.test.js | 7 ++++++ test/validate.test.js | 56 ++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/format.js b/src/format.js index 1d50a31..5876682 100644 --- a/src/format.js +++ b/src/format.js @@ -319,6 +319,7 @@ function printAST(ast, options, level = 0, parentValue) { const unsupportedCats = [ ...Object.keys(ast.options).filter(o => !/^=\d+$/.test(o)) , ...sortedCats].filter(cat => !supportedCats.includes(cat)); if (add) { supportedCats.forEach(cat => { + // TODO: check actual locale plural categories if (!/^(fr|pt)/.test(locale) && cat === 'one' && ast.options['=1']) return; // don't create orphaned `one` // add missing supported categories ast.options[cat] ??= { ...(ast.options.other || ast.options.many || ast.options.few || Object.values(ast.options).at(-1)) }; @@ -330,11 +331,20 @@ function printAST(ast, options, level = 0, parentValue) { if (ast.options['=1'] && JSON.stringify(ast.options['=1']) === JSON.stringify(ast.options['one'])) { delete ast.options['=1']; } - } else if (ast.options['=1'] && /(? { diff --git a/test/format.test.js b/test/format.test.js index a213a2b..6d93c22 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -143,6 +143,13 @@ describe('formatMessage', () => { expect(formatted).to.equal(expected); }); + it(`should convert "=1" keys to "one" when the only differences from "other" are "s" or "es"`, async() => { + const message = `{a, plural, =1 {Some value} other {Some values}}`; + const expected = `{a, plural, one {Some value} other {Some values}}`; + const formatted = await formatMessage(message, { locale, sourceLocale: 'en', dedupe: true }); + expect(formatted).to.equal(expected); + }); + it(`should remove "=1" cases when they can be converted to a duplicate case with the "dedupe" option`, async() => { const message = `{a, plural, =1 {value 1} other {value {a}}}`; const expected = `{a, plural, other {value {a}}}`; diff --git a/test/validate.test.js b/test/validate.test.js index 0fff76e..b472d51 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -224,9 +224,59 @@ describe('validate', () => { expect(reporter.issues.length).to.equal(0); }); - it('generates a "nest-ideal" error with plural inside select', () => { - const sourceMessage = '{a, plural, one {} other {{b, select, other {}}}}'; - const targetMessage = '{a, plural, one {} other {{b, select, other {}}}}'; + it('generates a "nest-ideal" error with select inside plural', () => { + const sourceMessage = `{outcomes, select, + competencies {{num, plural, + one {Remove lower-level competency?} + other {Remove lower-level competencies?} + }} + expectations {{num, plural, + one {Remove lower-level expectation?} + other {Remove lower-level expectations?} + }} + objectives {{num, plural, + one {Remove lower-level objective?} + other {Remove lower-level objectives?} + }} + outcomes {{num, plural, + one {Remove lower-level outcome?} + other {Remove lower-level outcomes?} + }} + standards {{num, plural, + one {Remove lower-level standard?} + other {Remove lower-level standards?} + }} + other {{num, plural, + one {Remove lower-level standard?} + other {Remove lower-level standards?} + }} + }`; + const targetMessage = `{outcomes, select, + competencies {{num, plural, + one {Remove lower-level competency?} + other {Remove lower-level competencies?} + }} + expectations {{num, plural, + one {Remove lower-level expectation?} + other {Remove lower-level expectations?} + }} + objectives {{num, plural, + one {Remove lower-level objective?} + other {Remove lower-level objectives?} + }} + outcomes {{num, plural, + one {Remove lower-level outcome?} + other {Remove lower-level outcomes?} + }} + standards {{num, plural, + one {Remove lower-level standard?} + other {Remove lower-level standards?} + }} + other {{num, plural, + one {Remove lower-level standard?} + other {Remove lower-level standards?} + }} + }`; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); expect(reporter.issues.length).to.equal(1); From 40dca551ceb5ec6ef8655027c33ee6652f680217 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 26 Jun 2025 09:32:19 -0400 Subject: [PATCH 80/90] 3.0.0-beta.14 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5a0f6b4..adac5c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.13", + "version": "3.0.0-beta.14", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.13", + "version": "3.0.0-beta.14", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 8a8b285..b761b38 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.13", + "version": "3.0.0-beta.14", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From b8dff77f794c4c2f32ce6ff34361c877c4b7588c Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 12 Jan 2026 10:05:24 -0500 Subject: [PATCH 81/90] Fix erroneous =1 swapping --- src/format.js | 17 ++++++++++------- test/format.test.js | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/format.js b/src/format.js index 5876682..dc443bc 100644 --- a/src/format.js +++ b/src/format.js @@ -326,18 +326,14 @@ function printAST(ast, options, level = 0, parentValue) { }); } - if (ast.options.one) { - // `one` and `=1` are the same - if (ast.options['=1'] && JSON.stringify(ast.options['=1']) === JSON.stringify(ast.options['one'])) { - delete ast.options['=1']; - } - } else if (ast.options['=1']) { - if (/(? o.type === 0).map(o => o.value).join('').includes('1')) { // TODO: recursively check actual options // `=1` exists with literal "1" text ast.options.one = ast.options['=1']; delete ast.options['=1']; swapOne.add(ast.value); } else if (locale === sourceLocale && locale.startsWith('en')) { + // `=1` exists with basic Englished pluralization differences const enBasicPluralsRegEx = /e|s|es(?=\s|$|\p{P})/gv; if (printAST(ast.options.other.value, { locale, args }).replace(enBasicPluralsRegEx, '') === printAST(ast.options['=1'].value, { locale, args }).replace(enBasicPluralsRegEx, '')) { ast.options.one = ast.options['=1']; @@ -347,6 +343,13 @@ function printAST(ast, options, level = 0, parentValue) { } } + if (ast.options.one && !ast.offset) { + // `one` and `=1` are the same + if (ast.options['=1'] && JSON.stringify(ast.options['=1']) === JSON.stringify(ast.options['one'])) { + delete ast.options['=1']; + } + } + if (dedupe && ast.options.other) { const otherPrinted = printAST(ast.options.other.value, { locale, args }); Object.entries(ast.options).forEach(([k, v]) => { diff --git a/test/format.test.js b/test/format.test.js index 6d93c22..63db7ae 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -136,6 +136,20 @@ describe('formatMessage', () => { expect(formatted).to.equal(expected); }); + it(`should not convert "=1" keys to "one" by default`, async() => { + const message = `{a, plural, =1 {value}}`; + const expected = `{a, plural, =1 {value}}`; + const formatted = await formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + + it(`should not convert "=1" keys to "one" with an argument`, async() => { + const message = `{a, plural, =1 {{value}}}`; + const expected = `{a, plural, =1 {{value}}}`; + const formatted = await formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + it(`should convert "=1" keys to "one" when it contains a literal "1"`, async() => { const message = `{a, plural, =1 {value 1}}`; const expected = `{a, plural, one {value {a}}}`; @@ -143,6 +157,13 @@ describe('formatMessage', () => { expect(formatted).to.equal(expected); }); + it(`should not convert "=1" keys to "one" with an offset`, async() => { + const message = `{a, plural, offset:2 =1 {value 1}}`; + const expected = `{a, plural, offset:2 =1 {value 1}}`; + const formatted = await formatMessage(message, { locale, dedupe: true }); + expect(formatted).to.equal(expected); + }); + it(`should convert "=1" keys to "one" when the only differences from "other" are "s" or "es"`, async() => { const message = `{a, plural, =1 {Some value} other {Some values}}`; const expected = `{a, plural, one {Some value} other {Some values}}`; From 51939d21b4ac0fab36ee6c64cdbb448fa84e318d Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 12 Jan 2026 10:11:43 -0500 Subject: [PATCH 82/90] 3.0.0-beta.15 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index adac5c8..6372cb5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.14", + "version": "3.0.0-beta.15", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.14", + "version": "3.0.0-beta.15", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index b761b38..63efbd1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.14", + "version": "3.0.0-beta.15", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From f6364536817961a49e6fa64492cfcc669c0e216c Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 12 Jan 2026 10:22:33 -0500 Subject: [PATCH 83/90] Fix argument checking in additional plural categories --- src/format.js | 14 +------------- src/utils.js | 12 ++++++++++++ src/validate.js | 9 ++++++--- test/format.test.js | 2 +- test/validate.test.js | 17 +++++++++++++---- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/format.js b/src/format.js index dc443bc..adc8860 100644 --- a/src/format.js +++ b/src/format.js @@ -1,18 +1,6 @@ import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator.js'; import { parse } from '@formatjs/icu-messageformat-parser'; -import { getLocaleData, getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from './utils.js'; - -function getArgs(asts, types = [1,2,3,4,5,6,8], args = []) { - asts.forEach(ast => { - if (types.includes(ast.type)) { - if (ast.type === 8) args.push([ast.value, ast.type]); // account for closing tag - args.push([ast.value, ast.type]); - } - if (ast.options) Object.values(ast.options).map(({ value: asts }) => getArgs(asts, types, args)); - if (ast.children) args.concat(getArgs(ast.children, types, args)); - }) - return args; -} +import { getArgs, getLocaleData, getPluralCats, paddedQuoteLocales, sortedCats, structureRegEx } from './utils.js'; function expandASTHashes(ast, parentValue) { if (Array.isArray(ast)) { diff --git a/src/utils.js b/src/utils.js index f97914a..7d4b30e 100644 --- a/src/utils.js +++ b/src/utils.js @@ -2,6 +2,18 @@ import { env } from 'node:process'; import findConfig from 'find-config'; import localeData from './locale-data.js'; +export function getArgs(asts, types = [1,2,3,4,5,6,8], args = []) { + asts.forEach(ast => { + if (types.includes(ast.type)) { + if (ast.type === 8) args.push([ast.value, ast.type]); // account for closing tag + args.push([ast.value, ast.type]); + } + if (ast.options) Object.values(ast.options).map(({ value: asts }) => getArgs(asts, types, args)); + if (ast.children) args.concat(getArgs(ast.children, types, args)); + }) + return args; +} + export async function getConfig(cwd) { const configPath = findConfig('mfv.config.json', { cwd }); const config = configPath ? (await import(`file://${configPath}`, { with: { type: 'json' } }))?.default ?? {} : {}; diff --git a/src/validate.js b/src/validate.js index e416b91..a26abb5 100644 --- a/src/validate.js +++ b/src/validate.js @@ -1,4 +1,4 @@ -import { formatList, getPluralCats, sortedCats, structureRegEx } from './utils.js'; +import { formatList, getArgs, getPluralCats, sortedCats, structureRegEx } from './utils.js'; import { Reporter } from './reporter.js'; import { parse } from '@formatjs/icu-messageformat-parser'; @@ -164,11 +164,14 @@ export function validateMessage({ targetMessage, targetLocale, sourceMessage, so const targetMap = _map(targetTokens); const sourceMap = _map(sourceTokens); - const argDiff = Array.from(targetMap.arguments).filter(arg => !Array.from(sourceMap.arguments).includes(arg)); + const srcArgs = new Set(getArgs(sourceTokens).map(e => e.join(''))); + const tgtArgs = new Set(getArgs(targetTokens).map(e => e.join(''))); + + const argDiff = Array.from(tgtArgs).filter(arg => !Array.from(srcArgs).includes(arg)); const badArgPos = targetMessage.indexOf(argDiff[0]); if (argDiff.length) { - msgReporter.error('argument', `Unrecognized ${argDiff.length === 1 ? 'argument' : 'arguments'} ${formatList(argDiff.map(i => `"${i}"`))}. Source message uses ${formatList(Array.from(sourceMap.arguments).map(i => `"${i}"`))}.`, { column: badArgPos }); + msgReporter.error('argument', `Unrecognized ${argDiff.length === 1 ? 'argument' : 'arguments'} ${formatList(argDiff.map(i => `"${i.slice(0, -1)}"`))}. Source message uses ${formatList([...new Set([...srcArgs].map(a => a.slice(0, -1)))].map(i => `"${i}"`))}.`, { column: badArgPos }); } checkNbsp(targetMessage, msgReporter); diff --git a/test/format.test.js b/test/format.test.js index 63db7ae..0307431 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -167,7 +167,7 @@ describe('formatMessage', () => { it(`should convert "=1" keys to "one" when the only differences from "other" are "s" or "es"`, async() => { const message = `{a, plural, =1 {Some value} other {Some values}}`; const expected = `{a, plural, one {Some value} other {Some values}}`; - const formatted = await formatMessage(message, { locale, sourceLocale: 'en', dedupe: true }); + const formatted = await formatMessage(message, { locale, sourceLocale: 'en' }); expect(formatted).to.equal(expected); }); diff --git a/test/validate.test.js b/test/validate.test.js index b472d51..44e2e81 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -117,14 +117,23 @@ describe('validate', () => { // arg it('generates an "argument" error with unrecognized argument', () => { - const sourceMessage = 'An {arg}'; - const targetMessage = 'An {arG}'; + const sourceMessage = 'An {arg} {arg} {arg2, plural, one {{arg2}} other {}}'; + const targetMessage = 'An {arG} {arg} {arg2}'; reporter.config(targetMessage, sourceMessage, 'key'); validateMessage({ targetMessage, targetLocale, sourceMessage, sourceLocale }, reporter); - expect(reporter.issues.length).to.equal(1); + expect(reporter.issues.length).to.equal(3); expect(reporter.issues[0].type).to.equal('argument'); expect(reporter.issues[0].level).to.equal('error'); - expect(reporter.issues[0].msg).to.equal('Unrecognized argument "arG". Source message uses "arg".'); + expect(reporter.issues[0].msg).to.equal('Unrecognized argument "arG". Source message uses "arg" and "arg2".'); + }); + + it('does not generate an "argument" error with repeat arguments', () => { + const sourceMessage = '{groupAmount, plural, one {{arg1}} other {{arg1}}}'; + const targetMessage = '{groupAmount, plural, zero {{arg1}} one {{arg1}} two {{arg1}} few {{arg1}} many {{arg1}} other {{arg1}}}' + reporter.config(targetMessage, sourceMessage, 'key'); + validateMessage({ targetMessage, targetLocale: 'ar', sourceMessage, sourceLocale }, reporter); + console.log(reporter.issues); + expect(reporter.issues.length).to.equal(0); }); // brace From f34c7c09d9a041e3c513ac5b53c2aee2520ede35 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 12 Jan 2026 10:23:31 -0500 Subject: [PATCH 84/90] 3.0.0-beta.16 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6372cb5..110b93c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.15", + "version": "3.0.0-beta.16", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.15", + "version": "3.0.0-beta.16", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 63efbd1..98fdf6d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.15", + "version": "3.0.0-beta.16", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 44a595c3f16b8bdd78810fde472aebfe38e69802 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 4 Mar 2026 11:04:24 -0500 Subject: [PATCH 85/90] Improve apostrophe tests --- test/format.test.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/format.test.js b/test/format.test.js index 0307431..11e94fb 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -25,7 +25,7 @@ describe('formatMessage', () => { expect(formatted).to.equal(expected); }); - it(`should replace source quotes with "${locale}" quotes when "quotes" option is "source"`, async () => { + it(`should replace source quotes with "${locale}" quotes when "quotes" option is "source"`, async() => { const message = `This isn’t “correct”.`; const formatted = await formatMessage(message, { locale, sourceLocale: 'en', quotes: 'source' }); expect(formatted).to.equal(expected); @@ -37,14 +37,22 @@ describe('formatMessage', () => { }); }); + it('should identifiy apostrophes', async () => { + const message = `{evidenceTitle} collected {dateCollected} - {studentName}'s Portfolio`; + const expected = `{evidenceTitle} collected {dateCollected} - {studentName}’s Portfolio`; + const formatted = await formatMessage(message, { locale: 'en', sourceLocale: 'en' }); + expect(formatted).to.equal(expected); + }); + [ { condition: 'no options are set', options: {} }, { condition: 'the "quotes" option is "straight"', options: { quotes: 'straight' } } ].forEach(({ condition, options }) => { it(`should preserve escapes when ${condition}`, async() => { - const message = `An '{escaped}' argument`; + const message = `An '{escaped}' argument and an {argument}'s escape`; + const expected = `An '{escaped}' argument and an {argument}’s escape`; const formatted = await formatMessage(message, { locale: 'en', sourceLocale: 'en', ...options }); - expect(formatted).to.equal(message); + expect(formatted).to.equal(expected); }); }); From 30b1a5c776196f3a111971d1ec297f57654c36b7 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 4 Mar 2026 11:35:30 -0500 Subject: [PATCH 86/90] Fix apostrophe recognition/escaping --- src/format.js | 18 +++++------------- test/format.test.js | 12 ++++++++++++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/format.js b/src/format.js index adc8860..ee1d326 100644 --- a/src/format.js +++ b/src/format.js @@ -121,8 +121,6 @@ function printAST(ast, options, level = 0, parentValue) { trim = false, args = [], localeData, - isFirst = true, - isLast = true } = options; const localeLower = locale.toLowerCase(); @@ -145,8 +143,6 @@ function printAST(ast, options, level = 0, parentValue) { } } return printAST(ast, { ...options, - isFirst: !idx, - isLast: idx === arr.length - 1, swapOne: swapOneClone, trim }, @@ -248,6 +244,11 @@ function printAST(ast, options, level = 0, parentValue) { /* eslint-enable no-useless-escape */ } } + if (!quotes && localeData?.[locale]) { + const { delimiters } = localeData[locale]; + const apostrophe = delimiters.apostrophe ?? '\u2019'; + msg = msg.replace(/'/g, apostrophe); + } return msg.replace(/\|_escape_\|/g, "'"); } @@ -258,15 +259,6 @@ function printAST(ast, options, level = 0, parentValue) { if (type === 0) { // straight text let escaped = ast.value; - // If this literal starts with a ' and its not the 1st node, this means the node before it is non-literal - // and the `'` needs to be unescaped - if (!isFirst && escaped[0] === "'") { - escaped = "''".concat(escaped.slice(1)); - } - // Same logic but for last el - if (!isLast && escaped[escaped.length - 1] === "'") { - escaped = "".concat(escaped.slice(0, escaped.length - 1), "''"); - } escaped = escaped.replace(/'([{}](?:.*[{}])?)'/gsu, "|_escape_|$1|_escape_|"); escaped = parentValue ? escaped.replace("'#'", "|_escape_|#|_escape_|") : escaped; diff --git a/test/format.test.js b/test/format.test.js index 11e94fb..ab7efbd 100644 --- a/test/format.test.js +++ b/test/format.test.js @@ -56,6 +56,18 @@ describe('formatMessage', () => { }); }); + it('should convert apostrophes in plain text without arguments', async() => { + const message = `It's a nice day`; + const formatted = await formatMessage(message, { locale: 'en' }); + expect(formatted).to.equal(`It’s a nice day`); + }); + + it('should convert apostrophes followed by whitespace', async() => { + const message = `{students}' assignments' titles`; + const formatted = await formatMessage(message, { locale: 'en' }); + expect(formatted).to.equal(`{students}’ assignments’ titles`); + }); + [ { locale: 'ar', expected: `{a, plural, From 451cb8ba27de22bca0b6672f12aa2d7a2c30f0ba Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 4 Mar 2026 12:46:37 -0500 Subject: [PATCH 87/90] Fic double backslash value parsing --- src/validate.js | 4 ++-- test/validate.test.js | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/validate.js b/src/validate.js index a26abb5..37d03e8 100644 --- a/src/validate.js +++ b/src/validate.js @@ -224,9 +224,9 @@ export function parseLocales(locales, useJSONObj) { }; const regex = useJSONObj - ? /("(?.*)"(\s*):(\s*)\{)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/gv + ? /("(?.*)"(\s*):(\s*)\{)*\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(?:[^\\]|\\[\s\S])*?)\k(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/gv //[ ][ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] - : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(.|\n)*?)(?(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/gv; + : /\s+(?["'`]?)(?.*?)\k(?\s*):(?\s*)(?["'`])(?(?:[^\\]|\\[\s\S])*?)\k(?!\s?([\w\p{L}]|\k))(?,?)(?.*)/gv; //[ ][ " ][ key ][ " ][ ][:][ ][ " ][ value ][ " ][ , ][ // comment ] const matches = Array.from(contents.matchAll(regex)); diff --git a/test/validate.test.js b/test/validate.test.js index 44e2e81..94273b1 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -429,4 +429,21 @@ describe('validate', () => { }); }); + + describe('parseLocales', () => { + + it('correctly parses a value that is a double backslash (\\\\)', () => { + const locales = parseLocales([{ + file: 'en.json', + contents: `{ + "intl-common:characters:backslash": "\\\\", // short name or description of the "\\" character +}` + }]); + + expect(locales['en'].parsed['intl-common:characters:backslash']).to.exist; + expect(locales['en'].parsed['intl-common:characters:backslash'].val).to.equal('\\\\'); + expect(locales['en'].parsed['intl-common:characters:backslash'].comment).to.equal(' // short name or description of the "\\" character'); + }); + + }); }); From dc50a149051c435e4489a444580602be8c0f97d8 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 4 Mar 2026 23:04:55 -0500 Subject: [PATCH 88/90] 3.0.0-beta.17 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 110b93c..4c23b86 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.16", + "version": "3.0.0-beta.17", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "messageformat-validator", - "version": "3.0.0-beta.16", + "version": "3.0.0-beta.17", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 98fdf6d..54a13da 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "messageformat-validator", - "version": "3.0.0-beta.16", + "version": "3.0.0-beta.17", "description": "Validates that ICU MessageFormat messages are well-formed, and that translated target messages are compatible with their source.", "type": "module", "repository": { From 7d6d263fd03137bd69287540226149aeba37c199 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 5 Mar 2026 12:24:01 -0500 Subject: [PATCH 89/90] Update README --- README.md | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 54eef57..70637db 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,11 @@ Output the `myMessage` message from the `es-es` file with all messageformat stru mfv -l es-es -p lang/ highlight myMessage ``` +Re-write messages to a standard format, using newlines and tabs for complex arguments, and removing any plural categories that are invalid for each locale (e.g. `zero` for English, `one` for Korean) +```shell +mfv -s en -p lang/ format -nr +``` + ### Options: `-V, --version` - output the version number @@ -49,7 +54,7 @@ mfv -l es-es -p lang/ highlight myMessage `--json-obj` - Indicate that the files to be parsed are JSON files with keys that have objects for values with their own keys: `translation` and `context` -`-h, --help` - display help for command +`-h, --help` - Display help for command ### Subommands: @@ -63,7 +68,7 @@ mfv -l es-es -p lang/ highlight myMessage `highlight ` - Output a message with all non-translatable ICU MessageFormat structure highlighted -`help [command]` - display help for command +`format` - Rewrite messages to a standard format ## Config File @@ -116,29 +121,3 @@ Some options can be configured with default values in `mfv.config.json` `split` - Split by a complex argument `untranslated` - Message has not been translated - - -## Overrides - -You can mark individual messages as - -`mfv override fr option` - -A global list of overrides is pre-loaded: - -v Expand Me - -## v3 - -- Always throws on error. The `--throw-errors` option has been removed. -- The `locales` option now takes an array when in the config files -- New `format` subcommand rewrites messages to a standard format -- Issue types renamed: - - `case` -> `option` - - `nest` -> `nest-source` and `nest-ideal` - - `duplicate-keys` -> `duplicate` - - `plural-key` -> `category` - - `categories` -> `category-missing` - - `source-error` -> `source` -- New issue types - - `option-missing` From 8dc342072dcebbc2068d8b0aa4ab68207115cf3b Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 5 Mar 2026 12:48:28 -0500 Subject: [PATCH 90/90] Remove rebase artifact --- test/validate.test.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/validate.test.js b/test/validate.test.js index 94273b1..4d68dc2 100644 --- a/test/validate.test.js +++ b/test/validate.test.js @@ -1,4 +1,4 @@ -import { parseLocales, structureRegEx, validateLocales, validateMessage } from '../src/validate.js'; +import { parseLocales, validateLocales, validateMessage } from '../src/validate.js'; import { Reporter } from '../src/reporter.js'; import { expect } from 'chai'; @@ -11,8 +11,6 @@ describe('validate', () => { reporter = new Reporter(targetLocale); }); - describe('structureRegEx', () => { - describe('validateMessage', () => { // untranslated