From b7244998a1555c0a0e9767279b60161ee64595b2 Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Tue, 18 Jun 2024 12:34:17 +0200 Subject: [PATCH] Few fixes to `ftl-to-json` and `localize-ftl` scripts (#3707) * Omit msg-ref-only strings from Transifex JSON * Fix msg-ref-only strings not included in translated .ftl files * Update ftl-tx. Simplify localize-ftl script. * Tweak FTL -> JSON conversion to produce a single file --- js-build/ftl-to-json.mjs | 33 +++++++++++------ js-build/localize-ftl.mjs | 78 ++++++++++++++++++++------------------- js-build/utils.js | 2 +- package-lock.json | 14 +++---- package.json | 2 +- 5 files changed, 71 insertions(+), 58 deletions(-) diff --git a/js-build/ftl-to-json.mjs b/js-build/ftl-to-json.mjs index c0326038a2..7defc1ba8d 100644 --- a/js-build/ftl-to-json.mjs +++ b/js-build/ftl-to-json.mjs @@ -6,25 +6,36 @@ import { ftlFileBaseNames as sourceFileBaseNames } from './config.js'; import { onError, onProgress, onSuccess } from './utils.js'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const TRANSIFEX_FILE_NAME = 'zotero.json'; async function getJSON() { const t1 = performance.now(); const sourceDir = join(ROOT, 'chrome', 'locale', 'en-US', 'zotero'); + const destFile = join(sourceDir, TRANSIFEX_FILE_NAME); + let messagesMap = new Map(); + for (let sourceFileBaseName of sourceFileBaseNames) { const sourceFile = join(sourceDir, sourceFileBaseName + '.ftl'); - const destFile = join(sourceDir, sourceFileBaseName + '.json'); const ftl = await fs.readFile(sourceFile, 'utf8'); - const json = ftlToJSON(ftl, { transformTerms: false, storeTermsInJSON: false }); - await fs.outputJSON(destFile, json, { spaces: '\t' }); - onProgress(destFile, destFile, 'json'); - } - const t2 = performance.now(); - return ({ - action: 'ftl->json', - count: sourceFileBaseNames.length, - totalCount: sourceFileBaseNames.length, - processingTime: t2 - t1 + const json = ftlToJSON(ftl, { transformTerms: false, storeTermsInJSON: false, skipRefOnly: true }); + Object.entries(json).forEach(([key, value]) => { + if (messagesMap.has(key)) { + throw new Error(`Duplicate key: ${key} found in file ${sourceFileBaseName}.ftl`); + } + messagesMap.set(key, value); }); + onProgress(`${sourceFileBaseName}.ftl`, TRANSIFEX_FILE_NAME, 'ftl->json'); + } + + const messagesJSON = Object.fromEntries(messagesMap); + await fs.outputJSON(destFile, messagesJSON, { spaces: '\t' }); + const t2 = performance.now(); + return ({ + action: 'ftl->json', + count: sourceFileBaseNames.length, + totalCount: sourceFileBaseNames.length, + processingTime: t2 - t1 + }); } if (process.argv[1] === fileURLToPath(import.meta.url)) { diff --git a/js-build/localize-ftl.mjs b/js-build/localize-ftl.mjs index 4f8e987f32..745dfca5d2 100644 --- a/js-build/localize-ftl.mjs +++ b/js-build/localize-ftl.mjs @@ -1,15 +1,13 @@ -import { extractTerms, ftlToJSON, JSONToFtl } from 'ftl-tx'; +import { ftlToJSON, JSONToFtl } from 'ftl-tx'; import fs from 'fs-extra'; import { dirname, join } from 'path'; import { fileURLToPath } from 'url'; import { ftlFileBaseNames as sourceFileBaseNames } from './config.js'; import { onError, onProgress, onSuccess } from './utils.js'; -import { exit } from 'process'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); const localesDir = join(ROOT, 'chrome', 'locale'); -const sourceDir = join(localesDir, 'en-US', 'zotero'); -const termsSourceFTLPath = join(ROOT, 'app', 'assets', 'branding', 'locale', 'brand.ftl'); +const TRANSIFEX_FILE_NAME = 'zotero.json'; function getLocaleDir(locale) { return join(localesDir, locale, 'zotero'); @@ -18,13 +16,6 @@ function getLocaleDir(locale) { async function getFTL() { const t1 = performance.now(); - if (!(await fs.pathExists(termsSourceFTLPath))) { - console.error(`Required file ${termsSourceFTLPath} does not exist`); - exit(1); - } - - const terms = extractTerms(await fs.readFile(termsSourceFTLPath, 'utf-8')); - const foundLocales = (await fs.readdir(localesDir, { withFileTypes: true })) .filter(dirent => dirent.isDirectory()) .map(dirent => dirent.name) @@ -32,54 +23,65 @@ async function getFTL() { .filter(name => /^[a-z]{2}(-[A-Z]{2})?$/.test(name)); let count = 0; - for (let sourceFileBaseName of sourceFileBaseNames) { - const fallbackJSONPath = join(sourceDir, sourceFileBaseName + '.json'); - if (!(await fs.pathExists(fallbackJSONPath))) { - console.error(`File ${fallbackJSONPath} does not exist -- please run 'ftl-to-json' first`); - exit(1); + for (let locale of foundLocales) { + // Skip source locale + if (locale == 'en-US') { + continue; } - - const fallbackJSON = await fs.readJSON(fallbackJSONPath); - - for (let locale of foundLocales) { - // Skip source locale - if (locale == 'en-US') { - continue; - } - + + const jsonFilePath = join(getLocaleDir(locale), TRANSIFEX_FILE_NAME); + let jsonFromTransifex = {}; + try { + const json = await fs.readJSON(jsonFilePath); + jsonFromTransifex = json; + } + catch (e) { + // no .json file from transifex + } + + for (let sourceFileBaseName of sourceFileBaseNames) { const ftlFilePath = join(getLocaleDir(locale), sourceFileBaseName + '.ftl'); let jsonFromLocalFTL = {}; try { const ftl = await fs.readFile(ftlFilePath, 'utf8'); - jsonFromLocalFTL = ftlToJSON(ftl, { transformTerms: false, storeTermsInJSON: false }); + jsonFromLocalFTL = ftlToJSON(ftl); } catch (e) { // no local .ftl file } - - const jsonFilePath = join(getLocaleDir(locale), sourceFileBaseName + `.json`); - let jsonFromTransifex = {}; + let jsonFromEnUSFTL = {}; try { - const json = await fs.readJSON(jsonFilePath); - jsonFromTransifex = json; + const enUSFtlPath = join(getLocaleDir('en-US'), sourceFileBaseName + '.ftl'); + const ftl = await fs.readFile(enUSFtlPath, 'utf8'); + jsonFromEnUSFTL = ftlToJSON(ftl); } catch (e) { - // no .json file from transifex + console.warn(`No en-US .ftl file for ${sourceFileBaseName}.`); } - - const mergedJSON = { ...fallbackJSON, ...jsonFromLocalFTL, ...jsonFromTransifex }; - const ftl = JSONToFtl(mergedJSON, { addTermsToFTL: false, storeTermsInJSON: false, transformTerms: false, terms }); - + const mergedSourceJSON = { ...jsonFromEnUSFTL, ...jsonFromLocalFTL }; + const sourceKeys = Object.keys(mergedSourceJSON); + const translated = new Map(); + + for (let key of sourceKeys) { + if (key in jsonFromTransifex) { + translated.set(key, jsonFromTransifex[key]); + } + else { + translated.set(key, mergedSourceJSON[key]); + } + } + + const ftl = JSONToFtl(Object.fromEntries(translated)); const outFtlPath = join(getLocaleDir(locale), sourceFileBaseName + '.ftl'); await fs.outputFile(outFtlPath, ftl); - onProgress(outFtlPath, outFtlPath, 'ftl'); + onProgress(`${locale}/${sourceFileBaseName}.ftl`, null, 'localize'); count++; } } const t2 = performance.now(); return ({ - action: 'ftl', + action: 'localize', count, totalCount: count, processingTime: t2 - t1 diff --git a/js-build/utils.js b/js-build/utils.js index d2a15c0dc3..2e286737b6 100644 --- a/js-build/utils.js +++ b/js-build/utils.js @@ -30,7 +30,7 @@ function onProgress(sourcefile, outfile, operation) { if ('isError' in global && global.isError) { return; } - if (NODE_ENV == 'debug') { + if (NODE_ENV === 'debug' && outfile) { console.log(`${colors.blue(`[${operation}]`)} ${sourcefile} -> ${outfile}`); } else { diff --git a/package-lock.json b/package-lock.json index 7bc7b45a3e..648fefdab1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,7 +40,7 @@ "eslint-plugin-react": "^7.28.0", "eslint-plugin-react-hooks": "^4.0.4", "fs-extra": "^3.0.1", - "ftl-tx": "^0.6.0", + "ftl-tx": "^0.9.0", "globby": "^6.1.0", "jspath": "^0.4.0", "mocha": "^10.4.0", @@ -3667,9 +3667,9 @@ } }, "node_modules/ftl-tx": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/ftl-tx/-/ftl-tx-0.6.0.tgz", - "integrity": "sha512-w7s0p6RNsaUKpiuQZ5Q0KIyiX2JIDItqI4qRKt49u0aMqdGp0gDc5G5Qnubgu6l2ww5PDt4wsyL7A1iZo1VGMQ==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/ftl-tx/-/ftl-tx-0.9.0.tgz", + "integrity": "sha512-cxjOfLulCEPL6K0boxTiHCYNs3UahgmfrC+u3a6oV665EWuR6LnZRWID2UKWv6JBIB4YU4k06Y5re5uA7HpEGg==", "dev": true, "dependencies": { "@fluent/syntax": "^0.19.0" @@ -10588,9 +10588,9 @@ } }, "ftl-tx": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/ftl-tx/-/ftl-tx-0.6.0.tgz", - "integrity": "sha512-w7s0p6RNsaUKpiuQZ5Q0KIyiX2JIDItqI4qRKt49u0aMqdGp0gDc5G5Qnubgu6l2ww5PDt4wsyL7A1iZo1VGMQ==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/ftl-tx/-/ftl-tx-0.9.0.tgz", + "integrity": "sha512-cxjOfLulCEPL6K0boxTiHCYNs3UahgmfrC+u3a6oV665EWuR6LnZRWID2UKWv6JBIB4YU4k06Y5re5uA7HpEGg==", "dev": true, "requires": { "@fluent/syntax": "^0.19.0" diff --git a/package.json b/package.json index 9dcf545905..832284bb58 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "eslint-plugin-react": "^7.28.0", "eslint-plugin-react-hooks": "^4.0.4", "fs-extra": "^3.0.1", - "ftl-tx": "^0.6.0", + "ftl-tx": "^0.9.0", "globby": "^6.1.0", "jspath": "^0.4.0", "mocha": "^10.4.0",