Issue 6171 - create CSV exporter and importer for translations

build/csv-export.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-present eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-/* globals require, process */
+/* globals process */
 
 "use strict";
 
 const fs = require("fs");
 const path = require("path");
 const csv = require("csv");
 const {promisify} = require("util");
 const execFile = promisify(require("child_process").execFile);
 const csvParser = promisify(csv.parse);
-const readDir = promisify(fs.readdir);
 const readFile = promisify(fs.readFile);
+const writeFile = promisify(fs.writeFile);
 const glob = promisify(require("glob").glob);
-const readJsonPromised = promisify(readJson);
 
 const localesDir = "locale";
 const defaultLocale = "en_US";
 
-let headers = ["filename", "StringID", "Description", "Placeholders",
-               defaultLocale];
-let outputFileName = "translations-{repo}-{hash}.csv";
+const headers = ["Filename", "StringID", "Description", "Placeholders",
+                 defaultLocale];
+let outputFileName = "translations.csv";
 
 /**
  * Export existing translation - files into CSV file
- * @param  {string[]} [filesFilter] - fileNames filter, if omitted all files
- *                                    will be exported
- */
-function exportTranslations(filesFilter)
-{
-  let mercurialCommands = [];
-  // Get Hash
-  mercurialCommands.push(execFile("hg", ["id", "-i"]));

[saroyanm, 2018/05/15 18:59:49]

This will not work in git.

Alternative:   mercurialCommands.push(execFile("git", ["rev-parse", "HEAD"]));

[saroyanm, 2018/05/17 17:27:37]

As discussed: We better remove this mercurial specific logic and introduce it
back if we think we really need it.

-  // Get repo path
-  mercurialCommands.push(execFile("hg", ["paths", "default"]));

[saroyanm, 2018/05/15 18:59:49]

This will not work in git.

Alternative: mercurialCommands.push(execFile("git", ["config", "--get",
"remote.origin.url"]));

-  Promise.all(mercurialCommands).then((outputs) =>
-  {
-    // Remove line endings and "+" sign from the end of the hash
-    let [hash, filePath] = outputs.map((item) =>
-                           item.stdout.replace(/\+\n|\n$/, ""));
-    // Update name of the file to be output
-    outputFileName = outputFileName.replace("{hash}", hash);
-    outputFileName = outputFileName.replace("{repo}", path.basename(filePath));
-
-    // Read all available locales and default files
-    return glob(`${localesDir}/**/*.json`, {});
-  }).then((filePaths) =>
+ */
+function exportTranslations()
+{
+  glob(`${localesDir}/**/*.json`).then((filePaths) =>
   {
     // Reading all existing translations files
-    return Promise.all(filePaths.map((filePath) => readJsonPromised(filePath)));
+    return Promise.all(filePaths.map((filePath) => readJson(filePath)));
   }).then(csvFromJsonFileObjects);
 }
 
 /**
  * Creating Matrix which reflects output CSV file
- * @param  {Array} fileObjects - array of file objects created by readJson
+ * @param  {Object[]} fileObjects - array of file objects created by readJson
  */
 function csvFromJsonFileObjects(fileObjects)
 {
-  let locales = [];
+  const locales = [];
+  const fileNames = [];
   // Create Object tree from the Objects array, for easier search
   // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
-  let dataTreeObj = fileObjects.reduce((accumulator, fileObject) =>
-  {
-    if (!fileObject)
-      return accumulator;
-
-    let {fileName, locale} = fileObject;
-    if (!locales.includes(locale))
+  const dataTreeObj = Object.create(null);
+  for (const fileObject of fileObjects)
+  {
+    const {fileName, locale, strings} = fileObject;
+
+    if (locale != defaultLocale && !locales.includes(locale))
       locales.push(locale);
 
-    if (!accumulator[fileName])
-    {
-      accumulator[fileName] = {};
-    }
-    accumulator[fileName][locale] = fileObject.strings;
-    return accumulator;
-  }, {});
-
-  let fileNames = Object.keys(dataTreeObj);
-  if (filesFilter.length)
-    fileNames = fileNames.filter((item) => filesFilter.includes(item));
-
-  locales = locales.filter((locale) => locale != defaultLocale).sort();
+    if ((!filesFilter.length || filesFilter.includes(fileName)) &&
+         !fileNames.includes(fileName))
+      fileNames.push(fileName);
+
+    if (!(fileName in dataTreeObj))
+      dataTreeObj[fileName] = Object.create(null);
+
+    dataTreeObj[fileName][locale] = strings;
+  }
   // Create two dimensional strings array that reflects CSV structure
-  let csvArray = [headers.concat(locales)];
-  for (let fileName of fileNames)
-  {
-    let strings = dataTreeObj[fileName][defaultLocale];
-    for (let stringID of Object.keys(strings))
-    {
-      let fileObj = dataTreeObj[fileName];
-      let {description, message, placeholders} = strings[stringID];
-      let row = [fileName, stringID, description || "",
-                 JSON.stringify(placeholders), message];
-
-      for (let locale of locales)
-      {
-        let localeFileObj = fileObj[locale];
-        let isTranslated = !!(localeFileObj && localeFileObj[stringID]);
+  const csvArray = [headers.concat(locales)];
+  for (const fileName of fileNames)
+  {
+    const strings = dataTreeObj[fileName][defaultLocale];
+    for (const stringID of Object.keys(strings))
+    {
+      const fileObj = dataTreeObj[fileName];
+      const {description, message, placeholders} = strings[stringID];
+      const row = [fileName, stringID, description || "",
+                   JSON.stringify(placeholders), message];
+
+      for (const locale of locales)
+      {
+        const localeFileObj = fileObj[locale];
+        const isTranslated = !!(localeFileObj && localeFileObj[stringID]);
         row.push(isTranslated ? localeFileObj[stringID].message : "");
       }
       csvArray.push(row);
     }
   }
   arrayToCsv(csvArray);
 }
 
 /**
  * Import strings from the CSV file
  * @param  {string} filePath - CSV file path to import from
  */
 function importTranslations(filePath)
 {
   readFile(filePath, "utf8").then((fileObjects) =>
   {
     return csvParser(fileObjects);
   }).then((dataMatrix) =>
   {
-    let headLocales = dataMatrix.shift().slice(4);
-    let dataTreeObj = {};
-    for (let rowId in dataMatrix)
-    {
-      let row = dataMatrix[rowId];
+    const headLocales = dataMatrix.shift().slice(4);
+    const dataTreeObj = {};
+    for (const rowId in dataMatrix)
+    {
+      const row = dataMatrix[rowId];
       let [currentFilename, stringId, description, placeholder, ...messages] =
         row;
       if (!stringId)
         continue;
 
       stringId = stringId.trim();
       // Check if it's the filename row
       if (!dataTreeObj[currentFilename])
         dataTreeObj[currentFilename] = {};
 
       description = description.trim();
       for (let i = 0; i < headLocales.length; i++)
       {
-        let locale = headLocales[i].trim();
-        let message = messages[i].trim();
+        const locale = headLocales[i].trim();
+        const message = messages[i].trim();
         if (!message)
           continue;
 
         // Create Object tree from the Objects array, for easier search
         // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
         if (!dataTreeObj[currentFilename][locale])
           dataTreeObj[currentFilename][locale] = {};
 
-        let localeObj = dataTreeObj[currentFilename][locale];
+        const localeObj = dataTreeObj[currentFilename][locale];
         localeObj[stringId] = {};
-        let stringObj = localeObj[stringId];
+        const stringObj = localeObj[stringId];
 
         // We keep string descriptions only in default locale files
-        if (locale == defaultLocale)
+        if (locale == defaultLocale && description)
           stringObj.description = description;
 
         stringObj.message = message;
         if (placeholder)
           stringObj.placeholders = JSON.parse(placeholder);
       }
     }
     writeJson(dataTreeObj);
   });
 }
 
 /**
  * Write locale files according to dataTreeObj
  * @param  {Object} dataTreeObj - ex.:
  * {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
  */
 function writeJson(dataTreeObj)
 {
-  for (let fileName in dataTreeObj)
-  {
-    for (let locale in dataTreeObj[fileName])
-    {
-      let filePath = path.join(localesDir, locale, fileName);
-      let orderedJSON = orderJSON(dataTreeObj[fileName][locale]);
-      let fileString = JSON.stringify(orderedJSON, null, 2);
+  for (const fileName in dataTreeObj)
+  {
+    for (const locale in dataTreeObj[fileName])
+    {
+      const filePath = path.join(localesDir, locale, fileName);
+      const sortedJson = sortJson(dataTreeObj[fileName][locale]);
+      let fileString = JSON.stringify(sortedJson, null, 2);
 
       // Newline at end of file to match Coding Style
       if (locale == defaultLocale)
         fileString += "\n";
-      fs.writeFile(filePath, fileString, "utf8", (err) =>
-      {
-        if (err)
-        {
-          console.error(err);
-        }
-        else
-        {
-          console.log(`Updated: ${filePath}`);

[saroyanm, 2018/05/04 13:56:27]

console.log doesn't pass eslint -> Unexpected console statement. (no-console)

This sounds more like an exceptional case for the csv-exporter that we would
like to have those outputs.

[Thomas Greiner, 2018/05/22 17:22:49]

Agreed, feel free to use "eslint-disable" to ignore this rule for this line.

[saroyanm, 2018/06/05 15:03:46]

Done.

-        }
+      writeFile(filePath, fileString, "utf8").then(() =>
+      {
+        console.log(`Updated: ${filePath}`); // eslint-disable-line no-console
+      }).catch((err) =>
+      {
+        console.error(err);
       });
     }
   }
 }
 
 /**
- * This function currently rely on nodeJS to sort the object by keys
+ * This function currently relies on V8 to sort the object by keys
  * @param {Object} unordered - json object
  * @returns {Object}
  */
-function orderJSON(unordered)
+function sortJson(unordered)
 {
   const ordered = {};
-  for (let key of Object.keys(unordered).sort())
+  for (const key of Object.keys(unordered).sort())
   {
     ordered[key] = unordered[key];
     if (unordered[key].placeholders)
-      ordered[key].placeholders = orderJSON(unordered[key].placeholders);
+      ordered[key].placeholders = sortJson(unordered[key].placeholders);
 
     ordered[key] = unordered[key];
   }
   return ordered;
 }
 
 /**
  * Convert two dimensional array to the CSV file
- * @param  {Array} csvArray - array to convert from
+ * @param  {Object[]} csvArray - array to convert from
  */
 function arrayToCsv(csvArray)
 {
   csv.stringify(csvArray, (err, output) =>
   {
-    fs.writeFile(outputFileName, output, "utf8", (error) =>
-    {
-      if (!error)
-        console.log(`${outputFileName} is created`);
-      else
-        console.error(error);
+    writeFile(outputFileName, output, "utf8").then(() =>
+    {
+      // eslint-disable-next-line no-console
+      console.log(`${outputFileName} is created`);
+    }).catch((error) =>
+    {
+      console.error(error);
     });
   });
 }
 
 /**
  * Reads JSON file and assign filename and locale to it
  * @param {string} filePath - ex.: "locales/en_US/desktop-options.json"
- * @param {function} callback - fileName, locale and strings of locale file
- *                   Parameters:
- *                     * Error message
- *                     * Object containing fileName, locale and strings
- */
-function readJson(filePath, callback)
-{
-  let {dir, base} = path.parse(filePath);
-  fs.readFile(filePath, "utf8", (err, data) =>
-  {
-    if (err)
-    {
-      callback(err);
-    }
-    else
-    {
-      callback(null, {fileName: base, locale: dir.split("/").pop(),
-        strings: JSON.parse(data)});
-    }
+ * @returns {Promise<Object>} resolves fileName, locale and strings of the
+ *                            locale file
+ */
+function readJson(filePath)
+{
+  return readFile(filePath, "utf8").then((data) =>
+  {
+    const {dir, base} = path.parse(filePath);
+    const locale = dir.split(path.sep).pop();
+    const strings = JSON.parse(data);
+    return {fileName: base, locale, strings};
   });
 }
 
 /**
  * Exit process and log error message
  * @param {String} error error message
  */
 function exitProcess(error)
 {
   console.error(error);
-  process.exit();
+  process.exit(1);
 }
 
 // CLI
-let helpText = `
+const helpText = `
 About: Converts locale files between CSV and JSON formats
 Usage: csv-export.js [option] [argument]
 Options:
   -f [FILENAME]         Name of the files to be exported ex.: -f firstRun.json
                         option can be used multiple times.
                         If omitted all files are being exported
 
   -o [FILENAME]         Output filename ex.:
-                        -f firstRun.json -o {hash}-firstRun.csv
-                        Placeholders:
-                          {hash} - Mercurial current revision hash
-                          {repo} - Name of the "Default" repository
+                        -f firstRun.json -o firstRun.csv
                         If omitted the output fileName is set to
-                        translations-{repo}-{hash}.csv
+                        translations.csv
 
   -i [FILENAME]         Import file path ex: -i issue-reporter.csv
 `;
 
-let argv = process.argv.slice(2);
+const argv = process.argv.slice(2);
 let stopExportScript = false;
 // Filter to be used export to the fileNames inside
-let filesFilter = [];
+const filesFilter = [];
 
 for (let i = 0; i < argv.length; i++)
 {
   switch (argv[i])

[saroyanm, 2018/05/04 13:51:11]

It would be great to use a library like 
[minimist](https://www.npmjs.com/package/minimist) to handle this, but as this
implementation might change
[here](https://gitlab.com/eyeo/adblockplus/adblockplusui/issues/74) I think we
can tackle that separately?

[Thomas Greiner, 2018/05/07 15:16:39]

Agreed.

   {
     case "-h":
-      console.log(helpText);
+      console.log(helpText); // eslint-disable-line no-console
       stopExportScript = true;
       break;
     case "-f":
-      // check if argument following option is specified
       if (!argv[i + 1])
       {
         exitProcess("Please specify the input filename");
       }
-      else
-      {
-        filesFilter.push(argv[i + 1]);
-      }
+      filesFilter.push(argv[i + 1]);
       break;
     case "-o":
       if (!argv[i + 1])
       {
         exitProcess("Please specify the output filename");
       }
-      else
-      {
-        outputFileName = argv[i + 1];
-      }
+      outputFileName = argv[i + 1];
       break;
     case "-i":
       if (!argv[i + 1])
       {
         exitProcess("Please specify the import file");
       }
-      else
-      {
-        let importFile = argv[i + 1];
-        importTranslations(importFile);
-        stopExportScript = true;
-      }
+      const importFile = argv[i + 1];
+      importTranslations(importFile);
+      stopExportScript = true;
       break;
   }
 }
 
 if (!stopExportScript)
   exportTranslations(filesFilter);