Issue 6171 - create CSV exporter and importer for translations

build/csv-export.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-present eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-/* globals require, process */
+/* globals process */
 
 "use strict";
 
 const fs = require("fs");
 const path = require("path");
 const csv = require("csv");
 const {promisify} = require("util");
 const execFile = promisify(require("child_process").execFile);
 const csvParser = promisify(csv.parse);
-const readDir = promisify(fs.readdir);
 const readFile = promisify(fs.readFile);
 const glob = promisify(require("glob").glob);
 const readJsonPromised = promisify(readJson);
 
 const localesDir = "locale";
 const defaultLocale = "en_US";
 
-let headers = ["filename", "StringID", "Description", "Placeholders",
+let headers = ["Filename", "StringID", "Description", "Placeholders",
                defaultLocale];
 let outputFileName = "translations-{repo}-{hash}.csv";
 
 /**
  * Export existing translation - files into CSV file
- * @param  {string[]} [filesFilter] - fileNames filter, if omitted all files
- *                                    will be exported
- */
-function exportTranslations(filesFilter)
+ */
+function exportTranslations()
 {
   let mercurialCommands = [];
   // Get Hash
   mercurialCommands.push(execFile("hg", ["id", "-i"]));

[saroyanm, 2018/05/15 18:59:49]

This will not work in git.

Alternative:   mercurialCommands.push(execFile("git", ["rev-parse", "HEAD"]));

[saroyanm, 2018/05/17 17:27:37]

As discussed: We better remove this mercurial specific logic and introduce it
back if we think we really need it.

   // Get repo path
   mercurialCommands.push(execFile("hg", ["paths", "default"]));

[saroyanm, 2018/05/15 18:59:49]

This will not work in git.

Alternative: mercurialCommands.push(execFile("git", ["config", "--get",
"remote.origin.url"]));

   Promise.all(mercurialCommands).then((outputs) =>
   {
     // Remove line endings and "+" sign from the end of the hash
-    let [hash, filePath] = outputs.map((item) =>
-                           item.stdout.replace(/\+\n|\n$/, ""));
+    let [hash, filePath] = outputs.map((output) =>
+                           output.stdout.replace(/\+\n|\n$/, ""));
     // Update name of the file to be output
     outputFileName = outputFileName.replace("{hash}", hash);
     outputFileName = outputFileName.replace("{repo}", path.basename(filePath));
 
     // Read all available locales and default files
-    return glob(`${localesDir}/**/*.json`, {});
+    return glob(`${localesDir}/**/*.json`);
   }).then((filePaths) =>
   {
     // Reading all existing translations files
     return Promise.all(filePaths.map((filePath) => readJsonPromised(filePath)));
   }).then(csvFromJsonFileObjects);
 }
 
 /**
  * Creating Matrix which reflects output CSV file
- * @param  {Array} fileObjects - array of file objects created by readJson
+ * @param  {Object[]} fileObjects - array of file objects created by readJson
  */
 function csvFromJsonFileObjects(fileObjects)
 {
   let locales = [];
   // Create Object tree from the Objects array, for easier search
   // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
-  let dataTreeObj = fileObjects.reduce((accumulator, fileObject) =>
-  {
-    if (!fileObject)
-      return accumulator;
-
-    let {fileName, locale} = fileObject;
+  let dataTreeObj = Object.create(null);
+  for (let fileObject of fileObjects)
+  {
+    const {fileName, locale, strings} = fileObject;
+
     if (!locales.includes(locale))
       locales.push(locale);
 
-    if (!accumulator[fileName])
-    {
-      accumulator[fileName] = {};
-    }
-    accumulator[fileName][locale] = fileObject.strings;
-    return accumulator;
-  }, {});
+    if (!dataTreeObj[fileName])
+      dataTreeObj[fileName] = {};
+    if (!dataTreeObj[fileName][locale])
+      dataTreeObj[fileName][locale] = {};
+    dataTreeObj[fileName][locale] = strings;
+  }
 
   let fileNames = Object.keys(dataTreeObj);
   if (filesFilter.length)
     fileNames = fileNames.filter((item) => filesFilter.includes(item));
 
   locales = locales.filter((locale) => locale != defaultLocale).sort();
   // Create two dimensional strings array that reflects CSV structure
   let csvArray = [headers.concat(locales)];
   for (let fileName of fileNames)
   {
@@ skipping 78 matching lines @@
  * @param  {Object} dataTreeObj - ex.:
  * {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
  */
 function writeJson(dataTreeObj)
 {
   for (let fileName in dataTreeObj)
   {
     for (let locale in dataTreeObj[fileName])
     {
       let filePath = path.join(localesDir, locale, fileName);
-      let orderedJSON = orderJSON(dataTreeObj[fileName][locale]);
-      let fileString = JSON.stringify(orderedJSON, null, 2);
+      let sortedJSON = orderJSON(dataTreeObj[fileName][locale]);
+      let fileString = JSON.stringify(sortedJSON, null, 2);
 
       // Newline at end of file to match Coding Style
       if (locale == defaultLocale)
         fileString += "\n";
       fs.writeFile(filePath, fileString, "utf8", (err) =>
       {
         if (err)
         {
           console.error(err);
         }
         else
         {
           console.log(`Updated: ${filePath}`);

[saroyanm, 2018/05/04 13:56:27]

console.log doesn't pass eslint -> Unexpected console statement. (no-console)

This sounds more like an exceptional case for the csv-exporter that we would
like to have those outputs.

[Thomas Greiner, 2018/05/22 17:22:49]

Agreed, feel free to use "eslint-disable" to ignore this rule for this line.

[saroyanm, 2018/06/05 15:03:46]

Done.

         }
       });
     }
   }
 }
 
 /**
- * This function currently rely on nodeJS to sort the object by keys
+ * This function currently relies on V8 to sort the object by keys
  * @param {Object} unordered - json object
  * @returns {Object}
  */
 function orderJSON(unordered)
 {
   const ordered = {};
   for (let key of Object.keys(unordered).sort())
   {
     ordered[key] = unordered[key];
     if (unordered[key].placeholders)
       ordered[key].placeholders = orderJSON(unordered[key].placeholders);
 
     ordered[key] = unordered[key];
   }
   return ordered;
 }
 
 /**
  * Convert two dimensional array to the CSV file
- * @param  {Array} csvArray - array to convert from
+ * @param  {Object[]} csvArray - array to convert from
  */
 function arrayToCsv(csvArray)
 {
   csv.stringify(csvArray, (err, output) =>
   {
     fs.writeFile(outputFileName, output, "utf8", (error) =>
     {
       if (!error)
         console.log(`${outputFileName} is created`);
       else
@@ skipping 14 matching lines @@
 {
   let {dir, base} = path.parse(filePath);
   fs.readFile(filePath, "utf8", (err, data) =>
   {
     if (err)
     {
       callback(err);
     }
     else
     {
-      callback(null, {fileName: base, locale: dir.split("/").pop(),
-        strings: JSON.parse(data)});
+      let locale = dir.split(path.sep).pop();
+      let strings = JSON.parse(data);
+      callback(null, {fileName: base, locale, strings});
     }
   });
 }
 
 /**
  * Exit process and log error message
  * @param {String} error error message
  */
 function exitProcess(error)
 {
   console.error(error);
-  process.exit();
+  process.exit(1);
 }
 
 // CLI
 let helpText = `
 About: Converts locale files between CSV and JSON formats
 Usage: csv-export.js [option] [argument]
 Options:
   -f [FILENAME]         Name of the files to be exported ex.: -f firstRun.json
                         option can be used multiple times.
                         If omitted all files are being exported
 
   -o [FILENAME]         Output filename ex.:
                         -f firstRun.json -o {hash}-firstRun.csv
                         Placeholders:
                           {hash} - Mercurial current revision hash
                           {repo} - Name of the "Default" repository
                         If omitted the output fileName is set to
                         translations-{repo}-{hash}.csv
 
   -i [FILENAME]         Import file path ex: -i issue-reporter.csv
 `;
 
 let argv = process.argv.slice(2);
 let stopExportScript = false;
 // Filter to be used export to the fileNames inside
 let filesFilter = [];
 
 for (let i = 0; i < argv.length; i++)
 {
   switch (argv[i])

[saroyanm, 2018/05/04 13:51:11]

It would be great to use a library like 
[minimist](https://www.npmjs.com/package/minimist) to handle this, but as this
implementation might change
[here](https://gitlab.com/eyeo/adblockplus/adblockplusui/issues/74) I think we
can tackle that separately?

[Thomas Greiner, 2018/05/07 15:16:39]

Agreed.

   {
     case "-h":
       console.log(helpText);
       stopExportScript = true;
       break;
     case "-f":
-      // check if argument following option is specified
       if (!argv[i + 1])
       {
         exitProcess("Please specify the input filename");
       }
-      else
-      {
-        filesFilter.push(argv[i + 1]);
-      }
+      filesFilter.push(argv[i + 1]);
       break;
     case "-o":
       if (!argv[i + 1])
       {
         exitProcess("Please specify the output filename");
       }
-      else
-      {
-        outputFileName = argv[i + 1];
-      }
+      outputFileName = argv[i + 1];
       break;
     case "-i":
       if (!argv[i + 1])
       {
         exitProcess("Please specify the import file");
       }
-      else
-      {
-        let importFile = argv[i + 1];
-        importTranslations(importFile);
-        stopExportScript = true;
-      }
+      let importFile = argv[i + 1];
+      importTranslations(importFile);
+      stopExportScript = true;
       break;
   }
 }
 
 if (!stopExportScript)
   exportTranslations(filesFilter);