Issue 6171 - create CSV exporter and importer for translations

build/csv-export.js

+/*
+ * This file is part of Adblock Plus <https://adblockplus.org/>,
+ * Copyright (C) 2006-present eyeo GmbH
+ *
+ * Adblock Plus is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * Adblock Plus is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* globals require, process */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const csv = require("csv");
+const {promisify} = require("util");
+const execFile = promisify(require("child_process").execFile);
+const csvParser = promisify(csv.parse);
+const readDir = promisify(fs.readdir);
+const readFile = promisify(fs.readFile);
+const glob = promisify(require("glob").glob);
+const readJsonPromised = promisify(readJson);
+
+const localesDir = "locale";
+const defaultLocale = "en_US";
+
+let headers = ["filename", "StringID", "Description", "Placeholders",
+               defaultLocale];
+let outputFileName = "translations-{repo}-{hash}.csv";
+
+/**
+ * Export existing translation - files into CSV file
+ * @param  {string[]} [filesFilter] - fileNames filter, if omitted all files
+ *                                    will be exported
+ */
+function exportTranslations(filesFilter)
+{
+  let mercurialCommands = [];
+  // Get Hash
+  mercurialCommands.push(execFile("hg", ["id", "-i"]));

[saroyanm, 2018/05/15 18:59:49]

This will not work in git.

Alternative:   mercurialCommands.push(execFile("git", ["rev-parse", "HEAD"]));

[saroyanm, 2018/05/17 17:27:37]

As discussed: We better remove this mercurial specific logic and introduce it
back if we think we really need it.

+  // Get repo path
+  mercurialCommands.push(execFile("hg", ["paths", "default"]));

[saroyanm, 2018/05/15 18:59:49]

This will not work in git.

Alternative: mercurialCommands.push(execFile("git", ["config", "--get",
"remote.origin.url"]));

+  Promise.all(mercurialCommands).then((outputs) =>
+  {
+    // Remove line endings and "+" sign from the end of the hash
+    let [hash, filePath] = outputs.map((item) =>
+                           item.stdout.replace(/\+\n|\n$/, ""));
+    // Update name of the file to be output
+    outputFileName = outputFileName.replace("{hash}", hash);
+    outputFileName = outputFileName.replace("{repo}", path.basename(filePath));
+
+    // Read all available locales and default files
+    return glob(`${localesDir}/**/*.json`, {});
+  }).then((filePaths) =>
+  {
+    // Reading all existing translations files
+    return Promise.all(filePaths.map((filePath) => readJsonPromised(filePath)));
+  }).then(csvFromJsonFileObjects);
+}
+
+/**
+ * Creating Matrix which reflects output CSV file
+ * @param  {Array} fileObjects - array of file objects created by readJson
+ */
+function csvFromJsonFileObjects(fileObjects)
+{
+  let locales = [];
+  // Create Object tree from the Objects array, for easier search
+  // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
+  let dataTreeObj = fileObjects.reduce((accumulator, fileObject) =>
+  {
+    if (!fileObject)
+      return accumulator;
+
+    let {fileName, locale} = fileObject;
+    if (!locales.includes(locale))
+      locales.push(locale);
+
+    if (!accumulator[fileName])
+    {
+      accumulator[fileName] = {};
+    }
+    accumulator[fileName][locale] = fileObject.strings;
+    return accumulator;
+  }, {});
+
+  let fileNames = Object.keys(dataTreeObj);
+  if (filesFilter.length)
+    fileNames = fileNames.filter((item) => filesFilter.includes(item));
+
+  locales = locales.filter((locale) => locale != defaultLocale).sort();
+  // Create two dimensional strings array that reflects CSV structure
+  let csvArray = [headers.concat(locales)];
+  for (let fileName of fileNames)
+  {
+    let strings = dataTreeObj[fileName][defaultLocale];
+    for (let stringID of Object.keys(strings))
+    {
+      let fileObj = dataTreeObj[fileName];
+      let {description, message, placeholders} = strings[stringID];
+      let row = [fileName, stringID, description || "",
+                 JSON.stringify(placeholders), message];
+
+      for (let locale of locales)
+      {
+        let localeFileObj = fileObj[locale];
+        let isTranslated = !!(localeFileObj && localeFileObj[stringID]);
+        row.push(isTranslated ? localeFileObj[stringID].message : "");
+      }
+      csvArray.push(row);
+    }
+  }
+  arrayToCsv(csvArray);
+}
+
+/**
+ * Import strings from the CSV file
+ * @param  {string} filePath - CSV file path to import from
+ */
+function importTranslations(filePath)
+{
+  readFile(filePath, "utf8").then((fileObjects) =>
+  {
+    return csvParser(fileObjects);
+  }).then((dataMatrix) =>
+  {
+    let headLocales = dataMatrix.shift().slice(4);
+    let dataTreeObj = {};
+    for (let rowId in dataMatrix)
+    {
+      let row = dataMatrix[rowId];
+      let [currentFilename, stringId, description, placeholder, ...messages] =
+        row;
+      if (!stringId)
+        continue;
+
+      stringId = stringId.trim();
+      // Check if it's the filename row
+      if (!dataTreeObj[currentFilename])
+        dataTreeObj[currentFilename] = {};
+
+      description = description.trim();
+      for (let i = 0; i < headLocales.length; i++)
+      {
+        let locale = headLocales[i].trim();
+        let message = messages[i].trim();
+        if (!message)
+          continue;
+
+        // Create Object tree from the Objects array, for easier search
+        // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
+        if (!dataTreeObj[currentFilename][locale])
+          dataTreeObj[currentFilename][locale] = {};
+
+        let localeObj = dataTreeObj[currentFilename][locale];
+        localeObj[stringId] = {};
+        let stringObj = localeObj[stringId];
+
+        // We keep string descriptions only in default locale files
+        if (locale == defaultLocale)
+          stringObj.description = description;
+
+        stringObj.message = message;
+        if (placeholder)
+          stringObj.placeholders = JSON.parse(placeholder);
+      }
+    }
+    writeJson(dataTreeObj);
+  });
+}
+
+/**
+ * Write locale files according to dataTreeObj
+ * @param  {Object} dataTreeObj - ex.:
+ * {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
+ */
+function writeJson(dataTreeObj)
+{
+  for (let fileName in dataTreeObj)
+  {
+    for (let locale in dataTreeObj[fileName])
+    {
+      let filePath = path.join(localesDir, locale, fileName);
+      let orderedJSON = orderJSON(dataTreeObj[fileName][locale]);
+      let fileString = JSON.stringify(orderedJSON, null, 2);
+
+      // Newline at end of file to match Coding Style
+      if (locale == defaultLocale)
+        fileString += "\n";
+      fs.writeFile(filePath, fileString, "utf8", (err) =>
+      {
+        if (err)
+        {
+          console.error(err);
+        }
+        else
+        {
+          console.log(`Updated: ${filePath}`);

[saroyanm, 2018/05/04 13:56:27]

console.log doesn't pass eslint -> Unexpected console statement. (no-console)

This sounds more like an exceptional case for the csv-exporter that we would
like to have those outputs.

[Thomas Greiner, 2018/05/22 17:22:49]

Agreed, feel free to use "eslint-disable" to ignore this rule for this line.

[saroyanm, 2018/06/05 15:03:46]

Done.

+        }
+      });
+    }
+  }
+}
+
+/**
+ * This function currently rely on nodeJS to sort the object by keys
+ * @param {Object} unordered - json object
+ * @returns {Object}
+ */
+function orderJSON(unordered)
+{
+  const ordered = {};
+  for (let key of Object.keys(unordered).sort())
+  {
+    ordered[key] = unordered[key];
+    if (unordered[key].placeholders)
+      ordered[key].placeholders = orderJSON(unordered[key].placeholders);
+
+    ordered[key] = unordered[key];
+  }
+  return ordered;
+}
+
+/**
+ * Convert two dimensional array to the CSV file
+ * @param  {Array} csvArray - array to convert from
+ */
+function arrayToCsv(csvArray)
+{
+  csv.stringify(csvArray, (err, output) =>
+  {
+    fs.writeFile(outputFileName, output, "utf8", (error) =>
+    {
+      if (!error)
+        console.log(`${outputFileName} is created`);
+      else
+        console.error(error);
+    });
+  });
+}
+
+/**
+ * Reads JSON file and assign filename and locale to it
+ * @param {string} filePath - ex.: "locales/en_US/desktop-options.json"
+ * @param {function} callback - fileName, locale and strings of locale file
+ *                   Parameters:
+ *                     * Error message
+ *                     * Object containing fileName, locale and strings
+ */
+function readJson(filePath, callback)
+{
+  let {dir, base} = path.parse(filePath);
+  fs.readFile(filePath, "utf8", (err, data) =>
+  {
+    if (err)
+    {
+      callback(err);
+    }
+    else
+    {
+      callback(null, {fileName: base, locale: dir.split("/").pop(),
+        strings: JSON.parse(data)});
+    }
+  });
+}
+
+/**
+ * Exit process and log error message
+ * @param {String} error error message
+ */
+function exitProcess(error)
+{
+  console.error(error);
+  process.exit();
+}
+
+// CLI
+let helpText = `
+About: Converts locale files between CSV and JSON formats
+Usage: csv-export.js [option] [argument]
+Options:
+  -f [FILENAME]         Name of the files to be exported ex.: -f firstRun.json
+                        option can be used multiple times.
+                        If omitted all files are being exported
+
+  -o [FILENAME]         Output filename ex.:
+                        -f firstRun.json -o {hash}-firstRun.csv
+                        Placeholders:
+                          {hash} - Mercurial current revision hash
+                          {repo} - Name of the "Default" repository
+                        If omitted the output fileName is set to
+                        translations-{repo}-{hash}.csv
+
+  -i [FILENAME]         Import file path ex: -i issue-reporter.csv
+`;
+
+let argv = process.argv.slice(2);
+let stopExportScript = false;
+// Filter to be used export to the fileNames inside
+let filesFilter = [];
+
+for (let i = 0; i < argv.length; i++)
+{
+  switch (argv[i])

[saroyanm, 2018/05/04 13:51:11]

It would be great to use a library like 
[minimist](https://www.npmjs.com/package/minimist) to handle this, but as this
implementation might change
[here](https://gitlab.com/eyeo/adblockplus/adblockplusui/issues/74) I think we
can tackle that separately?

[Thomas Greiner, 2018/05/07 15:16:39]

Agreed.

+  {
+    case "-h":
+      console.log(helpText);
+      stopExportScript = true;
+      break;
+    case "-f":
+      // check if argument following option is specified
+      if (!argv[i + 1])
+      {
+        exitProcess("Please specify the input filename");
+      }
+      else
+      {
+        filesFilter.push(argv[i + 1]);
+      }
+      break;
+    case "-o":
+      if (!argv[i + 1])
+      {
+        exitProcess("Please specify the output filename");
+      }
+      else
+      {
+        outputFileName = argv[i + 1];
+      }
+      break;
+    case "-i":
+      if (!argv[i + 1])
+      {
+        exitProcess("Please specify the import file");
+      }
+      else
+      {
+        let importFile = argv[i + 1];
+        importTranslations(importFile);
+        stopExportScript = true;
+      }
+      break;
+  }
+}
+
+if (!stopExportScript)
+  exportTranslations(filesFilter);