Issue 6171 - create CSV exporter and importer for translations

csv-export.js

+const fs = require("fs");
+const {exec} = require("child_process");
+
+const localesDir = "locale";
+const defaultLocale = "en_US";
+
+let filesNames = []; // ex.: desktop-options.json
+let locales = [];  // List of all available locale codes
+let headers = ["StringID", "Description", "Placeholders", defaultLocale];
+let outputFileName = "translations-{repo}-{hash}.csv";
+
+/**
+ * Export existing translation files into CSV file
+ * @param  {[type]} filesFilter Optional parameter which allow include only
+ *                              fileNames in the array, if ommited all files
+ *                              will be exported
+ */
+function exportTranslations(filesFilter)
+{
+  let mercurialCommands = [];
+  mercurialCommands.push(executeMercurial("hg id -i")); // Get Hash
+  mercurialCommands.push(executeMercurial("hg paths default")); // Get repo path
+  Promise.all(mercurialCommands).then((outputs) =>
+  {
+    // Remove line endings and "+" sign from the end of the hash
+    let [hash, path] = outputs.map((item) => item.replace(/\+\n|\n$/, ""));
+    // Update name of the file to be outputted
+    outputFileName = outputFileName.replace("{hash}", hash);
+    outputFileName = outputFileName.replace("{repo}", path.split("/").pop());
+
+    // Prepare to read all available locales and default files  
+    let readDirectories = [];
+    readDirectories.push(readDir(`${localesDir}/${defaultLocale}`));
+    readDirectories.push(readDir(localesDir));
+    return Promise.all(readDirectories);
+  }).then((files) =>
+  {
+    [filesNames, locales] = files;
+    // Filter files
+    if (filesFilter.length)
+      filesNames = filesNames.filter((item) => filesFilter.includes(item));
+
+    let readJsonPromises = [];
+    for(let file of filesNames)
+      for(let locale of locales)
+        readJsonPromises.push(readJson(locale, file));
+
+    // Reading all existing translations files
+    return Promise.all(readJsonPromises);
+  }).then((fileObjects) =>
+  {
+    // Create Object tree from the Objects array, for easier search
+    // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
+    let dataTreeObj = fileObjects.reduce((acc, fileObject) =>
+    {
+      if (!fileObject)
+        return acc;
+
+      let filename = fileObject.filename;
+      let locale = fileObject.locale;
+      if (!acc[filename])
+      {
+        acc[filename] = {};
+      }
+      acc[filename][locale] = fileObject.strings;
+      return acc;
+    }, {});
+
+    // Create two dimentional strings array that reflects CSV structure
+    let localesWithoutDefault = locales.filter((item) => item != defaultLocale);
+    let csvArray = [headers.concat(localesWithoutDefault)];
+    for (let file of filesNames)
+    {
+      csvArray.push([file]);
+      for (let stringID in dataTreeObj[file][defaultLocale])
+      {
+        let fileObj = dataTreeObj[file];
+        let stringObj = fileObj[defaultLocale][stringID];
+        let {description, message, placeholders} = stringObj;
+
+        // Use yaml-like format for easy extraction, rather sensitive char hacks
+        let yamlPlaceholder = "";
+        for (let placeholder in placeholders)
+        {
+          yamlPlaceholder += `${placeholder}:\n`;
+          let {content, example} = placeholders[placeholder];
+          yamlPlaceholder += `  content: ${content}\n`;
+          yamlPlaceholder += `  example: ${example}\n`;
+        }
+
+        let row = [stringID, description || "", yamlPlaceholder, message];
+        for (let locale of localesWithoutDefault)
+        {
+          let localeFileObj = fileObj[locale];
+          let isTranslated = localeFileObj && localeFileObj[stringID];
+          row.push(isTranslated ? localeFileObj[stringID].message : "");
+        }
+        csvArray.push(row);
+      }
+    }
+    arrayToCsv(csvArray); // Convert matrix to CSV
+  });
+}
+
+/**
+ * Import strings from the CSV file
+ * @param  {[type]} filePath CSV file path to import from
+ */
+function importTranslations(filePath)

[saroyanm, 2017/12/19 19:48:23]

Important translations order are different from the crowdin synchronization
script. Currently the order of the Strings in locales matching the one in the
defaultLocale, while it's not true for existing locales.

I think this needs to be fixed as well.
Let's discuss that after holidays @Thomas.

+{
+  readCsv(filePath).then((fileObjects) =>
+  {
+    let dataMatrix = csvToArray(fileObjects);
+    let headers = dataMatrix.splice(0, 1)[0];
+    let dataTreeObj = {};
+    let currentFilename = "";
+    for(let rowId in dataMatrix)
+    {
+      let row = dataMatrix[rowId];
+      let [stringId, description, placeholder] = row;
+      if (!stringId)
+        continue;
+
+      stringId = stringId.trim();
+      if (stringId.endsWith(".json")) // Check if it's the filename row
+      {
+        currentFilename = stringId;
+        dataTreeObj[currentFilename] = {};
+        continue;
+      }
+
+      description = description.trim();
+      placeholder = placeholder.trim();
+      for (let i = 3; i < headers.length; i++)
+      {
+        let locale = headers[i].trim();
+        let message = row[i].trim();
+        if (!message)
+          continue;
+
+        // Create Object tree from the Objects array, for easier search
+        // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
+        if (!dataTreeObj[currentFilename][locale])
+          dataTreeObj[currentFilename][locale] = {};
+
+        let localeObj = dataTreeObj[currentFilename][locale];
+        localeObj[stringId] = {};
+
+        // We keep string descriptions only in default locale files
+        if (locale == defaultLocale)
+          localeObj[stringId].description = description;
+
+        localeObj[stringId].message = message;
+        if (placeholder)
+        {
+          let placeholders = placeholder.split("\n");
+          let placeholderName = "";
+          localeObj[stringId].placeholders = placeholders.reduce((acc, item) =>
+          {
+            /*
+             Placeholders use YAML like syntax in CSV files, ex:
+               tracking:
+                content: $1
+                example: Block additional tracking
+              acceptableAds:
+                content: $2
+                example: Allow Acceptable Ads
+            */
+            if (item.startsWith("  "))
+            {
+              let [key, value] = item.trim().split(":");
+              acc[placeholderName][key] = value.trim();
+            }
+            else
+            {
+              placeholderName = item.trim().replace(":", "");
+              acc[placeholderName] = {};
+            }
+            return acc;
+          }, {});
+        }
+      }
+    }
+    writeJson(dataTreeObj);
+  });
+}
+
+/**
+ * Write locale files according to dataTreeObj which look like:
+ * @param  {Object} dataTreeObj which look like:
+ * {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
+ */
+function writeJson(dataTreeObj)
+{
+  for (let filename in dataTreeObj)
+  {
+    for (let locale in dataTreeObj[filename])
+    {
+      let path = `${localesDir}/${locale}/${filename}`;
+      let fileString = JSON.stringify(dataTreeObj[filename][locale], null, 2);
+      fileString += "\n"; // Newline at end of file to match Coding Style
+      fs.writeFile(path, fileString, 'utf8', (err)=>
+        {
+          if (!err)
+          {
+            console.log(`Updated: ${path}`);
+          }
+          else
+          {
+            console.log(err);
+          }
+        });
+    }
+  }
+}
+
+/**
+ * Parse CSV string and return array
+ * @param  {String} csvText Array to convert from
+ * @return {Array} two dimentional array
+ */
+function csvToArray(csvText)
+{
+  let previouseChar = "";
+  let row = []; // Holds parsed CSV data representing a row/line 
+  let column = 0; // Pointer of the column in the row
+  let csvArray = []; // Two dimentional array that holds rows
+  let parseSpecialChars = true; // Like comma(,) and quotation(")
+  for (let charIndex in csvText)
+  {
+    currentChar = csvText[charIndex];
+    if (!row[column])
+      row[column] = "";
+
+    if ('"' == currentChar)
+    {
+      // Double quote is like escaping quote char in CSV
+      if (currentChar === previouseChar && parseSpecialChars)
+        row[column] += currentChar;
+
+      parseSpecialChars = !parseSpecialChars;
+    }
+    else if (currentChar == "," && parseSpecialChars)
+    {
+      currentChar = "";
+      column++; // Update columns, because comma(,) separates columns
+    }
+    else if (currentChar == "\n" && parseSpecialChars)
+    {
+      if ("\r" === previouseChar) // In case of \r\n
+        row[column] = row[column].slice(0, -1);
+ 
+      csvArray.push(row);
+      // Reset pointers for the new row
+      row = [];
+      column = 0;
+      currentChar = "";
+    }
+    else
+    {
+      row[column] += currentChar;
+    }
+    previouseChar = currentChar;
+  }
+  csvArray.push(row);
+  return csvArray;
+}
+
+
+/**
+ * Convert two dimentional array to the CSV file
+ * @param  {Array} csvArray Array to convert from
+ */
+function arrayToCsv(csvArray)
+{
+  let dataToWrite = "";
+  for (let row of csvArray)
+  {
+    let columnString = row.reduce((accum, col) =>
+    {
+      // Escape single quote with quote before
+      accum += `","${col.replace(/\"/g, '""')}`;
+      return accum;
+    });
+    dataToWrite += `"${columnString}"\r\n`;
+  }
+  dataToWrite += "\r\n";
+  fs.writeFile(outputFileName, dataToWrite, "utf8", function (err)
+  {
+    if (!err)
+      console.log(`${outputFileName} is created`);
+  });
+}
+
+/**
+ * Reads JSON file and assign filename and locale to it
+ * @param  {String} locale ex.: "en_US", "de"...
+ * @param  {String} fileName ex.: "desktop-options.json" 
+ * @return {Promise} Promise object
+ */
+function readJson(locale, file)
+{
+  let path = `${localesDir}/${locale}/${file}`;
+  return new Promise((resolve, reject) =>
+  {
+    fs.readFile(path, (err, data) => { 
+      if (err)
+      {
+        reject(err);
+      }
+      else
+      {
+        let json = {};
+        json.filename = file;
+        json.locale = locale;
+        json.strings = JSON.parse(data);
+        resolve(json);
+      }
+    });
+  }).catch(reason => // Continue Promise.All even if rejected.
+  { 
+    // Commented out log not to spam the output.
+    // TODO: Think about more meaningful output without spaming
+    // console.log(`Reading ${path} was rejected: ${reason}`);
+  });
+}
+
+/**
+ * Reads CSV file
+ * @param  {String} file path
+ * @return {Promise} Promise object
+ */
+function readCsv(filePath)
+{
+  return new Promise((resolve, reject) =>
+  {
+    fs.readFile(filePath, "utf8", (err, data) => { 
+      if (err)
+        reject(err);
+      else
+        resolve(data);
+    });
+  });
+}
+
+/**
+ * Read files and folder names inside of the directory
+ * @param  {String} dir patch of the folder
+ * @return {Promise} Promise object
+ */
+function readDir(dir)
+{
+  return new Promise((resolve, reject) =>
+  {
+    fs.readdir(dir, (err, folders) => { 
+      if (err)
+        reject(err);
+      else
+        resolve(folders);
+    });
+  });
+}
+
+/**
+ * Executing mercurial commands on the system level
+ * @param  {String} command mercurial command ex.:"hg ..."
+ * @return {Promise} Promise object containing output from the command
+ */
+function executeMercurial(command)
+{
+  // Limit only to Mercurial commands to minimize the missuse risk
+  if (command.substring(0, 3) !== "hg ")
+  {
+    console.error("You are only allowed to run Mercurial commands('hg ...')");
+    return;
+  }
+
+  return new Promise((resolve, reject) =>
+  {
+    exec(command, (err, output) =>
+    {
+      if (err)
+        reject(err);
+      else
+        resolve(output);
+    });
+  });
+}
+
+// CLI
+let helpText = `
+About: This script exports locales into .csv format
+Usage: node csv-export.js [option] [argument]
+Options:
+  -f                    Name of the files to be exported ex.: -f firstRun.json
+                        option can be used multiple timeString.
+                        If ommited all files are being exported
+
+  -o                    Output filename ex.:
+                        -f firstRun.json -o {hash}-firstRun.csv
+                        Placeholders:
+                          {hash} - Mercurial current revision hash
+                          {repo} - Name of the "Default" repository
+                        If ommited the output fileName is set to
+                        translations-{repo}-{hash}.csv
+
+  -i                    Import file path ex: -i issue-reporter.csv
+`;
+
+let arguments = process.argv.slice(2);
+let stopExportScript = false;
+let filesFilter = []; // Filter to be used export to the fileNames inside
+
+for (let i = 0; i < arguments.length; i++)
+{
+  switch (arguments[i])
+  {
+    case "-h":
+      console.log(helpText);
+      stopExportScript = true;
+      break;
+    case "-f":
+      if (!arguments[i + 1]) // check if argument following option is specified
+      {
+        console.error("Please specify the input filename");
+        stopExportScript = true;
+      }
+      else
+      {
+        filesFilter.push(arguments[i + 1]);
+      }
+      break;
+    case "-o":
+      if (!arguments[i + 1])
+      {
+        console.error("Please specify the output filename");
+        stopExportScript = true;
+      }
+      else
+      {
+        outputFileName = arguments[i + 1];
+      }
+      break;
+    case "-i":
+      if (!arguments[i + 1])
+      {
+        console.error("Please specify the input filename");
+      }
+      else
+      {
+        let importFile = arguments[i + 1];
+        importTranslations(importFile);
+      }
+      stopExportScript = true;
+      break;
+  }
+}
+
+if (!stopExportScript)
+  exportTranslations(filesFilter);