Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: build/csv-export.js

Issue 29636585: Issue 6171 - create CSV exporter and importer for translations (Closed)
Patch Set: Addressed Thomas comments Created June 5, 2018, 1:47 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « README.md ('k') | package.json » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 /*
2 * This file is part of Adblock Plus <https://adblockplus.org/>,
3 * Copyright (C) 2006-present eyeo GmbH
4 *
5 * Adblock Plus is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 3 as
7 * published by the Free Software Foundation.
8 *
9 * Adblock Plus is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 /* globals process */
19
20 "use strict";
21
22 const fs = require("fs");
23 const path = require("path");
24 const csv = require("csv");
25 const {promisify} = require("util");
26 const execFile = promisify(require("child_process").execFile);
27 const csvParser = promisify(csv.parse);
28 const readFile = promisify(fs.readFile);
29 const glob = promisify(require("glob").glob);
30 const readJsonPromised = promisify(readJson);
31
32 const localesDir = "locale";
33 const defaultLocale = "en_US";
34
35 let headers = ["Filename", "StringID", "Description", "Placeholders",
36 defaultLocale];
37 let outputFileName = "translations.csv";
38
39 /**
40 * Export existing translation - files into CSV file
41 */
42 function exportTranslations()
43 {
44 glob(`${localesDir}/**/*.json`).then((filePaths) =>
45 {
46 // Reading all existing translations files
47 return Promise.all(filePaths.map((filePath) => readJsonPromised(filePath)));
48 }).then(csvFromJsonFileObjects);
49 }
50
51 /**
52 * Creating Matrix which reflects output CSV file
53 * @param {Object[]} fileObjects - array of file objects created by readJson
54 */
55 function csvFromJsonFileObjects(fileObjects)
56 {
57 let locales = [];
58 // Create Object tree from the Objects array, for easier search
59 // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
60 let dataTreeObj = Object.create(null);
61 let fileNames = [];
62 for (let fileObject of fileObjects)
63 {
64 const {fileName, locale, strings} = fileObject;
65
66 if (locale != defaultLocale && !locales.includes(locale))
67 locales.push(locale);
68
69 if (!filesFilter.length || filesFilter.includes(fileName))
70 fileNames.push(fileName);
71
72 if (!(fileName in dataTreeObj))
73 dataTreeObj[fileName] = Object.create(null);
74
75 dataTreeObj[fileName][locale] = strings;
76 }
77 // Create two dimensional strings array that reflects CSV structure
78 let csvArray = [headers.concat(locales)];
79 for (let fileName of fileNames)
80 {
81 let strings = dataTreeObj[fileName][defaultLocale];
82 for (let stringID of Object.keys(strings))
83 {
84 let fileObj = dataTreeObj[fileName];
85 let {description, message, placeholders} = strings[stringID];
86 let row = [fileName, stringID, description || "",
87 JSON.stringify(placeholders), message];
88
89 for (let locale of locales)
90 {
91 let localeFileObj = fileObj[locale];
92 let isTranslated = !!(localeFileObj && localeFileObj[stringID]);
93 row.push(isTranslated ? localeFileObj[stringID].message : "");
94 }
95 csvArray.push(row);
96 }
97 }
98 arrayToCsv(csvArray);
99 }
100
101 /**
102 * Import strings from the CSV file
103 * @param {string} filePath - CSV file path to import from
104 */
105 function importTranslations(filePath)
106 {
107 readFile(filePath, "utf8").then((fileObjects) =>
108 {
109 return csvParser(fileObjects);
110 }).then((dataMatrix) =>
111 {
112 let headLocales = dataMatrix.shift().slice(4);
113 let dataTreeObj = {};
114 for (let rowId in dataMatrix)
115 {
116 let row = dataMatrix[rowId];
117 let [currentFilename, stringId, description, placeholder, ...messages] =
118 row;
119 if (!stringId)
120 continue;
121
122 stringId = stringId.trim();
123 // Check if it's the filename row
124 if (!dataTreeObj[currentFilename])
125 dataTreeObj[currentFilename] = {};
126
127 description = description.trim();
128 for (let i = 0; i < headLocales.length; i++)
129 {
130 let locale = headLocales[i].trim();
131 let message = messages[i].trim();
132 if (!message)
133 continue;
134
135 // Create Object tree from the Objects array, for easier search
136 // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
137 if (!dataTreeObj[currentFilename][locale])
138 dataTreeObj[currentFilename][locale] = {};
139
140 let localeObj = dataTreeObj[currentFilename][locale];
141 localeObj[stringId] = {};
142 let stringObj = localeObj[stringId];
143
144 // We keep string descriptions only in default locale files
145 if (locale == defaultLocale && description)
146 stringObj.description = description;
147
148 stringObj.message = message;
149 if (placeholder)
150 stringObj.placeholders = JSON.parse(placeholder);
151 }
152 }
153 writeJson(dataTreeObj);
154 });
155 }
156
157 /**
158 * Write locale files according to dataTreeObj
159 * @param {Object} dataTreeObj - ex.:
160 * {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
161 */
162 function writeJson(dataTreeObj)
163 {
164 for (let fileName in dataTreeObj)
165 {
166 for (let locale in dataTreeObj[fileName])
167 {
168 let filePath = path.join(localesDir, locale, fileName);
169 let sortedJSON = orderJSON(dataTreeObj[fileName][locale]);
170 let fileString = JSON.stringify(sortedJSON, null, 2);
171
172 // Newline at end of file to match Coding Style
173 if (locale == defaultLocale)
174 fileString += "\n";
175 fs.writeFile(filePath, fileString, "utf8", (err) =>
176 {
177 if (err)
178 {
179 console.error(err);
180 }
181 else
182 {
183 console.log(`Updated: ${filePath}`);
184 }
185 });
186 }
187 }
188 }
189
190 /**
191 * This function currently relies on V8 to sort the object by keys
192 * @param {Object} unordered - json object
193 * @returns {Object}
194 */
195 function orderJSON(unordered)
196 {
197 const ordered = {};
198 for (let key of Object.keys(unordered).sort())
199 {
200 ordered[key] = unordered[key];
201 if (unordered[key].placeholders)
202 ordered[key].placeholders = orderJSON(unordered[key].placeholders);
203
204 ordered[key] = unordered[key];
205 }
206 return ordered;
207 }
208
209 /**
210 * Convert two dimensional array to the CSV file
211 * @param {Object[]} csvArray - array to convert from
212 */
213 function arrayToCsv(csvArray)
214 {
215 csv.stringify(csvArray, (err, output) =>
216 {
217 fs.writeFile(outputFileName, output, "utf8", (error) =>
218 {
219 if (!error)
220 console.log(`${outputFileName} is created`);
221 else
222 console.error(error);
223 });
224 });
225 }
226
227 /**
228 * Reads JSON file and assign filename and locale to it
229 * @param {string} filePath - ex.: "locales/en_US/desktop-options.json"
230 * @param {function} callback - fileName, locale and strings of locale file
231 * Parameters:
232 * * Error message
233 * * Object containing fileName, locale and strings
234 */
235 function readJson(filePath, callback)
236 {
237 const {dir, base} = path.parse(filePath);
238 readFile(filePath, "utf8").then((data) =>
239 {
240 const locale = dir.split(path.sep).pop();
241 const strings = JSON.parse(data);
242 callback(null, {fileName: base, locale, strings});
243 }).catch((err) =>
244 {
245 callback(err);
246 });
247 }
248
249 /**
250 * Exit process and log error message
251 * @param {String} error error message
252 */
253 function exitProcess(error)
254 {
255 console.error(error);
256 process.exit(1);
257 }
258
259 // CLI
260 let helpText = `
261 About: Converts locale files between CSV and JSON formats
262 Usage: csv-export.js [option] [argument]
263 Options:
264 -f [FILENAME] Name of the files to be exported ex.: -f firstRun.json
265 option can be used multiple times.
266 If omitted all files are being exported
267
268 -o [FILENAME] Output filename ex.:
269 -f firstRun.json -o {hash}-firstRun.csv
270 Placeholders:
271 {hash} - Mercurial current revision hash
272 If omitted the output fileName is set to
273 translations-{repo}-{hash}.csv
274
275 -i [FILENAME] Import file path ex: -i issue-reporter.csv
276 `;
277
278 let argv = process.argv.slice(2);
279 let stopExportScript = false;
280 // Filter to be used export to the fileNames inside
281 let filesFilter = [];
282
283 for (let i = 0; i < argv.length; i++)
284 {
285 switch (argv[i])
286 {
287 case "-h":
288 console.log(helpText);
289 stopExportScript = true;
290 break;
291 case "-f":
292 if (!argv[i + 1])
293 {
294 exitProcess("Please specify the input filename");
295 }
296 filesFilter.push(argv[i + 1]);
297 break;
298 case "-o":
299 if (!argv[i + 1])
300 {
301 exitProcess("Please specify the output filename");
302 }
303 outputFileName = argv[i + 1];
304 break;
305 case "-i":
306 if (!argv[i + 1])
307 {
308 exitProcess("Please specify the import file");
309 }
310 let importFile = argv[i + 1];
311 importTranslations(importFile);
312 stopExportScript = true;
313 break;
314 }
315 }
316
317 if (!stopExportScript)
318 exportTranslations(filesFilter);
OLDNEW
« no previous file with comments | « README.md ('k') | package.json » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld