Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: build/csv-export.js

Issue 29636585: Issue 6171 - create CSV exporter and importer for translations (Closed)
Patch Set: Use consistent naming and promisify fs.writefile Created June 5, 2018, 2:05 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « README.md ('k') | package.json » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 /*
2 * This file is part of Adblock Plus <https://adblockplus.org/>,
3 * Copyright (C) 2006-present eyeo GmbH
4 *
5 * Adblock Plus is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 3 as
7 * published by the Free Software Foundation.
8 *
9 * Adblock Plus is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 /* globals process */
19
20 "use strict";
21
22 const fs = require("fs");
23 const path = require("path");
24 const csv = require("csv");
25 const {promisify} = require("util");
26 const execFile = promisify(require("child_process").execFile);
27 const csvParser = promisify(csv.parse);
28 const readFile = promisify(fs.readFile);
29 const writeFile = promisify(fs.writeFile);
30 const glob = promisify(require("glob").glob);
31 const readJsonPromised = promisify(readJson);
32
33 const localesDir = "locale";
34 const defaultLocale = "en_US";
35
36 const headers = ["Filename", "StringID", "Description", "Placeholders",
37 defaultLocale];
38 let outputFileName = "translations.csv";
39
40 /**
41 * Export existing translation - files into CSV file
42 */
43 function exportTranslations()
44 {
45 glob(`${localesDir}/**/*.json`).then((filePaths) =>
46 {
47 // Reading all existing translations files
48 return Promise.all(filePaths.map((filePath) => readJsonPromised(filePath)));
49 }).then(csvFromJsonFileObjects);
50 }
51
52 /**
53 * Creating Matrix which reflects output CSV file
54 * @param {Object[]} fileObjects - array of file objects created by readJson
55 */
56 function csvFromJsonFileObjects(fileObjects)
57 {
58 const locales = [];
59 const fileNames = [];
60 // Create Object tree from the Objects array, for easier search
61 // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
62 const dataTreeObj = Object.create(null);
63 for (const fileObject of fileObjects)
64 {
65 const {fileName, locale, strings} = fileObject;
66
67 if (locale != defaultLocale && !locales.includes(locale))
68 locales.push(locale);
69
70 if (!filesFilter.length || filesFilter.includes(fileName))
71 fileNames.push(fileName);
72
73 if (!(fileName in dataTreeObj))
74 dataTreeObj[fileName] = Object.create(null);
75
76 dataTreeObj[fileName][locale] = strings;
77 }
78 // Create two dimensional strings array that reflects CSV structure
79 const csvArray = [headers.concat(locales)];
80 for (const fileName of fileNames)
81 {
82 const strings = dataTreeObj[fileName][defaultLocale];
83 for (const stringID of Object.keys(strings))
84 {
85 const fileObj = dataTreeObj[fileName];
86 const {description, message, placeholders} = strings[stringID];
87 const row = [fileName, stringID, description || "",
88 JSON.stringify(placeholders), message];
89
90 for (const locale of locales)
91 {
92 const localeFileObj = fileObj[locale];
93 const isTranslated = !!(localeFileObj && localeFileObj[stringID]);
94 row.push(isTranslated ? localeFileObj[stringID].message : "");
95 }
96 csvArray.push(row);
97 }
98 }
99 arrayToCsv(csvArray);
100 }
101
102 /**
103 * Import strings from the CSV file
104 * @param {string} filePath - CSV file path to import from
105 */
106 function importTranslations(filePath)
107 {
108 readFile(filePath, "utf8").then((fileObjects) =>
109 {
110 return csvParser(fileObjects);
111 }).then((dataMatrix) =>
112 {
113 const headLocales = dataMatrix.shift().slice(4);
114 const dataTreeObj = {};
115 for (const rowId in dataMatrix)
116 {
117 const row = dataMatrix[rowId];
118 let [currentFilename, stringId, description, placeholder, ...messages] =
119 row;
120 if (!stringId)
121 continue;
122
123 stringId = stringId.trim();
124 // Check if it's the filename row
125 if (!dataTreeObj[currentFilename])
126 dataTreeObj[currentFilename] = {};
127
128 description = description.trim();
129 for (let i = 0; i < headLocales.length; i++)
130 {
131 const locale = headLocales[i].trim();
132 const message = messages[i].trim();
133 if (!message)
134 continue;
135
136 // Create Object tree from the Objects array, for easier search
137 // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
138 if (!dataTreeObj[currentFilename][locale])
139 dataTreeObj[currentFilename][locale] = {};
140
141 const localeObj = dataTreeObj[currentFilename][locale];
142 localeObj[stringId] = {};
143 const stringObj = localeObj[stringId];
144
145 // We keep string descriptions only in default locale files
146 if (locale == defaultLocale && description)
147 stringObj.description = description;
148
149 stringObj.message = message;
150 if (placeholder)
151 stringObj.placeholders = JSON.parse(placeholder);
152 }
153 }
154 writeJson(dataTreeObj);
155 });
156 }
157
158 /**
159 * Write locale files according to dataTreeObj
160 * @param {Object} dataTreeObj - ex.:
161 * {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
162 */
163 function writeJson(dataTreeObj)
164 {
165 for (const fileName in dataTreeObj)
166 {
167 for (const locale in dataTreeObj[fileName])
168 {
169 const filePath = path.join(localesDir, locale, fileName);
170 const sortedJson = sortJson(dataTreeObj[fileName][locale]);
171 let fileString = JSON.stringify(sortedJson, null, 2);
172
173 // Newline at end of file to match Coding Style
174 if (locale == defaultLocale)
175 fileString += "\n";
176 writeFile(filePath, fileString, "utf8").then(() =>
177 {
178 console.log(`Updated: ${filePath}`);
179 }).catch((err) =>
180 {
181 console.error(err);
182 });
183 }
184 }
185 }
186
187 /**
188 * This function currently relies on V8 to sort the object by keys
189 * @param {Object} unordered - json object
190 * @returns {Object}
191 */
192 function sortJson(unordered)
193 {
194 const ordered = {};
195 for (const key of Object.keys(unordered).sort())
196 {
197 ordered[key] = unordered[key];
198 if (unordered[key].placeholders)
199 ordered[key].placeholders = sortJson(unordered[key].placeholders);
200
201 ordered[key] = unordered[key];
202 }
203 return ordered;
204 }
205
206 /**
207 * Convert two dimensional array to the CSV file
208 * @param {Object[]} csvArray - array to convert from
209 */
210 function arrayToCsv(csvArray)
211 {
212 csv.stringify(csvArray, (err, output) =>
213 {
214 writeFile(outputFileName, output, "utf8").then(() =>
215 {
216 console.log(`${outputFileName} is created`);
217 }).catch((error) =>
218 {
219 console.error(error);
220 });
221 });
222 }
223
224 /**
225 * Reads JSON file and assign filename and locale to it
226 * @param {string} filePath - ex.: "locales/en_US/desktop-options.json"
227 * @param {function} callback - fileName, locale and strings of locale file
228 * Parameters:
229 * * Error message
230 * * Object containing fileName, locale and strings
231 */
232 function readJson(filePath, callback)
233 {
234 const {dir, base} = path.parse(filePath);
235 readFile(filePath, "utf8").then((data) =>
236 {
237 const locale = dir.split(path.sep).pop();
238 const strings = JSON.parse(data);
239 callback(null, {fileName: base, locale, strings});
240 }).catch((err) =>
241 {
242 callback(err);
243 });
244 }
245
246 /**
247 * Exit process and log error message
248 * @param {String} error error message
249 */
250 function exitProcess(error)
251 {
252 console.error(error);
253 process.exit(1);
254 }
255
256 // CLI
257 const helpText = `
258 About: Converts locale files between CSV and JSON formats
259 Usage: csv-export.js [option] [argument]
260 Options:
261 -f [FILENAME] Name of the files to be exported ex.: -f firstRun.json
262 option can be used multiple times.
263 If omitted all files are being exported
264
265 -o [FILENAME] Output filename ex.:
266 -f firstRun.json -o {hash}-firstRun.csv
267 Placeholders:
268 {hash} - Mercurial current revision hash
269 If omitted the output fileName is set to
270 translations-{repo}-{hash}.csv
271
272 -i [FILENAME] Import file path ex: -i issue-reporter.csv
273 `;
274
275 const argv = process.argv.slice(2);
276 let stopExportScript = false;
277 // Filter to be used export to the fileNames inside
278 const filesFilter = [];
279
280 for (let i = 0; i < argv.length; i++)
281 {
282 switch (argv[i])
283 {
284 case "-h":
285 console.log(helpText);
286 stopExportScript = true;
287 break;
288 case "-f":
289 if (!argv[i + 1])
290 {
291 exitProcess("Please specify the input filename");
292 }
293 filesFilter.push(argv[i + 1]);
294 break;
295 case "-o":
296 if (!argv[i + 1])
297 {
298 exitProcess("Please specify the output filename");
299 }
300 outputFileName = argv[i + 1];
301 break;
302 case "-i":
303 if (!argv[i + 1])
304 {
305 exitProcess("Please specify the import file");
306 }
307 const importFile = argv[i + 1];
308 importTranslations(importFile);
309 stopExportScript = true;
310 break;
311 }
312 }
313
314 if (!stopExportScript)
315 exportTranslations(filesFilter);
OLDNEW
« no previous file with comments | « README.md ('k') | package.json » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld