Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: build/csv-export.js

Issue 29636585: Issue 6171 - create CSV exporter and importer for translations (Closed)
Left Patch Set: Removed mercurial commands Created May 17, 2018, 5:23 p.m.
Right Patch Set: Created June 18, 2018, 2:49 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « README.md ('k') | package.json » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 /* 1 /*
saroyanm 2018/05/24 17:00:59 In some stage of import/export This "\u00A0" speci
Thomas Greiner 2018/05/25 10:23:28 Based on what I see in https://gitlab.com/eyeo/adb
saroyanm 2018/05/28 13:37:08 Yes, it's there because I've added them manually.
Thomas Greiner 2018/05/28 17:12:46 No particular reason. It was just easier to write
2 * This file is part of Adblock Plus <https://adblockplus.org/>, 2 * This file is part of Adblock Plus <https://adblockplus.org/>,
3 * Copyright (C) 2006-present eyeo GmbH 3 * Copyright (C) 2006-present eyeo GmbH
4 * 4 *
5 * Adblock Plus is free software: you can redistribute it and/or modify 5 * Adblock Plus is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 3 as 6 * it under the terms of the GNU General Public License version 3 as
7 * published by the Free Software Foundation. 7 * published by the Free Software Foundation.
8 * 8 *
9 * Adblock Plus is distributed in the hope that it will be useful, 9 * Adblock Plus is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16 */ 16 */
17 17
18 /* globals process */ 18 /* globals process */
19 19
20 "use strict"; 20 "use strict";
21 21
22 const fs = require("fs"); 22 const fs = require("fs");
23 const path = require("path"); 23 const path = require("path");
24 const csv = require("csv"); 24 const csv = require("csv");
25 const {promisify} = require("util"); 25 const {promisify} = require("util");
26 const execFile = promisify(require("child_process").execFile); 26 const execFile = promisify(require("child_process").execFile);
27 const csvParser = promisify(csv.parse); 27 const csvParser = promisify(csv.parse);
28 const readFile = promisify(fs.readFile); 28 const readFile = promisify(fs.readFile);
29 const writeFile = promisify(fs.writeFile);
29 const glob = promisify(require("glob").glob); 30 const glob = promisify(require("glob").glob);
30 const readJsonPromised = promisify(readJson);
31 31
32 const localesDir = "locale"; 32 const localesDir = "locale";
33 const defaultLocale = "en_US"; 33 const defaultLocale = "en_US";
34 34
35 let headers = ["Filename", "StringID", "Description", "Placeholders", 35 const headers = ["Filename", "StringID", "Description", "Placeholders",
36 defaultLocale]; 36 defaultLocale];
37 let outputFileName = "translations.csv"; 37 let outputFileName = "translations.csv";
38 38
39 /** 39 /**
40 * Export existing translation - files into CSV file 40 * Export existing translation - files into CSV file
41 */ 41 */
42 function exportTranslations() 42 function exportTranslations()
43 { 43 {
44 glob(`${localesDir}/**/*.json`).then((filePaths) => 44 glob(`${localesDir}/**/*.json`).then((filePaths) =>
45 { 45 {
46 // Reading all existing translations files 46 // Reading all existing translations files
47 return Promise.all(filePaths.map((filePath) => readJsonPromised(filePath))); 47 return Promise.all(filePaths.map((filePath) => readJson(filePath)));
48 }).then(csvFromJsonFileObjects); 48 }).then(csvFromJsonFileObjects);
49 } 49 }
50 50
51 /** 51 /**
52 * Creating Matrix which reflects output CSV file 52 * Creating Matrix which reflects output CSV file
53 * @param {Object[]} fileObjects - array of file objects created by readJson 53 * @param {Object[]} fileObjects - array of file objects created by readJson
54 */ 54 */
55 function csvFromJsonFileObjects(fileObjects) 55 function csvFromJsonFileObjects(fileObjects)
56 { 56 {
57 let locales = []; 57 const locales = [];
58 const fileNames = [];
58 // Create Object tree from the Objects array, for easier search 59 // Create Object tree from the Objects array, for easier search
59 // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}} 60 // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
60 let dataTreeObj = Object.create(null); 61 const dataTreeObj = Object.create(null);
61 for (let fileObject of fileObjects) 62 for (const fileObject of fileObjects)
62 { 63 {
63 const {fileName, locale, strings} = fileObject; 64 const {fileName, locale, strings} = fileObject;
64 65
65 if (!locales.includes(locale)) 66 if (locale != defaultLocale && !locales.includes(locale))
66 locales.push(locale); 67 locales.push(locale);
67 68
68 if (!dataTreeObj[fileName]) 69 if ((!filesFilter.length || filesFilter.includes(fileName)) &&
69 dataTreeObj[fileName] = {}; 70 !fileNames.includes(fileName))
70 if (!dataTreeObj[fileName][locale]) 71 fileNames.push(fileName);
71 dataTreeObj[fileName][locale] = {}; 72
73 if (!(fileName in dataTreeObj))
74 dataTreeObj[fileName] = Object.create(null);
75
72 dataTreeObj[fileName][locale] = strings; 76 dataTreeObj[fileName][locale] = strings;
73 } 77 }
74
75 let fileNames = Object.keys(dataTreeObj);
76 if (filesFilter.length)
77 fileNames = fileNames.filter((item) => filesFilter.includes(item));
78
79 locales = locales.filter((locale) => locale != defaultLocale).sort();
Thomas Greiner 2018/06/12 14:50:41 Is there no need for sorting locales anymore?
saroyanm 2018/06/12 15:22:49 If you mean default strings: See -> https://gitlab
80 // Create two dimensional strings array that reflects CSV structure 78 // Create two dimensional strings array that reflects CSV structure
81 let csvArray = [headers.concat(locales)]; 79 const csvArray = [headers.concat(locales)];
82 for (let fileName of fileNames) 80 for (const fileName of fileNames)
83 { 81 {
84 let strings = dataTreeObj[fileName][defaultLocale]; 82 const strings = dataTreeObj[fileName][defaultLocale];
85 for (let stringID of Object.keys(strings)) 83 for (const stringID of Object.keys(strings))
86 { 84 {
87 let fileObj = dataTreeObj[fileName]; 85 const fileObj = dataTreeObj[fileName];
88 let {description, message, placeholders} = strings[stringID]; 86 const {description, message, placeholders} = strings[stringID];
89 let row = [fileName, stringID, description || "", 87 const row = [fileName, stringID, description || "",
90 JSON.stringify(placeholders), message]; 88 JSON.stringify(placeholders), message];
91 89
92 for (let locale of locales) 90 for (const locale of locales)
93 { 91 {
94 let localeFileObj = fileObj[locale]; 92 const localeFileObj = fileObj[locale];
95 let isTranslated = !!(localeFileObj && localeFileObj[stringID]); 93 const isTranslated = !!(localeFileObj && localeFileObj[stringID]);
96 row.push(isTranslated ? localeFileObj[stringID].message : ""); 94 row.push(isTranslated ? localeFileObj[stringID].message : "");
97 } 95 }
98 csvArray.push(row); 96 csvArray.push(row);
99 } 97 }
100 } 98 }
101 arrayToCsv(csvArray); 99 arrayToCsv(csvArray);
102 } 100 }
103 101
104 /** 102 /**
105 * Import strings from the CSV file 103 * Import strings from the CSV file
106 * @param {string} filePath - CSV file path to import from 104 * @param {string} filePath - CSV file path to import from
107 */ 105 */
108 function importTranslations(filePath) 106 function importTranslations(filePath)
109 { 107 {
110 readFile(filePath, "utf8").then((fileObjects) => 108 readFile(filePath, "utf8").then((fileObjects) =>
111 { 109 {
112 return csvParser(fileObjects); 110 return csvParser(fileObjects);
saroyanm 2018/05/24 16:40:02 I think there might a bug in csvParser -> https://
113 }).then((dataMatrix) => 111 }).then((dataMatrix) =>
114 { 112 {
115 let headLocales = dataMatrix.shift().slice(4); 113 const headLocales = dataMatrix.shift().slice(4);
116 let dataTreeObj = {}; 114 const dataTreeObj = {};
117 for (let rowId in dataMatrix) 115 for (const rowId in dataMatrix)
118 { 116 {
119 let row = dataMatrix[rowId]; 117 const row = dataMatrix[rowId];
120 let [currentFilename, stringId, description, placeholder, ...messages] = 118 let [currentFilename, stringId, description, placeholder, ...messages] =
121 row; 119 row;
122 if (!stringId) 120 if (!stringId)
123 continue; 121 continue;
124 122
125 stringId = stringId.trim(); 123 stringId = stringId.trim();
126 // Check if it's the filename row 124 // Check if it's the filename row
127 if (!dataTreeObj[currentFilename]) 125 if (!dataTreeObj[currentFilename])
128 dataTreeObj[currentFilename] = {}; 126 dataTreeObj[currentFilename] = {};
129 127
130 description = description.trim(); 128 description = description.trim();
131 for (let i = 0; i < headLocales.length; i++) 129 for (let i = 0; i < headLocales.length; i++)
132 { 130 {
133 let locale = headLocales[i].trim(); 131 const locale = headLocales[i].trim();
134 let message = messages[i].trim(); 132 const message = messages[i].trim();
135 if (!message) 133 if (!message)
136 continue; 134 continue;
137 135
138 // Create Object tree from the Objects array, for easier search 136 // Create Object tree from the Objects array, for easier search
139 // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}} 137 // ex.: {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
140 if (!dataTreeObj[currentFilename][locale]) 138 if (!dataTreeObj[currentFilename][locale])
141 dataTreeObj[currentFilename][locale] = {}; 139 dataTreeObj[currentFilename][locale] = {};
142 140
143 let localeObj = dataTreeObj[currentFilename][locale]; 141 const localeObj = dataTreeObj[currentFilename][locale];
144 localeObj[stringId] = {}; 142 localeObj[stringId] = {};
145 let stringObj = localeObj[stringId]; 143 const stringObj = localeObj[stringId];
146 144
147 // We keep string descriptions only in default locale files 145 // We keep string descriptions only in default locale files
148 if (locale == defaultLocale) 146 if (locale == defaultLocale && description)
saroyanm 2018/05/24 13:29:26 I've changed this into if (locale == defaultLocale
Thomas Greiner 2018/05/24 13:49:42 I think your suggested solution (i.e. only include
149 stringObj.description = description; 147 stringObj.description = description;
150 148
151 stringObj.message = message; 149 stringObj.message = message;
152 if (placeholder) 150 if (placeholder)
153 stringObj.placeholders = JSON.parse(placeholder); 151 stringObj.placeholders = JSON.parse(placeholder);
154 } 152 }
155 } 153 }
156 writeJson(dataTreeObj); 154 writeJson(dataTreeObj);
157 }); 155 });
158 } 156 }
159 157
160 /** 158 /**
161 * Write locale files according to dataTreeObj 159 * Write locale files according to dataTreeObj
162 * @param {Object} dataTreeObj - ex.: 160 * @param {Object} dataTreeObj - ex.:
163 * {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}} 161 * {dektop-options.json: {en_US: {...}, {de: {...}, {ru: {...}}}
164 */ 162 */
165 function writeJson(dataTreeObj) 163 function writeJson(dataTreeObj)
166 { 164 {
167 for (let fileName in dataTreeObj) 165 for (const fileName in dataTreeObj)
168 { 166 {
169 for (let locale in dataTreeObj[fileName]) 167 for (const locale in dataTreeObj[fileName])
170 { 168 {
171 let filePath = path.join(localesDir, locale, fileName); 169 const filePath = path.join(localesDir, locale, fileName);
172 let sortedJSON = orderJSON(dataTreeObj[fileName][locale]); 170 const sortedJson = sortJson(dataTreeObj[fileName][locale]);
saroyanm 2018/05/24 15:29:54 Sorting the Default locale produce an unreadable d
Thomas Greiner 2018/05/24 15:46:46 I understand. In that case let's sort the default
saroyanm 2018/06/05 15:03:47 I've created a gitlab issue in order to discuss th
173 let fileString = JSON.stringify(sortedJSON, null, 2); 171 let fileString = JSON.stringify(sortedJson, null, 2);
174 172
175 // Newline at end of file to match Coding Style 173 // Newline at end of file to match Coding Style
176 if (locale == defaultLocale) 174 if (locale == defaultLocale)
177 fileString += "\n"; 175 fileString += "\n";
178 fs.writeFile(filePath, fileString, "utf8", (err) => 176 writeFile(filePath, fileString, "utf8").then(() =>
179 { 177 {
180 if (err) 178 console.log(`Updated: ${filePath}`); // eslint-disable-line no-console
181 { 179 }).catch((err) =>
182 console.error(err); 180 {
183 } 181 console.error(err);
184 else
185 {
186 console.log(`Updated: ${filePath}`);
187 }
188 }); 182 });
189 } 183 }
190 } 184 }
191 } 185 }
192 186
193 /** 187 /**
194 * This function currently relies on V8 to sort the object by keys 188 * This function currently relies on V8 to sort the object by keys
195 * @param {Object} unordered - json object 189 * @param {Object} unordered - json object
196 * @returns {Object} 190 * @returns {Object}
197 */ 191 */
198 function orderJSON(unordered) 192 function sortJson(unordered)
199 { 193 {
200 const ordered = {}; 194 const ordered = {};
201 for (let key of Object.keys(unordered).sort()) 195 for (const key of Object.keys(unordered).sort())
202 { 196 {
203 ordered[key] = unordered[key]; 197 ordered[key] = unordered[key];
204 if (unordered[key].placeholders) 198 if (unordered[key].placeholders)
205 ordered[key].placeholders = orderJSON(unordered[key].placeholders); 199 ordered[key].placeholders = sortJson(unordered[key].placeholders);
206 200
207 ordered[key] = unordered[key]; 201 ordered[key] = unordered[key];
208 } 202 }
209 return ordered; 203 return ordered;
210 } 204 }
211 205
212 /** 206 /**
213 * Convert two dimensional array to the CSV file 207 * Convert two dimensional array to the CSV file
214 * @param {Object[]} csvArray - array to convert from 208 * @param {Object[]} csvArray - array to convert from
215 */ 209 */
216 function arrayToCsv(csvArray) 210 function arrayToCsv(csvArray)
217 { 211 {
218 csv.stringify(csvArray, (err, output) => 212 csv.stringify(csvArray, (err, output) =>
219 { 213 {
220 fs.writeFile(outputFileName, output, "utf8", (error) => 214 writeFile(outputFileName, output, "utf8").then(() =>
221 { 215 {
222 if (!error) 216 // eslint-disable-next-line no-console
223 console.log(`${outputFileName} is created`); 217 console.log(`${outputFileName} is created`);
224 else 218 }).catch((error) =>
225 console.error(error); 219 {
220 console.error(error);
226 }); 221 });
227 }); 222 });
228 } 223 }
229 224
230 /** 225 /**
231 * Reads JSON file and assign filename and locale to it 226 * Reads JSON file and assign filename and locale to it
232 * @param {string} filePath - ex.: "locales/en_US/desktop-options.json" 227 * @param {string} filePath - ex.: "locales/en_US/desktop-options.json"
233 * @param {function} callback - fileName, locale and strings of locale file 228 * @returns {Promise<Object>} resolves fileName, locale and strings of the
234 * Parameters: 229 * locale file
235 * * Error message 230 */
236 * * Object containing fileName, locale and strings 231 function readJson(filePath)
237 */ 232 {
238 function readJson(filePath, callback) 233 return readFile(filePath, "utf8").then((data) =>
239 { 234 {
240 let {dir, base} = path.parse(filePath); 235 const {dir, base} = path.parse(filePath);
241 fs.readFile(filePath, "utf8", (err, data) => 236 const locale = dir.split(path.sep).pop();
242 { 237 const strings = JSON.parse(data);
243 if (err) 238 return {fileName: base, locale, strings};
244 {
245 callback(err);
246 }
247 else
248 {
249 let locale = dir.split(path.sep).pop();
250 let strings = JSON.parse(data);
251 callback(null, {fileName: base, locale, strings});
252 }
253 }); 239 });
254 } 240 }
255 241
256 /** 242 /**
257 * Exit process and log error message 243 * Exit process and log error message
258 * @param {String} error error message 244 * @param {String} error error message
259 */ 245 */
260 function exitProcess(error) 246 function exitProcess(error)
261 { 247 {
262 console.error(error); 248 console.error(error);
263 process.exit(1); 249 process.exit(1);
264 } 250 }
265 251
266 // CLI 252 // CLI
267 let helpText = ` 253 const helpText = `
268 About: Converts locale files between CSV and JSON formats 254 About: Converts locale files between CSV and JSON formats
269 Usage: csv-export.js [option] [argument] 255 Usage: csv-export.js [option] [argument]
270 Options: 256 Options:
271 -f [FILENAME] Name of the files to be exported ex.: -f firstRun.json 257 -f [FILENAME] Name of the files to be exported ex.: -f firstRun.json
272 option can be used multiple times. 258 option can be used multiple times.
273 If omitted all files are being exported 259 If omitted all files are being exported
274 260
275 -o [FILENAME] Output filename ex.: 261 -o [FILENAME] Output filename ex.:
276 -f firstRun.json -o {hash}-firstRun.csv 262 -f firstRun.json -o firstRun.csv
277 Placeholders:
278 {hash} - Mercurial current revision hash
279 {repo} - Name of the "Default" repository
Thomas Greiner 2018/05/22 17:22:50 Detail: These placeholders no longer exist.
saroyanm 2018/06/05 15:03:47 Done.
280 If omitted the output fileName is set to 263 If omitted the output fileName is set to
281 translations-{repo}-{hash}.csv 264 translations.csv
282 265
283 -i [FILENAME] Import file path ex: -i issue-reporter.csv 266 -i [FILENAME] Import file path ex: -i issue-reporter.csv
284 `; 267 `;
285 268
286 let argv = process.argv.slice(2); 269 const argv = process.argv.slice(2);
287 let stopExportScript = false; 270 let stopExportScript = false;
288 // Filter to be used export to the fileNames inside 271 // Filter to be used export to the fileNames inside
289 let filesFilter = []; 272 const filesFilter = [];
290 273
291 for (let i = 0; i < argv.length; i++) 274 for (let i = 0; i < argv.length; i++)
292 { 275 {
293 switch (argv[i]) 276 switch (argv[i])
294 { 277 {
295 case "-h": 278 case "-h":
296 console.log(helpText); 279 console.log(helpText); // eslint-disable-line no-console
297 stopExportScript = true; 280 stopExportScript = true;
298 break; 281 break;
299 case "-f": 282 case "-f":
300 if (!argv[i + 1]) 283 if (!argv[i + 1])
301 { 284 {
302 exitProcess("Please specify the input filename"); 285 exitProcess("Please specify the input filename");
303 } 286 }
304 filesFilter.push(argv[i + 1]); 287 filesFilter.push(argv[i + 1]);
305 break; 288 break;
306 case "-o": 289 case "-o":
307 if (!argv[i + 1]) 290 if (!argv[i + 1])
308 { 291 {
309 exitProcess("Please specify the output filename"); 292 exitProcess("Please specify the output filename");
310 } 293 }
311 outputFileName = argv[i + 1]; 294 outputFileName = argv[i + 1];
312 break; 295 break;
313 case "-i": 296 case "-i":
314 if (!argv[i + 1]) 297 if (!argv[i + 1])
315 { 298 {
316 exitProcess("Please specify the import file"); 299 exitProcess("Please specify the import file");
317 } 300 }
318 let importFile = argv[i + 1]; 301 const importFile = argv[i + 1];
319 importTranslations(importFile); 302 importTranslations(importFile);
320 stopExportScript = true; 303 stopExportScript = true;
321 break; 304 break;
322 } 305 }
323 } 306 }
324 307
325 if (!stopExportScript) 308 if (!stopExportScript)
326 exportTranslations(filesFilter); 309 exportTranslations(filesFilter);
LEFTRIGHT

Powered by Google App Engine
This is Rietveld