Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: localeTools.py

Issue 29336281: Issue 2109 - Allow for translation of app independent repositories (Closed)
Patch Set: Addressed Wladimir's initial feedback Created Feb. 12, 2016, 6:23 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« build.py ('K') | « build.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This Source Code Form is subject to the terms of the Mozilla Public 3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this 4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 6
7 import re, os, sys, codecs, json, urllib, urllib2 7 import re, os, sys, codecs, json, urllib, urllib2
8 from StringIO import StringIO 8 from StringIO import StringIO
9 from ConfigParser import SafeConfigParser 9 from ConfigParser import SafeConfigParser
10 from zipfile import ZipFile 10 from zipfile import ZipFile
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after
101 yield (key, self[key]) 101 yield (key, self[key])
102 done.add(key) 102 done.add(key)
103 103
104 def escapeEntity(value): 104 def escapeEntity(value):
105 return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').r eplace('"', '&quot;') 105 return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').r eplace('"', '&quot;')
106 106
107 def unescapeEntity(value): 107 def unescapeEntity(value):
108 return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').r eplace('&quot;', '"') 108 return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').r eplace('&quot;', '"')
109 109
110 def mapLocale(type, locale): 110 def mapLocale(type, locale):
111 mapping = langMappingChrome if type == 'chrome' else langMappingGecko 111 mapping = langMappingChrome if type == 'ISO-15897' else langMappingGecko
112 return mapping.get(locale, locale) 112 return mapping.get(locale, locale)
113 113
114 def parseDTDString(data, path): 114 def parseDTDString(data, path):
115 result = [] 115 result = []
116 currentComment = [None] 116 currentComment = [None]
117 117
118 parser = ParserCreate() 118 parser = ParserCreate()
119 parser.UseForeignDTD(True) 119 parser.UseForeignDTD(True)
120 parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS) 120 parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS)
121 121
(...skipping 143 matching lines...) Expand 10 before | Expand all | Expand 10 after
265 265
266 # Delete description from translations 266 # Delete description from translations
267 for key, value in parsed.iteritems(): 267 for key, value in parsed.iteritems():
268 if "description" in value: 268 if "description" in value:
269 del value["description"] 269 del value["description"]
270 270
271 file = codecs.open(path, 'wb', encoding='utf-8') 271 file = codecs.open(path, 'wb', encoding='utf-8')
272 json.dump(parsed, file, ensure_ascii=False, sort_keys=True, indent=2, separato rs=(',', ': ')) 272 json.dump(parsed, file, ensure_ascii=False, sort_keys=True, indent=2, separato rs=(',', ': '))
273 file.close() 273 file.close()
274 274
275 def setupTranslations(type, locales, projectName, key): 275 def setupTranslations(localeConfig, projectName, key):
276 # Copy locales list, we don't want to change the parameter 276 # Make a new set from the locales list, mapping to Crowdin friendly format
277 locales = set(locales) 277 locales = {mapLocale(localeConfig['name_format'], locale)
278 for locale in localeConfig['locales']}
278 279
279 # Fill up with locales that we don't have but the browser supports 280 # Fill up with locales that we don't have but the browser supports
280 if type == 'chrome': 281 if 'chrome' in localeConfig['target_platforms']:
281 for locale in chromeLocales: 282 for locale in chromeLocales:
282 locales.add(locale) 283 locales.add(mapLocale('ISO-15897', locale))
283 else: 284
285 if 'gecko' in localeConfig['target_platforms']:
284 firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.h tml').read() 286 firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.h tml').read()
285 for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales): 287 for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
286 locales.add(mapLocale(type, match.group(1))) 288 locales.add(mapLocale('BCP-47', match.group(1)))
287 langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/langua ge-tools/').read() 289 langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/langua ge-tools/').read()
288 for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S): 290 for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
289 if match.group(0).find('Install Language Pack') >= 0: 291 if match.group(0).find('Install Language Pack') >= 0:
290 match2 = re.search(r'lang="([\w\-]+)"', match.group(0)) 292 match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
291 if match2: 293 if match2:
292 locales.add(mapLocale(type, match2.group(1))) 294 locales.add(mapLocale('BCP-47', match2.group(1)))
293
294 # Convert locale codes to the ones that Crowdin will understand
295 locales = set(map(lambda locale: mapLocale(type, locale), locales))
296 295
297 allowed = set() 296 allowed = set()
298 allowedLocales = urllib2.urlopen('http://crowdin.net/page/language-codes').rea d() 297 allowedLocales = urllib2.urlopen('http://crowdin.net/page/language-codes').rea d()
299 for match in re.finditer(r'<tr>\s*<td\b[^<>]*>([\w\-]+)</td>', allowedLocales, re.S): 298 for match in re.finditer(r'<tr>\s*<td\b[^<>]*>([\w\-]+)</td>', allowedLocales, re.S):
300 allowed.add(match.group(1)) 299 allowed.add(match.group(1))
301 if not allowed.issuperset(locales): 300 if not allowed.issuperset(locales):
302 print 'Warning, following locales aren\'t allowed by server: ' + ', '.join(l ocales - allowed) 301 print 'Warning, following locales aren\'t allowed by server: ' + ', '.join(l ocales - allowed)
303 302
304 locales = list(locales & allowed) 303 locales = list(locales & allowed)
305 locales.sort() 304 locales.sort()
(...skipping 14 matching lines...) Expand all
320 body += '--%s--\r\n' % boundary 319 body += '--%s--\r\n' % boundary
321 320
322 body = body.encode('utf-8') 321 body = body.encode('utf-8')
323 request = urllib2.Request(url, StringIO(body)) 322 request = urllib2.Request(url, StringIO(body))
324 request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % bounda ry) 323 request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % bounda ry)
325 request.add_header('Content-Length', len(body)) 324 request.add_header('Content-Length', len(body))
326 result = urllib2.urlopen(request).read() 325 result = urllib2.urlopen(request).read()
327 if result.find('<success') < 0: 326 if result.find('<success') < 0:
328 raise Exception('Server indicated that the operation was not successful\n' + result) 327 raise Exception('Server indicated that the operation was not successful\n' + result)
329 328
330 def updateTranslationMaster(type, metadata, dir, projectName, key): 329 def updateTranslationMaster(localeConfig, metadata, dir, projectName, key):
331 result = json.load(urllib2.urlopen('http://api.crowdin.net/api/project/%s/info ?key=%s&json=1' % (projectName, key))) 330 result = json.load(urllib2.urlopen('http://api.crowdin.net/api/project/%s/info ?key=%s&json=1' % (projectName, key)))
332 331
333 existing = set(map(lambda f: f['name'], result['files'])) 332 existing = set(map(lambda f: f['name'], result['files']))
334 add = [] 333 add = []
335 update = [] 334 update = []
336 for file in os.listdir(dir): 335 for file in os.listdir(dir):
337 path = os.path.join(dir, file) 336 path = os.path.join(dir, file)
338 if os.path.isfile(path): 337 if os.path.isfile(path):
339 if type == 'chrome' and file.endswith('.json'): 338 if localeConfig['file_format'] == 'chrome-json' and file.endswith('.json') :
340 data = preprocessChromeLocale(path, metadata, True) 339 data = preprocessChromeLocale(path, metadata, True)
341 newName = file 340 newName = file
342 elif type == 'chrome': 341 elif localeConfig['file_format'] == 'chrome-json':
343 fileHandle = codecs.open(path, 'rb', encoding='utf-8') 342 fileHandle = codecs.open(path, 'rb', encoding='utf-8')
344 data = json.dumps({file: {'message': fileHandle.read()}}) 343 data = json.dumps({file: {'message': fileHandle.read()}})
345 fileHandle.close() 344 fileHandle.close()
346 newName = file + '.json' 345 newName = file + '.json'
347 else: 346 else:
348 data = toJSON(path) 347 data = toJSON(path)
349 newName = file + '.json' 348 newName = file + '.json'
350 349
351 if data: 350 if data:
352 if newName in existing: 351 if newName in existing:
353 update.append((newName, data)) 352 update.append((newName, data))
354 existing.remove(newName) 353 existing.remove(newName)
355 else: 354 else:
356 add.append((newName, data)) 355 add.append((newName, data))
357 356
358 if len(add): 357 if len(add):
359 titles = urllib.urlencode([('titles[%s]' % name, re.sub(r'\.json', '', name) ) for name, data in add]) 358 titles = urllib.urlencode([('titles[%s]' % name, re.sub(r'\.json', '', name) ) for name, data in add])
360 postFiles(add, 'http://api.crowdin.net/api/project/%s/add-file?key=%s&type=c hrome&%s' % (projectName, key, titles)) 359 postFiles(add, 'http://api.crowdin.net/api/project/%s/add-file?key=%s&type=c hrome&%s' % (projectName, key, titles))
361 if len(update): 360 if len(update):
362 postFiles(update, 'http://api.crowdin.net/api/project/%s/update-file?key=%s' % (projectName, key)) 361 postFiles(update, 'http://api.crowdin.net/api/project/%s/update-file?key=%s' % (projectName, key))
363 for file in existing: 362 for file in existing:
364 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/delete-file? key=%s&file=%s' % (projectName, key, file)).read() 363 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/delete-file? key=%s&file=%s' % (projectName, key, file)).read()
365 if result.find('<success') < 0: 364 if result.find('<success') < 0:
366 raise Exception('Server indicated that the operation was not successful\n' + result) 365 raise Exception('Server indicated that the operation was not successful\n' + result)
367 366
368 def uploadTranslations(type, metadata, dir, locale, projectName, key): 367 def uploadTranslations(localeConfig, metadata, dir, locale, projectName, key):
369 files = [] 368 files = []
370 for file in os.listdir(dir): 369 for file in os.listdir(dir):
371 path = os.path.join(dir, file) 370 path = os.path.join(dir, file)
372 if os.path.isfile(path): 371 if os.path.isfile(path):
373 if type == 'chrome' and file.endswith('.json'): 372 if localeConfig['file_format'] == 'chrome-json' and file.endswith('.json') :
374 data = preprocessChromeLocale(path, metadata, False) 373 data = preprocessChromeLocale(path, metadata, False)
375 newName = file 374 newName = file
376 elif type == 'chrome': 375 elif localeConfig['file_format'] == 'chrome-json':
377 fileHandle = codecs.open(path, 'rb', encoding='utf-8') 376 fileHandle = codecs.open(path, 'rb', encoding='utf-8')
378 data = json.dumps({file: {'message': fileHandle.read()}}) 377 data = json.dumps({file: {'message': fileHandle.read()}})
379 fileHandle.close() 378 fileHandle.close()
380 newName = file + '.json' 379 newName = file + '.json'
381 else: 380 else:
382 data = toJSON(path) 381 data = toJSON(path)
383 newName = file + '.json' 382 newName = file + '.json'
384 383
385 if data: 384 if data:
386 files.append((newName, data)) 385 files.append((newName, data))
387 if len(files): 386 if len(files):
388 postFiles(files, 'http://api.crowdin.net/api/project/%s/upload-translation?k ey=%s&language=%s' % (projectName, key, mapLocale(type, locale))) 387 postFiles(files, 'http://api.crowdin.net/api/project/%s/upload-translation?k ey=%s&language=%s' % (
388 projectName, key, mapLocale(localeConfig['name_format'], locale))
389 )
389 390
390 def getTranslations(type, localesDir, defaultLocale, projectName, key): 391 def getTranslations(localeConfig, projectName, key):
391 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read() 392 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read()
392 if result.find('<success') < 0: 393 if result.find('<success') < 0:
393 raise Exception('Server indicated that the operation was not successful\n' + result) 394 raise Exception('Server indicated that the operation was not successful\n' + result)
394 395
395 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.z ip?key=%s' % (projectName, key)).read() 396 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.z ip?key=%s' % (projectName, key)).read()
396 zip = ZipFile(StringIO(result)) 397 zip = ZipFile(StringIO(result))
397 dirs = {} 398 dirs = {}
398 for info in zip.infolist(): 399 for info in zip.infolist():
399 if not info.filename.endswith('.json'): 400 if not info.filename.endswith('.json'):
400 continue 401 continue
401 402
402 dir, file = os.path.split(info.filename) 403 dir, file = os.path.split(info.filename)
403 if not re.match(r'^[\w\-]+$', dir) or dir == defaultLocale: 404 if not re.match(r'^[\w\-]+$', dir) or dir == localeConfig['default_locale']:
Wladimir Palant 2016/02/12 19:05:44 This check won't work correctly for Chrome, we wil
kzar 2016/02/12 19:40:26 Good point but I think we also need to replace the
404 continue 405 continue
405 if type == 'chrome' and file.count('.') == 1: 406 if localeConfig['file_format'] == 'chrome-json' and file.count('.') == 1:
406 origFile = file 407 origFile = file
407 else: 408 else:
408 origFile = re.sub(r'\.json$', '', file) 409 origFile = re.sub(r'\.json$', '', file)
409 if type == 'gecko' and not origFile.endswith('.dtd') and not origFile.endswi th('.properties'): 410 if (localeConfig['file_format'] == 'gecko-dtd' and
411 not origFile.endswith('.dtd') and
412 not origFile.endswith('.properties')):
410 continue 413 continue
411 414
412 mapping = langMappingChrome if type == 'chrome' else langMappingGecko 415 if localeConfig['name_format'] == 'ISO-15897':
416 mapping = langMappingChrome
417 else:
418 mapping = langMappingGecko
419
413 for key, value in mapping.iteritems(): 420 for key, value in mapping.iteritems():
414 if value == dir: 421 if value == dir:
415 dir = key 422 dir = key
416 if type == 'chrome': 423 if localeConfig['name_format'] == 'ISO-15897':
417 dir = dir.replace('-', '_') 424 dir = dir.replace('-', '_')
418 425
419 data = zip.open(info.filename).read() 426 data = zip.open(info.filename).read()
420 if data == '[]': 427 if data == '[]':
421 continue 428 continue
422 429
423 if not dir in dirs: 430 if not dir in dirs:
424 dirs[dir] = set() 431 dirs[dir] = set()
425 dirs[dir].add(origFile) 432 dirs[dir].add(origFile)
426 433
427 path = os.path.join(localesDir, dir, origFile) 434 path = os.path.join(localeConfig['base_path'], dir, origFile)
428 if not os.path.exists(os.path.dirname(path)): 435 if not os.path.exists(os.path.dirname(path)):
429 os.makedirs(os.path.dirname(path)) 436 os.makedirs(os.path.dirname(path))
430 if type == 'chrome' and origFile.endswith('.json'): 437 if localeConfig['file_format'] == 'chrome-json' and file.endswith('.json'):
431 postprocessChromeLocale(path, data) 438 postprocessChromeLocale(path, data)
432 elif type == 'chrome': 439 elif localeConfig['file_format'] == 'chrome-json':
433 data = json.loads(data) 440 data = json.loads(data)
434 if origFile in data: 441 if origFile in data:
435 fileHandle = codecs.open(path, 'wb', encoding='utf-8') 442 fileHandle = codecs.open(path, 'wb', encoding='utf-8')
436 fileHandle.write(data[origFile]['message']) 443 fileHandle.write(data[origFile]['message'])
437 fileHandle.close() 444 fileHandle.close()
438 else: 445 else:
439 fromJSON(path, data) 446 fromJSON(path, data)
440 447
441 # Remove any extra files 448 # Remove any extra files
442 for dir, files in dirs.iteritems(): 449 for dir, files in dirs.iteritems():
443 baseDir = os.path.join(localesDir, dir) 450 baseDir = os.path.join(localeConfig['base_path'], dir)
444 if not os.path.exists(baseDir): 451 if not os.path.exists(baseDir):
445 continue 452 continue
446 for file in os.listdir(baseDir): 453 for file in os.listdir(baseDir):
447 path = os.path.join(baseDir, file) 454 path = os.path.join(baseDir, file)
448 if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.pro perties') or file.endswith('.dtd')) and not file in files: 455 if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.pro perties') or file.endswith('.dtd')) and not file in files:
449 os.remove(path) 456 os.remove(path)
OLDNEW
« build.py ('K') | « build.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld