Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: localeTools.py

Issue 29556601: Issue 5777 - Update crowdin interface (Closed)
Patch Set: Print actual HTTP Error Code on Exception Created Sept. 28, 2017, 12:11 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This Source Code Form is subject to the terms of the Mozilla Public 1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this 2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
5 import re 5 import re
6 import os 6 import os
7 import sys 7 import sys
8 import codecs 8 import codecs
9 import json 9 import json
10 import urlparse
10 import urllib 11 import urllib
11 import urllib2 12 import urllib2
12 from StringIO import StringIO 13 from StringIO import StringIO
13 from ConfigParser import SafeConfigParser 14 from ConfigParser import SafeConfigParser
14 from zipfile import ZipFile 15 from zipfile import ZipFile
15 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS 16 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
16 17
17 langMappingGecko = { 18 langMappingGecko = {
18 'bn-BD': 'bn', 19 'bn-BD': 'bn',
19 'br': 'br-FR', 20 'br': 'br-FR',
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after
85 'ta', 86 'ta',
86 'te', 87 'te',
87 'th', 88 'th',
88 'tr', 89 'tr',
89 'uk', 90 'uk',
90 'vi', 91 'vi',
91 'zh-CN', 92 'zh-CN',
92 'zh-TW', 93 'zh-TW',
93 ] 94 ]
94 95
96 CROWDIN_AP_URL = 'https://api.crowdin.com/api/project/{}/{}'
97
98
99 def crowdin_url(project_name, action, key, get={}):
100 """Create a valid url for a crowdin endpoint."""
101 url = CROWDIN_AP_URL.format(project_name, action)
102 get['key'] = key
103 get['json'] = 1
104
105 scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
106
107 query = dict(urlparse.parse_qsl(query))
Vasily Kuznetsov 2017/09/28 14:35:00 Sorry that I'm only mentioning this now, but is th
tlucas 2017/09/28 15:53:41 You are right - Done.
108 query.update(get)
109
110 return urlparse.urlunparse((
111 scheme, netloc, path, params, urllib.urlencode(query), fragment
112 ))
113
114
115 def crowdin_request(project_name, action, key, get={}, post_data=None,
116 headers={}, raw=False):
117 """Perform a call to crowdin and raise an Exception on failure."""
118 request = urllib2.Request(
119 crowdin_url(project_name, action, key, get),
120 post_data,
121 headers,
122 )
123
124 try:
125 result = urllib2.urlopen(request).read()
126 except urllib2.HTTPError as e:
127 raise Exception('Server returned HTTP Error {}:\n{}'.format(e.code,
128 e.read()))
129
130 if not raw:
131 return json.loads(result)
132
133 return result
134
95 135
96 class OrderedDict(dict): 136 class OrderedDict(dict):
97 def __init__(self): 137 def __init__(self):
98 self.__order = [] 138 self.__order = []
99 139
100 def __setitem__(self, key, value): 140 def __setitem__(self, key, value):
101 self.__order.append(key) 141 self.__order.append(key)
102 dict.__setitem__(self, key, value) 142 dict.__setitem__(self, key, value)
103 143
104 def iteritems(self): 144 def iteritems(self):
(...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after
290 for match in re.finditer(r'&lang=([\w\-]+)"', firefoxLocales): 330 for match in re.finditer(r'&lang=([\w\-]+)"', firefoxLocales):
291 locales.add(mapLocale('BCP-47', match.group(1))) 331 locales.add(mapLocale('BCP-47', match.group(1)))
292 langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/la nguage-tools/').read() 332 langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/la nguage-tools/').read()
293 for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S): 333 for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
294 if match.group(0).find('Install Language Pack') >= 0: 334 if match.group(0).find('Install Language Pack') >= 0:
295 match2 = re.search(r'lang="([\w\-]+)"', match.group(0)) 335 match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
296 if match2: 336 if match2:
297 locales.add(mapLocale('BCP-47', match2.group(1))) 337 locales.add(mapLocale('BCP-47', match2.group(1)))
298 338
299 allowed = set() 339 allowed = set()
300 allowedLocales = json.load(urllib2.urlopen( 340 allowedLocales = crowdin_request(projectName, 'supported-languages', key)
301 'https://crowdin.com/languages/languages_list?callback=' 341
302 ))
303 for locale in allowedLocales: 342 for locale in allowedLocales:
304 allowed.add(locale['code']) 343 allowed.add(locale['crowdin_code'])
305 if not allowed.issuperset(locales): 344 if not allowed.issuperset(locales):
306 print "Warning, following locales aren't allowed by server: " + ', '.joi n(locales - allowed) 345 print "Warning, following locales aren't allowed by server: " + ', '.joi n(locales - allowed)
307 346
308 locales = list(locales & allowed) 347 locales = list(locales & allowed)
309 locales.sort() 348 locales.sort()
310 params = urllib.urlencode([('languages[]', locale) for locale in locales]) 349 params = urllib.urlencode([('languages[]', locale) for locale in locales])
311 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/edit-project ?key=%s' % (projectName, key), params).read() 350
312 if result.find('<success') < 0: 351 crowdin_request(projectName, 'edit-project', key, post_data=params)
313 raise Exception('Server indicated that the operation was not successful\ n' + result)
314 352
315 353
316 def postFiles(files, url): 354 def crowdin_body_headers(files):
Vasily Kuznetsov 2017/09/28 14:35:00 This function name is a bit unclear, probably `cro
tlucas 2017/09/28 15:53:40 Fair enough - Done.
355 """Create a post body and according headers, which Crowdin can handle."""
Vasily Kuznetsov 2017/09/28 14:35:00 I think "matching" instead of "according" would be
tlucas 2017/09/28 15:53:40 Done.
317 boundary = '----------ThIs_Is_tHe_bouNdaRY_$' 356 boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
318 body = '' 357 body = ''
319 for file, data in files: 358 for file, data in files:
320 body += '--%s\r\n' % boundary 359 body += '--%s\r\n' % boundary
321 body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s" \r\n' % (file, file) 360 body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s" \r\n' % (file, file)
322 body += 'Content-Type: application/octet-stream\r\n' 361 body += 'Content-Type: application/octet-stream\r\n'
323 body += 'Content-Transfer-Encoding: binary\r\n' 362 body += 'Content-Transfer-Encoding: binary\r\n'
324 body += '\r\n' + data + '\r\n' 363 body += '\r\n' + data + '\r\n'
325 body += '--%s--\r\n' % boundary 364 body += '--%s--\r\n' % boundary
326 365
327 body = body.encode('utf-8') 366 body = body.encode('utf-8')
328 request = urllib2.Request(url, StringIO(body)) 367 return (
329 request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boun dary) 368 StringIO(body),
330 request.add_header('Content-Length', len(body)) 369 {
331 result = urllib2.urlopen(request).read() 370 'Content-Type': ('multipart/form-data; ; charset=utf-8; '
332 if result.find('<success') < 0: 371 'boundary=' + boundary),
333 raise Exception('Server indicated that the operation was not successful\ n' + result) 372 'Content-Length': len(body)
373 }
374 )
334 375
335 376
336 def updateTranslationMaster(localeConfig, metadata, dir, projectName, key): 377 def updateTranslationMaster(localeConfig, metadata, dir, projectName, key):
337 result = json.load(urllib2.urlopen('http://api.crowdin.net/api/project/%s/in fo?key=%s&json=1' % (projectName, key))) 378 result = crowdin_request(projectName, 'info', key)
338 379
339 existing = set(map(lambda f: f['name'], result['files'])) 380 existing = set(map(lambda f: f['name'], result['files']))
340 add = [] 381 add = []
341 update = [] 382 update = []
342 for file in os.listdir(dir): 383 for file in os.listdir(dir):
343 path = os.path.join(dir, file) 384 path = os.path.join(dir, file)
344 if os.path.isfile(path): 385 if os.path.isfile(path):
345 if localeConfig['file_format'] == 'chrome-json' and file.endswith('. json'): 386 if localeConfig['file_format'] == 'chrome-json' and file.endswith('. json'):
346 data = preprocessChromeLocale(path, metadata, True) 387 data = preprocessChromeLocale(path, metadata, True)
347 newName = file 388 newName = file
348 elif localeConfig['file_format'] == 'chrome-json': 389 elif localeConfig['file_format'] == 'chrome-json':
349 fileHandle = codecs.open(path, 'rb', encoding='utf-8') 390 fileHandle = codecs.open(path, 'rb', encoding='utf-8')
350 data = json.dumps({file: {'message': fileHandle.read()}}) 391 data = json.dumps({file: {'message': fileHandle.read()}})
351 fileHandle.close() 392 fileHandle.close()
352 newName = file + '.json' 393 newName = file + '.json'
353 else: 394 else:
354 data = toJSON(path) 395 data = toJSON(path)
355 newName = file + '.json' 396 newName = file + '.json'
356 397
357 if data: 398 if data:
358 if newName in existing: 399 if newName in existing:
359 update.append((newName, data)) 400 update.append((newName, data))
360 existing.remove(newName) 401 existing.remove(newName)
361 else: 402 else:
362 add.append((newName, data)) 403 add.append((newName, data))
363 404
364 if len(add): 405 if len(add):
365 titles = urllib.urlencode([('titles[%s]' % name, re.sub(r'\.json', '', n ame)) for name, data in add]) 406 data = {'titles[{}]'.format(name): re.sub(r'\.json', '', name)
366 postFiles(add, 'http://api.crowdin.net/api/project/%s/add-file?key=%s&ty pe=chrome&%s' % (projectName, key, titles)) 407 for name, data in add}
408 data['type'] = 'chrome'
409 data, headers = crowdin_body_headers(add)
410 crowdin_request(projectName, 'add-file', key, post_data=data,
411 headers=headers)
367 if len(update): 412 if len(update):
368 postFiles(update, 'http://api.crowdin.net/api/project/%s/update-file?key =%s' % (projectName, key)) 413 data, headers = crowdin_body_headers(update)
414 crowdin_request(projectName, 'update-file', key, post_data=data,
415 headers=headers)
369 for file in existing: 416 for file in existing:
370 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/delete-f ile?key=%s&file=%s' % (projectName, key, file)).read() 417 crowdin_request(projectName, 'delete-file', key, {'file': file})
371 if result.find('<success') < 0:
372 raise Exception('Server indicated that the operation was not success ful\n' + result)
373 418
374 419
375 def uploadTranslations(localeConfig, metadata, dir, locale, projectName, key): 420 def uploadTranslations(localeConfig, metadata, dir, locale, projectName, key):
376 files = [] 421 files = []
377 for file in os.listdir(dir): 422 for file in os.listdir(dir):
378 path = os.path.join(dir, file) 423 path = os.path.join(dir, file)
379 if os.path.isfile(path): 424 if os.path.isfile(path):
380 if localeConfig['file_format'] == 'chrome-json' and file.endswith('. json'): 425 if localeConfig['file_format'] == 'chrome-json' and file.endswith('. json'):
381 data = preprocessChromeLocale(path, metadata, False) 426 data = preprocessChromeLocale(path, metadata, False)
382 newName = file 427 newName = file
383 elif localeConfig['file_format'] == 'chrome-json': 428 elif localeConfig['file_format'] == 'chrome-json':
384 fileHandle = codecs.open(path, 'rb', encoding='utf-8') 429 fileHandle = codecs.open(path, 'rb', encoding='utf-8')
385 data = json.dumps({file: {'message': fileHandle.read()}}) 430 data = json.dumps({file: {'message': fileHandle.read()}})
386 fileHandle.close() 431 fileHandle.close()
387 newName = file + '.json' 432 newName = file + '.json'
388 else: 433 else:
389 data = toJSON(path) 434 data = toJSON(path)
390 newName = file + '.json' 435 newName = file + '.json'
391 436
392 if data: 437 if data:
393 files.append((newName, data)) 438 files.append((newName, data))
394 if len(files): 439 if len(files):
395 postFiles(files, 'http://api.crowdin.net/api/project/%s/upload-translati on?key=%s&language=%s' % ( 440 language = mapLocale(localeConfig['name_format'], locale)
396 projectName, key, mapLocale(localeConfig['name_format'], locale)) 441 data, headers = crowdin_body_headers(files)
397 ) 442 crowdin_request(projectName, 'upload-translation', key,
443 {'language': language}, post_data=data,
444 headers=headers)
398 445
399 446
400 def getTranslations(localeConfig, projectName, key): 447 def getTranslations(localeConfig, projectName, key):
401 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=% s' % (projectName, key)).read() 448 """Download all available translations from crowdin.
402 if result.find('<success') < 0:
403 raise Exception('Server indicated that the operation was not successful\ n' + result)
404 449
405 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all .zip?key=%s' % (projectName, key)).read() 450 Trigger crowdin to build the available export, wait for crowdin to
451 finish the job and download the generated zip afterwards.
452 """
453 crowdin_request(projectName, 'export', key)
454
455 result = crowdin_request(projectName, 'download/all.zip', key, raw=True)
406 zip = ZipFile(StringIO(result)) 456 zip = ZipFile(StringIO(result))
407 dirs = {} 457 dirs = {}
408 458
409 normalizedDefaultLocale = localeConfig['default_locale'] 459 normalizedDefaultLocale = localeConfig['default_locale']
410 if localeConfig['name_format'] == 'ISO-15897': 460 if localeConfig['name_format'] == 'ISO-15897':
411 normalizedDefaultLocale = normalizedDefaultLocale.replace('_', '-') 461 normalizedDefaultLocale = normalizedDefaultLocale.replace('_', '-')
412 normalizedDefaultLocale = mapLocale(localeConfig['name_format'], 462 normalizedDefaultLocale = mapLocale(localeConfig['name_format'],
413 normalizedDefaultLocale) 463 normalizedDefaultLocale)
414 464
415 for info in zip.infolist(): 465 for info in zip.infolist():
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after
463 513
464 # Remove any extra files 514 # Remove any extra files
465 for dir, files in dirs.iteritems(): 515 for dir, files in dirs.iteritems():
466 baseDir = os.path.join(localeConfig['base_path'], dir) 516 baseDir = os.path.join(localeConfig['base_path'], dir)
467 if not os.path.exists(baseDir): 517 if not os.path.exists(baseDir):
468 continue 518 continue
469 for file in os.listdir(baseDir): 519 for file in os.listdir(baseDir):
470 path = os.path.join(baseDir, file) 520 path = os.path.join(baseDir, file)
471 if os.path.isfile(path) and (file.endswith('.json') or file.endswith ('.properties') or file.endswith('.dtd')) and not file in files: 521 if os.path.isfile(path) and (file.endswith('.json') or file.endswith ('.properties') or file.endswith('.dtd')) and not file in files:
472 os.remove(path) 522 os.remove(path)
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld