Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: localeTools.py

Issue 29561557: Issue 5763 - Target languages supported by Firefox (Closed)
Left Patch Set: Removed unused target_platforms key Created Oct. 4, 2017, 1:37 a.m.
Right Patch Set: Fixed undefined variable, put URLS in globals, removed redundand flake8 ignores Created Oct. 5, 2017, 8:54 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « build.py ('k') | packagerChrome.py » ('j') | packagerChrome.py » ('J')
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # This Source Code Form is subject to the terms of the Mozilla Public 1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this 2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
5 import re 5 import re
6 import os 6 import os
7 import sys 7 import sys
8 import codecs 8 import codecs
9 import json 9 import json
10 import urlparse 10 import urlparse
11 import urllib 11 import urllib
12 import urllib2 12 import urllib2
13 import mimetypes 13 import mimetypes
14 from StringIO import StringIO 14 from StringIO import StringIO
15 from ConfigParser import SafeConfigParser 15 from ConfigParser import SafeConfigParser
16 from zipfile import ZipFile 16 from zipfile import ZipFile
17 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS 17 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
18
19 CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
20 18
21 CROWDIN_LANG_MAPPING = { 19 CROWDIN_LANG_MAPPING = {
22 'br': 'br-FR', 20 'br': 'br-FR',
23 'dsb': 'dsb-DE', 21 'dsb': 'dsb-DE',
24 'es': 'es-ES', 22 'es': 'es-ES',
25 'fur': 'fur-IT', 23 'fur': 'fur-IT',
26 'fy': 'fy-NL', 24 'fy': 'fy-NL',
27 'ga': 'ga-IE', 25 'ga': 'ga-IE',
28 'gu': 'gu-IN', 26 'gu': 'gu-IN',
29 'hsb': 'hsb-DE', 27 'hsb': 'hsb-DE',
30 'hy': 'hy-AM', 28 'hy': 'hy-AM',
31 'ml': 'ml-IN', 29 'ml': 'ml-IN',
32 'nn': 'nn-NO', 30 'nn': 'nn-NO',
33 'pa': 'pa-IN', 31 'pa': 'pa-IN',
34 'rm': 'rm-CH', 32 'rm': 'rm-CH',
35 'si': 'si-LK', 33 'si': 'si-LK',
36 'sv': 'sv-SE', 34 'sv': 'sv-SE',
37 'ur': 'ur-PK', 35 'ur': 'ur-PK',
38 } 36 }
39 37
38 CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
39 FIREFOX_RELEASES_URL = 'http://www.mozilla.org/en-US/firefox/all.html'
40 FIREFOX_LP_URL = 'https://addons.mozilla.org/en-US/firefox/language-tools/'
41 CHROMIUM_DEB_URL = 'https://packages.debian.org/sid/all/chromium-l10n/filelist'
42
40 43
41 def crowdin_request(project_name, action, key, get={}, post_data=None, 44 def crowdin_request(project_name, action, key, get={}, post_data=None,
42 headers={}, raw=False): 45 headers={}, raw=False):
43 """Perform a call to crowdin and raise an Exception on failure.""" 46 """Perform a call to crowdin and raise an Exception on failure."""
44 request = urllib2.Request( 47 request = urllib2.Request(
45 '{}/{}/{}?{}'.format(CROWDIN_AP_URL, 48 '{}/{}/{}?{}'.format(CROWDIN_AP_URL,
46 urllib.quote(project_name), 49 urllib.quote(project_name),
47 urllib.quote(action), 50 urllib.quote(action),
48 urllib.urlencode(dict(get, key=key, json=1))), 51 urllib.urlencode(dict(get, key=key, json=1))),
49 post_data, 52 post_data,
(...skipping 186 matching lines...) Expand 10 before | Expand all | Expand 10 after
236 239
237 file = codecs.open(path, 'wb', encoding='utf-8') 240 file = codecs.open(path, 'wb', encoding='utf-8')
238 json.dump(parsed, file, ensure_ascii=False, sort_keys=True, indent=2, separa tors=(',', ': ')) 241 json.dump(parsed, file, ensure_ascii=False, sort_keys=True, indent=2, separa tors=(',', ': '))
239 file.close() 242 file.close()
240 243
241 244
242 def setupTranslations(localeConfig, projectName, key): 245 def setupTranslations(localeConfig, projectName, key):
243 locales = set() 246 locales = set()
244 247
245 # Languages supported by Firefox 248 # Languages supported by Firefox
246 data = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read () 249 data = urllib2.urlopen(FIREFOX_RELEASES_URL).read()
247 for match in re.finditer(r'&lang=([\w\-]+)"', data): 250 for match in re.finditer(r'&lang=([\w\-]+)"', data):
248 locales.add(match.group(1)) 251 locales.add(match.group(1))
249 252
250 # Languages supported by Firefox Language Packs 253 # Languages supported by Firefox Language Packs
251 data = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-to ols/').read() 254 data = urllib2.urlopen(FIREFOX_LP_URL).read()
252 for match in re.finditer(r'<tr>.*?</tr>', data, re.S): 255 for match in re.finditer(r'<tr>.*?</tr>', data, re.S):
253 if match.group(0).find('Install Language Pack') >= 0: 256 if match.group(0).find('Install Language Pack') >= 0:
254 match2 = re.search(r'lang="([\w\-]+)"', match.group(0)) 257 match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
255 if match2: 258 if match2:
256 locales.add(match2.group(1)) 259 locales.add(match2.group(1))
257 260
258 # Languages supported by Chrome (excluding es-419) 261 # Languages supported by Chrome (excluding es-419)
259 data = urllib2.urlopen('https://packages.debian.org/sid/all/chromium-l10n/fi lelist').read() 262 data = urllib2.urlopen(CHROMIUM_DEB_URL).read()
260 for match in re.finditer(r'locales/(?!es-419)([\w\-]+)\.pak', data): 263 for match in re.finditer(r'locales/(?!es-419)([\w\-]+)\.pak', data):
261 locales.add(match.group(1)) 264 locales.add(match.group(1))
262 265
263 # We don't translate indvidual dialects of languages 266 # We don't translate indvidual dialects of languages
264 # other than English, Spanish, Portuguese and Chinese. 267 # other than English, Spanish, Portuguese and Chinese.
265 for locale in list(locales): 268 for locale in list(locales):
266 prefix = locale.split('-')[0] 269 prefix = locale.split('-')[0]
267 if prefix not in {'en', 'es', 'pt', 'zh'}: 270 if prefix not in {'en', 'es', 'pt', 'zh'}:
268 locales.remove(locale) 271 locales.remove(locale)
269 locales.add(prefix) 272 locales.add(prefix)
270 273
271 # Add languages with existing translations. 274 # Add languages with existing translations.
tlucas 2017/10/04 09:37:39 flake8 fails on this line: W291 trailing whitespac
Sebastian Noack 2017/10/04 21:33:50 Done.
272 locales.update(localeConfig['locales']) 275 locales.update(localeConfig['locales'])
273 276
274 # Don't add the language we translate from as target translation. 277 # Don't add the language we translate from as target translation.
275 locales.remove(localeConfig['default_locale'].replace('_', '-')) 278 locales.remove(localeConfig['default_locale'].replace('_', '-'))
276 279
277 # Convert to locales understood by Crowdin. 280 # Convert to locales understood by Crowdin.
278 locales = {CROWDIN_LANG_MAPPING.get(locale, locale) for locale in locales} 281 locales = {CROWDIN_LANG_MAPPING.get(locale, locale) for locale in locales}
279 allowed = {locale['crowdin_code'] for locale in 282 allowed = {locale['crowdin_code'] for locale in
280 crowdin_request(projectName, 'supported-languages', key)} 283 crowdin_request(projectName, 'supported-languages', key)}
281 if not allowed.issuperset(locales): 284 if not allowed.issuperset(locales):
(...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after
447 450
448 # Remove any extra files 451 # Remove any extra files
449 for dir, files in dirs.iteritems(): 452 for dir, files in dirs.iteritems():
450 baseDir = os.path.join(localeConfig['base_path'], dir) 453 baseDir = os.path.join(localeConfig['base_path'], dir)
451 if not os.path.exists(baseDir): 454 if not os.path.exists(baseDir):
452 continue 455 continue
453 for file in os.listdir(baseDir): 456 for file in os.listdir(baseDir):
454 path = os.path.join(baseDir, file) 457 path = os.path.join(baseDir, file)
455 if os.path.isfile(path) and (file.endswith('.json') or file.endswith ('.properties') or file.endswith('.dtd')) and not file in files: 458 if os.path.isfile(path) and (file.endswith('.json') or file.endswith ('.properties') or file.endswith('.dtd')) and not file in files:
456 os.remove(path) 459 os.remove(path)
LEFTRIGHT

Powered by Google App Engine
This is Rietveld