Index: localeTools.py
===================================================================
--- a/localeTools.py
+++ b/localeTools.py
@@ -16,85 +16,29 @@
from zipfile import ZipFile
from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
-langMappingGecko = {
- 'bn-BD': 'bn',
+CROWDIN_LANG_MAPPING = {
'br': 'br-FR',
'dsb': 'dsb-DE',
- 'fj-FJ': 'fj',
+ 'es': 'es-ES',
+ 'fur': 'fur-IT',
+ 'fy': 'fy-NL',
+ 'ga': 'ga-IE',
+ 'gu': 'gu-IN',
'hsb': 'hsb-DE',
- 'hi-IN': 'hi',
+ 'hy': 'hy-AM',
'ml': 'ml-IN',
- 'nb-NO': 'nb',
+ 'nn': 'nn-NO',
+ 'pa': 'pa-IN',
'rm': 'rm-CH',
- 'ta-LK': 'ta',
- 'wo-SN': 'wo',
-}
-
-langMappingChrome = {
- 'es-419': 'es-MX',
- 'es': 'es-ES',
+ 'si': 'si-LK',
'sv': 'sv-SE',
- 'ml': 'ml-IN',
- 'gu': 'gu-IN',
+ 'ur': 'ur-PK',
}
-chromeLocales = [
- 'am',
- 'ar',
- 'bg',
- 'bn',
- 'ca',
- 'cs',
- 'da',
- 'de',
- 'el',
- 'en-GB',
- 'en-US',
- 'es-419',
- 'es',
- 'et',
- 'fa',
- 'fi',
- 'fil',
- 'fr',
- 'gu',
- 'he',
- 'hi',
- 'hr',
- 'hu',
- 'id',
- 'it',
- 'ja',
- 'kn',
- 'ko',
- 'lt',
- 'lv',
- 'ml',
- 'mr',
- 'ms',
- 'nb',
- 'nl',
- 'pl',
- 'pt-BR',
- 'pt-PT',
- 'ro',
- 'ru',
- 'sk',
- 'sl',
- 'sr',
- 'sv',
- 'sw',
- 'ta',
- 'te',
- 'th',
- 'tr',
- 'uk',
- 'vi',
- 'zh-CN',
- 'zh-TW',
-]
-
CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
+FIREFOX_RELEASES_URL = 'http://www.mozilla.org/en-US/firefox/all.html'
+FIREFOX_LP_URL = 'https://addons.mozilla.org/en-US/firefox/language-tools/'
+CHROMIUM_DEB_URL = 'https://packages.debian.org/sid/all/chromium-l10n/filelist'
def crowdin_request(project_name, action, key, get={}, post_data=None,
@@ -145,11 +89,6 @@
return value.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"')
-def mapLocale(type, locale):
- mapping = langMappingChrome if type == 'ISO-15897' else langMappingGecko
- return mapping.get(locale, locale)
-
-
def parseDTDString(data, path):
result = []
currentComment = [None]
@@ -304,38 +243,49 @@
def setupTranslations(localeConfig, projectName, key):
- # Make a new set from the locales list, mapping to Crowdin friendly format
- locales = {mapLocale(localeConfig['name_format'], locale)
- for locale in localeConfig['locales']}
-
- # Fill up with locales that we don't have but the browser supports
- if 'chrome' in localeConfig['target_platforms']:
- for locale in chromeLocales:
- locales.add(mapLocale('ISO-15897', locale))
+ locales = set()
- if 'gecko' in localeConfig['target_platforms']:
- firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
- for match in re.finditer(r'&lang=([\w\-]+)"', firefoxLocales):
- locales.add(mapLocale('BCP-47', match.group(1)))
- langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
- for match in re.finditer(r'.*?
', langPacks, re.S):
- if match.group(0).find('Install Language Pack') >= 0:
- match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
- if match2:
- locales.add(mapLocale('BCP-47', match2.group(1)))
+ # Languages supported by Firefox
+ data = urllib2.urlopen(FIREFOX_RELEASES_URL).read()
+ for match in re.finditer(r'&lang=([\w\-]+)"', data):
+ locales.add(match.group(1))
- allowed = set()
- allowedLocales = crowdin_request(projectName, 'supported-languages', key)
+ # Languages supported by Firefox Language Packs
+ data = urllib2.urlopen(FIREFOX_LP_URL).read()
+ for match in re.finditer(r'.*?
', data, re.S):
+ if match.group(0).find('Install Language Pack') >= 0:
+ match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
+ if match2:
+ locales.add(match2.group(1))
- for locale in allowedLocales:
- allowed.add(locale['crowdin_code'])
+ # Languages supported by Chrome (excluding es-419)
+ data = urllib2.urlopen(CHROMIUM_DEB_URL).read()
+ for match in re.finditer(r'locales/(?!es-419)([\w\-]+)\.pak', data):
+ locales.add(match.group(1))
+
+ # We don't translate indvidual dialects of languages
+ # other than English, Spanish, Portuguese and Chinese.
+ for locale in list(locales):
+ prefix = locale.split('-')[0]
+ if prefix not in {'en', 'es', 'pt', 'zh'}:
+ locales.remove(locale)
+ locales.add(prefix)
+
+ # Add languages with existing translations.
+ locales.update(localeConfig['locales'])
+
+ # Don't add the language we translate from as target translation.
+ locales.remove(localeConfig['default_locale'].replace('_', '-'))
+
+ # Convert to locales understood by Crowdin.
+ locales = {CROWDIN_LANG_MAPPING.get(locale, locale) for locale in locales}
+ allowed = {locale['crowdin_code'] for locale in
+ crowdin_request(projectName, 'supported-languages', key)}
if not allowed.issuperset(locales):
print "Warning, following locales aren't allowed by server: " + ', '.join(locales - allowed)
- locales = list(locales & allowed)
- locales.sort()
+ locales = sorted(locales & allowed)
params = urllib.urlencode([('languages[]', locale) for locale in locales])
-
crowdin_request(projectName, 'edit-project', key, post_data=params)
@@ -429,7 +379,7 @@
if data:
files.append((newName, data))
if len(files):
- language = mapLocale(localeConfig['name_format'], locale)
+ language = CROWDIN_LANG_MAPPING.get(locale, locale)
data, headers = crowdin_prepare_upload(files)
crowdin_request(projectName, 'upload-translation', key,
{'language': language}, post_data=data,
@@ -451,8 +401,8 @@
normalizedDefaultLocale = localeConfig['default_locale']
if localeConfig['name_format'] == 'ISO-15897':
normalizedDefaultLocale = normalizedDefaultLocale.replace('_', '-')
- normalizedDefaultLocale = mapLocale(localeConfig['name_format'],
- normalizedDefaultLocale)
+ normalizedDefaultLocale = CROWDIN_LANG_MAPPING.get(normalizedDefaultLocale,
+ normalizedDefaultLocale)
for info in zip.infolist():
if not info.filename.endswith('.json'):
@@ -470,12 +420,7 @@
not origFile.endswith('.properties')):
continue
- if localeConfig['name_format'] == 'ISO-15897':
- mapping = langMappingChrome
- else:
- mapping = langMappingGecko
-
- for key, value in mapping.iteritems():
+ for key, value in CROWDIN_LANG_MAPPING.iteritems():
if value == dir:
dir = key
if localeConfig['name_format'] == 'ISO-15897':