Issue 5763 - Target languages supported by Firefox

localeTools.py

Index: localeTools.py
===================================================================
--- a/localeTools.py
+++ b/localeTools.py
@@ -16,86 +16,27 @@
 from zipfile import ZipFile
 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
 
-langMappingGecko = {
-    'bn-BD': 'bn',
+CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
+
+CROWDIN_LANG_MAPPING = {
     'br': 'br-FR',
     'dsb': 'dsb-DE',
-    'fj-FJ': 'fj',
+    'es': 'es-ES',
+    'fur': 'fur-IT',
+    'fy': 'fy-NL',
+    'ga': 'ga-IE',
+    'gu': 'gu-IN',
     'hsb': 'hsb-DE',
-    'hi-IN': 'hi',
+    'hy': 'hy-AM',
     'ml': 'ml-IN',
-    'nb-NO': 'nb',
+    'nn': 'nn-NO',
+    'pa': 'pa-IN',
     'rm': 'rm-CH',
-    'ta-LK': 'ta',
-    'wo-SN': 'wo',
-}
-
-langMappingChrome = {
-    'es-419': 'es-MX',
-    'es': 'es-ES',
+    'si': 'si-LK',
     'sv': 'sv-SE',
-    'ml': 'ml-IN',
-    'gu': 'gu-IN',
+    'ur': 'ur-PK',
 }
 
-chromeLocales = [
-    'am',
-    'ar',
-    'bg',
-    'bn',
-    'ca',
-    'cs',
-    'da',
-    'de',
-    'el',
-    'en-GB',
-    'en-US',
-    'es-419',
-    'es',
-    'et',
-    'fa',
-    'fi',
-    'fil',
-    'fr',
-    'gu',
-    'he',
-    'hi',
-    'hr',
-    'hu',
-    'id',
-    'it',
-    'ja',
-    'kn',
-    'ko',
-    'lt',
-    'lv',
-    'ml',
-    'mr',
-    'ms',
-    'nb',
-    'nl',
-    'pl',
-    'pt-BR',
-    'pt-PT',
-    'ro',
-    'ru',
-    'sk',
-    'sl',
-    'sr',
-    'sv',
-    'sw',
-    'ta',
-    'te',
-    'th',
-    'tr',
-    'uk',
-    'vi',
-    'zh-CN',
-    'zh-TW',
-]
-
-CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
-
 
 def crowdin_request(project_name, action, key, get={}, post_data=None,
                     headers={}, raw=False):
@@ -145,11 +86,6 @@
     return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"')
 
 
-def mapLocale(type, locale):
-    mapping = langMappingChrome if type == 'ISO-15897' else langMappingGecko
-    return mapping.get(locale, locale)
-
-
 def parseDTDString(data, path):
     result = []
     currentComment = [None]
@@ -304,38 +240,49 @@
 
 
 def setupTranslations(localeConfig, projectName, key):
-    # Make a new set from the locales list, mapping to Crowdin friendly format
-    locales = {mapLocale(localeConfig['name_format'], locale)
-               for locale in localeConfig['locales']}
-
-    # Fill up with locales that we don't have but the browser supports
-    if 'chrome' in localeConfig['target_platforms']:
-        for locale in chromeLocales:
-            locales.add(mapLocale('ISO-15897', locale))
+    locales = set()
 
-    if 'gecko' in localeConfig['target_platforms']:
-        firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
-        for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
-            locales.add(mapLocale('BCP-47', match.group(1)))
-        langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
-        for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
-            if match.group(0).find('Install Language Pack') >= 0:
-                match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
-                if match2:
-                    locales.add(mapLocale('BCP-47', match2.group(1)))
+    # Languages supported by Firefox
+    data = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()

[Vasily Kuznetsov, 2017/10/05 20:26:03]

What do you think about moving all these long URLs into constants at the
beginning of the file? This way we'd get rid of the hardcoded strings and of the
lines that are too long in this function at the same time.

[Sebastian Noack, 2017/10/05 20:58:38]

Done.

+    for match in re.finditer(r'&lang=([\w\-]+)"', data):
+        locales.add(match.group(1))
 
-    allowed = set()
-    allowedLocales = crowdin_request(projectName, 'supported-languages', key)
+    # Languages supported by Firefox Language Packs
+    data = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
+    for match in re.finditer(r'<tr>.*?</tr>', data, re.S):
+        if match.group(0).find('Install Language Pack') >= 0:
+            match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
+            if match2:
+                locales.add(match2.group(1))
 
-    for locale in allowedLocales:
-        allowed.add(locale['crowdin_code'])
+    # Languages supported by Chrome (excluding es-419)
+    data = urllib2.urlopen('https://packages.debian.org/sid/all/chromium-l10n/filelist').read()
+    for match in re.finditer(r'locales/(?!es-419)([\w\-]+)\.pak', data):
+        locales.add(match.group(1))
+
+    # We don't translate indvidual dialects of languages
+    # other than English, Spanish, Portuguese and Chinese.
+    for locale in list(locales):
+        prefix = locale.split('-')[0]
+        if prefix not in {'en', 'es', 'pt', 'zh'}:
+            locales.remove(locale)
+            locales.add(prefix)
+
+    # Add languages with existing translations.
+    locales.update(localeConfig['locales'])
+
+    # Don't add the language we translate from as target translation.
+    locales.remove(localeConfig['default_locale'].replace('_', '-'))
+
+    # Convert to locales understood by Crowdin.
+    locales = {CROWDIN_LANG_MAPPING.get(locale, locale) for locale in locales}
+    allowed = {locale['crowdin_code'] for locale in
+               crowdin_request(projectName, 'supported-languages', key)}
     if not allowed.issuperset(locales):
         print "Warning, following locales aren't allowed by server: " + ', '.join(locales - allowed)
 
-    locales = list(locales & allowed)
-    locales.sort()
+    locales = sorted(locales & allowed)
     params = urllib.urlencode([('languages[]', locale) for locale in locales])
-
     crowdin_request(projectName, 'edit-project', key, post_data=params)
 
 
@@ -429,7 +376,7 @@
             if data:
                 files.append((newName, data))
     if len(files):
-        language = mapLocale(localeConfig['name_format'], locale)
+        language = CROWDIN_LANG_MAPPING.get(locale, locale)
         data, headers = crowdin_prepare_upload(files)
         crowdin_request(projectName, 'upload-translation', key,
                         {'language': language}, post_data=data,
@@ -451,8 +398,8 @@
     normalizedDefaultLocale = localeConfig['default_locale']
     if localeConfig['name_format'] == 'ISO-15897':
         normalizedDefaultLocale = normalizedDefaultLocale.replace('_', '-')
-    normalizedDefaultLocale = mapLocale(localeConfig['name_format'],
-                                        normalizedDefaultLocale)
+    normalizedDefaultLocale = CROWDIN_LANG_MAPPING.get(normalizedDefaultLocale,
+                                                       normalizedDefaultLocale)
 
     for info in zip.infolist():
         if not info.filename.endswith('.json'):
@@ -470,12 +417,7 @@
             not origFile.endswith('.properties')):
             continue
 
-        if localeConfig['name_format'] == 'ISO-15897':
-            mapping = langMappingChrome
-        else:
-            mapping = langMappingGecko
-
-        for key, value in mapping.iteritems():
+        for key, value in CROWDIN_LANG_MAPPING.iteritems():
             if value == dir:
                 dir = key
         if localeConfig['name_format'] == 'ISO-15897':