Issue 5763 - Target languages supported by Firefox

localeTools.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
 import os
 import sys
 import codecs
 import json
 import urlparse
 import urllib
 import urllib2
 import mimetypes
 from StringIO import StringIO
 from ConfigParser import SafeConfigParser
 from zipfile import ZipFile
 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
 
-langMappingGecko = {
-    'bn-BD': 'bn',
-    'br': 'br-FR',
-    'dsb': 'dsb-DE',
-    'fj-FJ': 'fj',
-    'hsb': 'hsb-DE',
-    'hi-IN': 'hi',
-    'ml': 'ml-IN',
-    'nb-NO': 'nb',
-    'rm': 'rm-CH',
-    'ta-LK': 'ta',
-    'wo-SN': 'wo',
-}
-
-langMappingChrome = {
+CROWDIN_LANG_MAPPING = {
     'br': 'br-FR',
     'dsb': 'dsb-DE',
     'es': 'es-ES',
     'fur': 'fur-IT',
     'fy': 'fy-NL',
     'ga': 'ga-IE',
     'gu': 'gu-IN',
     'hsb': 'hsb-DE',
     'hy': 'hy-AM',
     'ml': 'ml-IN',
     'nn': 'nn-NO',
     'pa': 'pa-IN',
     'rm': 'rm-CH',
     'si': 'si-LK',
     'sv': 'sv-SE',
     'ur': 'ur-PK',
 }
 
-chromeLocales = [
-    'am',
-    'ar',
-    'bg',
-    'bn',
-    'ca',
-    'cs',
-    'da',
-    'de',
-    'el',
-    'en-GB',
-    'en-US',
-    'es-419',

[tlucas, 2017/10/02 09:25:43]

iirc, removing the mapping for this will trigger a warning when someone would
run "setuptrans" - do we ignore this?

[Sebastian Noack, 2017/10/02 23:01:33]

The behavior of setuptrans is outdated. I initially planned to address this
separately but here we go.

1. The hard-coded list of chrome locales is outdated. Unfortunately, there is no
official source we can get these data from on-the-fly, but there is a quite
reliable third-party source, i.e. Debian. ;)
2. The distinction between Gecko and Chrome locales is outdated. We no longer
have Chrome-only or Gecko-only extensions, so all translation projects need to
be translated into all languages supported by any of the two.
3. There have been some further inconsistencies where Firefox supports different
dialects of a language, but we only provide one translation (e.g. Firefox has
"bn-BD" and "bn-IN", but we only translate "bn").

I did changes to setupTranslations() to account for all of these, and I
confirmed that the languages it would add to a project are tha same languages we
currently have in our translation projects.

It still gives one warning though, about "cak" (Kaqchikel), since Crowdin
doesn't support this language at the moment, but Firefox does. However, note
that this warning isn't new.

-    'es',
-    'et',
-    'fa',
-    'fi',
-    'fil',
-    'fr',
-    'gu',
-    'he',
-    'hi',
-    'hr',
-    'hu',
-    'id',
-    'it',
-    'ja',
-    'kn',
-    'ko',
-    'lt',
-    'lv',
-    'ml',
-    'mr',
-    'ms',
-    'nb',
-    'nl',
-    'pl',
-    'pt-BR',
-    'pt-PT',
-    'ro',
-    'ru',
-    'sk',
-    'sl',
-    'sr',
-    'sv',
-    'sw',
-    'ta',
-    'te',
-    'th',
-    'tr',
-    'uk',
-    'vi',
-    'zh-CN',
-    'zh-TW',
-]
-
 CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
+FIREFOX_RELEASES_URL = 'http://www.mozilla.org/en-US/firefox/all.html'
+FIREFOX_LP_URL = 'https://addons.mozilla.org/en-US/firefox/language-tools/'
+CHROMIUM_DEB_URL = 'https://packages.debian.org/sid/all/chromium-l10n/filelist'
 
 
 def crowdin_request(project_name, action, key, get={}, post_data=None,
                     headers={}, raw=False):
     """Perform a call to crowdin and raise an Exception on failure."""
     request = urllib2.Request(
         '{}/{}/{}?{}'.format(CROWDIN_AP_URL,
                              urllib.quote(project_name),
                              urllib.quote(action),
                              urllib.urlencode(dict(get, key=key, json=1))),
@@ skipping 28 matching lines @@
                 yield (key, self[key])
                 done.add(key)
 
 
 def escapeEntity(value):
     return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
 
 
 def unescapeEntity(value):
     return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"')
-
-
-def mapLocale(type, locale):
-    mapping = langMappingChrome if type == 'ISO-15897' else langMappingGecko
-    return mapping.get(locale, locale)
 
 
 def parseDTDString(data, path):
     result = []
     currentComment = [None]
 
     parser = ParserCreate()
     parser.UseForeignDTD(True)
     parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS)
 
@@ skipping 136 matching lines @@
     for key, value in parsed.iteritems():
         if 'description' in value:
             del value['description']
 
     file = codecs.open(path, 'wb', encoding='utf-8')
     json.dump(parsed, file, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))
     file.close()
 
 
 def setupTranslations(localeConfig, projectName, key):
-    # Make a new set from the locales list, mapping to Crowdin friendly format
-    locales = {mapLocale(localeConfig['name_format'], locale)
-               for locale in localeConfig['locales']}
-
-    # Fill up with locales that we don't have but the browser supports
-    if 'chrome' in localeConfig['target_platforms']:
-        for locale in chromeLocales:
-            locales.add(mapLocale('ISO-15897', locale))
-
-    if 'gecko' in localeConfig['target_platforms']:
-        firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
-        for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
-            locales.add(mapLocale('BCP-47', match.group(1)))
-        langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
-        for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
-            if match.group(0).find('Install Language Pack') >= 0:
-                match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
-                if match2:
-                    locales.add(mapLocale('BCP-47', match2.group(1)))
-
-    allowed = set()
-    allowedLocales = crowdin_request(projectName, 'supported-languages', key)
-
-    for locale in allowedLocales:
-        allowed.add(locale['crowdin_code'])
+    locales = set()
+
+    # Languages supported by Firefox
+    data = urllib2.urlopen(FIREFOX_RELEASES_URL).read()
+    for match in re.finditer(r'&amp;lang=([\w\-]+)"', data):
+        locales.add(match.group(1))
+
+    # Languages supported by Firefox Language Packs
+    data = urllib2.urlopen(FIREFOX_LP_URL).read()
+    for match in re.finditer(r'<tr>.*?</tr>', data, re.S):
+        if match.group(0).find('Install Language Pack') >= 0:
+            match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
+            if match2:
+                locales.add(match2.group(1))
+
+    # Languages supported by Chrome (excluding es-419)
+    data = urllib2.urlopen(CHROMIUM_DEB_URL).read()
+    for match in re.finditer(r'locales/(?!es-419)([\w\-]+)\.pak', data):
+        locales.add(match.group(1))
+
+    # We don't translate indvidual dialects of languages
+    # other than English, Spanish, Portuguese and Chinese.
+    for locale in list(locales):
+        prefix = locale.split('-')[0]
+        if prefix not in {'en', 'es', 'pt', 'zh'}:
+            locales.remove(locale)
+            locales.add(prefix)
+
+    # Add languages with existing translations.
+    locales.update(localeConfig['locales'])
+
+    # Don't add the language we translate from as target translation.
+    locales.remove(localeConfig['default_locale'].replace('_', '-'))
+
+    # Convert to locales understood by Crowdin.
+    locales = {CROWDIN_LANG_MAPPING.get(locale, locale) for locale in locales}
+    allowed = {locale['crowdin_code'] for locale in
+               crowdin_request(projectName, 'supported-languages', key)}
     if not allowed.issuperset(locales):
         print "Warning, following locales aren't allowed by server: " + ', '.join(locales - allowed)
 
-    locales = list(locales & allowed)
-    locales.sort()
+    locales = sorted(locales & allowed)
     params = urllib.urlencode([('languages[]', locale) for locale in locales])
-
     crowdin_request(projectName, 'edit-project', key, post_data=params)
 
 
 def crowdin_prepare_upload(files):
     """Create a post body and matching headers, which Crowdin can handle."""
     boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
     body = ''
     for name, data in files:
         body += (
             '--{boundary}\r\n'
@@ skipping 73 matching lines @@
                 data = json.dumps({file: {'message': fileHandle.read()}})
                 fileHandle.close()
                 newName = file + '.json'
             else:
                 data = toJSON(path)
                 newName = file + '.json'
 
             if data:
                 files.append((newName, data))
     if len(files):
-        language = mapLocale(localeConfig['name_format'], locale)
+        language = CROWDIN_LANG_MAPPING.get(locale, locale)
         data, headers = crowdin_prepare_upload(files)
         crowdin_request(projectName, 'upload-translation', key,
                         {'language': language}, post_data=data,
                         headers=headers)
 
 
 def getTranslations(localeConfig, projectName, key):
     """Download all available translations from crowdin.
 
     Trigger crowdin to build the available export, wait for crowdin to
     finish the job and download the generated zip afterwards.
     """
     crowdin_request(projectName, 'export', key)
 
     result = crowdin_request(projectName, 'download/all.zip', key, raw=True)
     zip = ZipFile(StringIO(result))
     dirs = {}
 
     normalizedDefaultLocale = localeConfig['default_locale']
     if localeConfig['name_format'] == 'ISO-15897':
         normalizedDefaultLocale = normalizedDefaultLocale.replace('_', '-')
-    normalizedDefaultLocale = mapLocale(localeConfig['name_format'],
-                                        normalizedDefaultLocale)
+    normalizedDefaultLocale = CROWDIN_LANG_MAPPING.get(normalizedDefaultLocale,
+                                                       normalizedDefaultLocale)
 
     for info in zip.infolist():
         if not info.filename.endswith('.json'):
             continue
 
         dir, file = os.path.split(info.filename)
         if not re.match(r'^[\w\-]+$', dir) or dir == normalizedDefaultLocale:
             continue
         if localeConfig['file_format'] == 'chrome-json' and file.count('.') == 1:
             origFile = file
         else:
             origFile = re.sub(r'\.json$', '', file)
         if (localeConfig['file_format'] == 'gecko-dtd' and
             not origFile.endswith('.dtd') and
             not origFile.endswith('.properties')):
             continue
 
-        if localeConfig['name_format'] == 'ISO-15897':
-            mapping = langMappingChrome
-        else:
-            mapping = langMappingGecko
-
-        for key, value in mapping.iteritems():
+        for key, value in CROWDIN_LANG_MAPPING.iteritems():
             if value == dir:
                 dir = key
         if localeConfig['name_format'] == 'ISO-15897':
             dir = dir.replace('-', '_')
 
         data = zip.open(info.filename).read()
         if data == '[]':
             continue
 
         if not dir in dirs:
@@ skipping 16 matching lines @@
 
     # Remove any extra files
     for dir, files in dirs.iteritems():
         baseDir = os.path.join(localeConfig['base_path'], dir)
         if not os.path.exists(baseDir):
             continue
         for file in os.listdir(baseDir):
             path = os.path.join(baseDir, file)
             if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
                 os.remove(path)