Issue 5763 - Target languages supported by Firefox

packagerChrome.py

Index: packagerChrome.py
===================================================================
--- a/packagerChrome.py
+++ b/packagerChrome.py
@@ -11,6 +11,7 @@
 import struct
 import sys
 import collections
+import glob
 
 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
                       getTemplate, Files)
@@ -213,40 +214,13 @@
 
 
 def import_locales(params, files):
-    import localeTools
-
-    # FIXME: localeTools doesn't use real Chrome locales, it uses dash as
-    # separator instead.
-    convert_locale_code = lambda code: code.replace('-', '_')
-
-    # We need to map Chrome locales to Gecko locales. Start by mapping Chrome
-    # locales to themselves, merely with the dash as separator.
-    locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocales}
-
-    # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
-    for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems():
-        locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale
-
-    # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
-    reverse_mapping = {v: k for k, v in locale_mapping.iteritems()}
-    for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems():
-        if crowdin_locale in reverse_mapping:
-            locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale
-
-    for target, source in locale_mapping.iteritems():
-        targetFile = '_locales/%s/messages.json' % target
-        if not targetFile in files:
-            continue
-
-        for item in params['metadata'].items('import_locales'):
-            fileName, keys = item
-            parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
-            sourceFile = os.path.join(os.path.dirname(item.source), *parts)
-            incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
-            if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):

[Sebastian Noack, 2017/10/05 19:01:43]

I didn't care to keep supporting the .incomplete marker, as this feature is
specific to legacy Gecko extensions.

-                continue
-
-            data = json.loads(files[targetFile].decode('utf-8'))

[Sebastian Noack, 2017/10/05 19:01:43]

I only tested the translation commands after removing localetools.chromeLocales,
but just realized that this variable is also used by the code here, when
building the extension.

Previously, we went to quite some length to compile a list of locales supported
by Chrome and the respective language code used by the legacy Gecko extension.
This was necessary as we were importing strings from the "adblockplus"
repository. Also adblockplusui is using the same notation, but since the legacy
Gecko extension is gone we can just rename the translation directories in
adblockplusui, to match what we have in adblockpluschrome, now.

Anyway, there no longer is a known list of languages supported by Chrome, and
relying on such a list would no longer be accurate, as we now also need
translations for all languages supported by Firefox. So instead I'm now
importing every available translation, assuming the language code matches.

+    for item in params['metadata'].items('import_locales'):
+        filename, keys = item
+        for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
+                                                 *filename.split('/'))):
+            locale = sourceFile.split(os.path.sep)[-2].replace('-', '_')

[Sebastian Noack, 2017/10/05 19:01:43]

Replacing dashes with underscore here is only necessary to support the BCP-47
format. Pelase remove this replacement with your patch that removes support for
the BCP-47 format.

+            targetFile = os.path.join('_locales', locale, 'messages.json')
+            data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
 
             try:
                 # The WebExtensions (.json) and Gecko format provide
@@ -258,6 +232,7 @@
                         sourceData = json.load(handle)
                     import_string = import_string_webext
                 else:
+                    import localeTools
                     sourceData = localeTools.readFile(sourceFile)
                     import_string = import_string_gecko
 
@@ -296,10 +271,7 @@
     return text[:length_limit - 1].rstrip() + u'\u2026'
 
 
-def fixTranslationsForCWS(files):
-    # Chrome Web Store requires messages used in manifest.json to be present in
-    # all languages. It also enforces length limits for extension names and
-    # descriptions.
+def fix_translations_for_chrome(files):
     defaults = {}
     data = json.loads(files['_locales/%s/messages.json' % defaultLocale])
     for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
@@ -313,17 +285,30 @@
         if match:
             limits[match.group(1)] = limit
 
-    for filename in files:
-        if not filename.startswith('_locales/') or not filename.endswith('/messages.json'):
+    for path in list(files):
+        match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
+        if not match:
             continue
 
-        data = json.loads(files[filename])
-        for name, info in defaults.iteritems():
-            data.setdefault(name, info)
-        for name, limit in limits.iteritems():
-            if name in data:
-                data[name]['message'] = truncate(data[name]['message'], limit)
-        files[filename] = toJson(data)
+        # The Chrome Web Store requires messages used in manifest.json to
+        # be present in all languages, and enforces length limits on
+        # extension name and description.
+        is_latam, is_mexican, filename = match.groups()
+        if filename == 'messages.json':
+            data = json.loads(files[path])
+            for name, info in defaults.iteritems():
+                data.setdefault(name, info)
+            for name, limit in limits.iteritems():
+                info = data.get(name)
+                if info:
+                    info['message'] = truncate(info['message'], limit)
+            files[path] = toJson(data)
+
+        # Chrome combines Latin American dialects of Spanish into es-419.
+        if is_latam:
+            data = files.pop(path)
+            if is_mexican:
+                files['_locales/es_419/' + filename] = data
 
 
 def signBinary(zipdata, keyFile):
@@ -403,7 +388,7 @@
 
     files['manifest.json'] = createManifest(params, files)
     if type == 'chrome':
-        fixTranslationsForCWS(files)
+        fix_translations_for_chrome(files)
 
     if devenv:
         import buildtools