Issue 5763 - Target languages supported by Firefox

packagerChrome.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import errno
 import io
 import json
 import os
 import re
 from StringIO import StringIO
 import struct
 import sys
 import collections
+import glob
 
 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
                       getTemplate, Files)
 
 defaultLocale = 'en_US'
 
 
 def getIgnoredFiles(params):
     return {'store.description'}
 
@@ skipping 182 matching lines @@
         value = match.group(1)
     else:
         index = value.find('&')
         if index >= 0:
             value = value[0:index] + value[index + 1:]
 
     data[key] = {'message': value}
 
 
 def import_locales(params, files):
-    import localeTools
-
-    # FIXME: localeTools doesn't use real Chrome locales, it uses dash as
-    # separator instead.
-    convert_locale_code = lambda code: code.replace('-', '_')
-
-    # We need to map Chrome locales to Gecko locales. Start by mapping Chrome
-    # locales to themselves, merely with the dash as separator.
-    locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocales}
-
-    # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
-    for chrome_locale, crowdin_locale in localeTools.CROWDIN_LANG_MAPPING.iteritems():
-        locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale
-
-    # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
-    reverse_mapping = {v: k for k, v in locale_mapping.iteritems()}
-    for gecko_locale, crowdin_locale in localeTools.CROWDIN_LANG_MAPPING.iteritems():
-        if crowdin_locale in reverse_mapping:
-            locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale
-
-    for target, source in locale_mapping.iteritems():
-        targetFile = '_locales/%s/messages.json' % target
-        if not targetFile in files:
-            continue
-
-        for item in params['metadata'].items('import_locales'):
-            fileName, keys = item
-            parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
-            sourceFile = os.path.join(os.path.dirname(item.source), *parts)
-            incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
-            if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
-                continue
-
-            data = json.loads(files[targetFile].decode('utf-8'))
+    for item in params['metadata'].items('import_locales'):
+        filename, keys = item
+        for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
+                                                 *filename.split('/'))):
+            parts = sourceFile.split(os.path.sep)

[Sebastian Noack, 2017/10/05 20:58:38]

I just noticed that the variable "parts" is used below. I didn't notice before,
because it's only used in the branch for legacy Gecko extensions below.

@Tristan: Feel free to remove this variable again when removing support for
legacy Gecko extensions.

[tlucas, 2017/10/06 08:53:53]

Acknowledged.

+            locale = parts[-2].replace('-', '_')
+            targetFile = os.path.join('_locales', locale, 'messages.json')
+            data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
 
             try:
                 # The WebExtensions (.json) and Gecko format provide
                 # translations differently and/or provide additional
                 # information like e.g. "placeholders". We want to adhere to
                 # that and preserve the addtional info.
                 if sourceFile.endswith('.json'):
                     with io.open(sourceFile, 'r', encoding='utf-8') as handle:
                         sourceData = json.load(handle)
                     import_string = import_string_webext
                 else:
+                    import localeTools
                     sourceData = localeTools.readFile(sourceFile)
                     import_string = import_string_gecko
 
                 # Resolve wildcard imports
                 if keys == '*' or keys == '=*':
                     importList = sourceData.keys()
                     importList = filter(lambda k: not k.startswith('_'), importList)
                     if keys == '=*':
                         importList = map(lambda k: '=' + k, importList)
                     keys = ' '.join(importList)
@@ skipping 34 matching lines @@
 
     limits = {}
     manifest = json.loads(files['manifest.json'])
     for key, limit in (('name', 45), ('description', 132), ('short_name', 12)):
         match = re.search(r'__MSG_(\S+)__', manifest.get(key, ''))
         if match:
             limits[match.group(1)] = limit
 
     for path in list(files):
         match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
-        if match:
-            isLatAm, isMexican, filename = match.groups()
-
-            # The Chrome Web Store requires messages used in manifest.json to
-            # be present in all languages, and enforces length limits on
-            # extension name and description.
-            if filename == 'messages.json':
-                data = json.loads(files[path])
-                for name, info in defaults.iteritems():
-                    data.setdefault(name, info)
-                for name, limit in limits.iteritems():
-                    info = data.get(name)
-                    if info:
-                        info['message'] = truncate(info['message'], limit)
-                files[path] = toJson(data)
-
-            # Chrome combines Latin American dialects of Spanish into es-419.
-            if isLatAm:
-                data = files.pop(path)
-                if isMexican:
-                    files['_locales/es_419/' + filename] = data
+        if not match:
+            continue
+
+        # The Chrome Web Store requires messages used in manifest.json to
+        # be present in all languages, and enforces length limits on
+        # extension name and description.
+        is_latam, is_mexican, filename = match.groups()
+        if filename == 'messages.json':
+            data = json.loads(files[path])
+            for name, info in defaults.iteritems():
+                data.setdefault(name, info)
+            for name, limit in limits.iteritems():
+                info = data.get(name)
+                if info:
+                    info['message'] = truncate(info['message'], limit)
+            files[path] = toJson(data)
+
+        # Chrome combines Latin American dialects of Spanish into es-419.
+        if is_latam:
+            data = files.pop(path)
+            if is_mexican:
+                files['_locales/es_419/' + filename] = data
 
 
 def signBinary(zipdata, keyFile):
     from Crypto.Hash import SHA
     from Crypto.PublicKey import RSA
     from Crypto.Signature import PKCS1_v1_5
 
     try:
         with open(keyFile, 'rb') as file:
             key = RSA.importKey(file.read())
@@ skipping 79 matching lines @@
                 params, 'testIndex.html.tmpl', ('general', 'testScripts')
             )
 
     zipdata = files.zipToString()
     signature = None
     pubkey = None
     if keyFile != None:
         signature = signBinary(zipdata, keyFile)
         pubkey = getPublicKey(keyFile)
     writePackage(outFile, pubkey, signature, zipdata)