Issue 5477 - Import everything from imported locales

packagerChrome.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import errno
 import io
 import json
 import os
 import re
 from StringIO import StringIO
@@ skipping 172 matching lines @@
         ).encode('utf-8')
 
 
 def toJson(data):
     return json.dumps(
         data, ensure_ascii=False, sort_keys=True,
         indent=2, separators=(',', ': ')
     ).encode('utf-8') + '\n'
 
 
-def importGeckoLocales(params, files):
+def import_locales(params, files):
     import localeTools
 
     # FIXME: localeTools doesn't use real Chrome locales, it uses dash as
     # separator instead.
     convert_locale_code = lambda code: code.replace('-', '_')
 
     # We need to map Chrome locales to Gecko locales. Start by mapping Chrome
     # locales to themselves, merely with the dash as separator.
     locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocales}
 
@@ skipping 16 matching lines @@
             fileName, keys = item
             parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
             sourceFile = os.path.join(os.path.dirname(item.source), *parts)
             incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
             if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
                 continue
 
             data = json.loads(files[targetFile].decode('utf-8'))
 
             try:
+                # .json and other formats provide translations differently and
+                # / or provide additional information like e.g. "placeholders".
+                # We want to adhere to that / preserve the addtional info
                 if sourceFile.endswith('.json'):
                     with io.open(sourceFile, 'r', encoding='utf-8') as handle:
-                        sourceData = {k: v['message'] for k, v in json.load(handle).iteritems()}
+                        sourceData = json.load(handle)
+
+                    def get_value(data, stringID):
+                        return data[stringID]['message']
+
+                    def set_translation(data, key, value, source, stringID):
+                        data[key] = {'messages': value}
+                        for k, v in source[stringID].items():
+                            data[key].setdefault(k, v)
                 else:
                     sourceData = localeTools.readFile(sourceFile)
 
+                    def get_value(data, stringID):
+                        return data[stringID]
+
+                    def set_translation(data, key, value, *args):
+                        data[key] = {'message': value}
+
                 # Resolve wildcard imports
                 if keys == '*' or keys == '=*':
                     importList = sourceData.keys()
                     importList = filter(lambda k: not k.startswith('_'), importList)
                     if keys == '=*':
                         importList = map(lambda k: '=' + k, importList)
                     keys = ' '.join(importList)
 
                 for stringID in keys.split():
                     noMangling = False
                     if stringID.startswith('='):
                         stringID = stringID[1:]
                         noMangling = True
 
                     if stringID in sourceData:
                         if noMangling:
                             key = re.sub(r'\W', '_', stringID)
                         else:
                             key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID)
                         if key in data:
                             print 'Warning: locale string %s defined multiple times' % key
 
                         # Remove access keys
-                        value = sourceData[stringID]
+                        value = get_value(sourceData, stringID)
                         match = re.search(r'^(.*?)\s*\(&.\)$', value)

[Sebastian Noack, 2017/08/18 07:44:28]

The logic to strip the access keys here is actually specific to Gecko
translations, i.e. irrelevant for .json files. IT seems we could simplify the
abstraction if we only perform this code for Gecko translations.

[tlucas, 2017/08/18 12:59:37]

Ok - i will move this to the specific "get_value" instead -> done

                         if match:
                             value = match.group(1)
                         else:
                             index = value.find('&')
                             if index >= 0:
                                 value = value[0:index] + value[index + 1:]
-                        data[key] = {'message': value}
+
+                        set_translation(data, key, value, sourceData, stringID)
             except Exception as e:
                 print 'Warning: error importing locale data from %s: %s' % (sourceFile, e)
 
             files[targetFile] = toJson(data)
 
 
 def truncate(text, length_limit):
     if len(text) <= length_limit:
         return text
     return text[:length_limit - 1].rstrip() + u'\u2026'
@@ skipping 95 matching lines @@
     if metadata.has_section('convert_js'):
         convertJS(params, files)
 
     if metadata.has_section('preprocess'):
         files.preprocess(
             [f for f, _ in metadata.items('preprocess')],
             {'needsExt': True}
         )
 
     if metadata.has_section('import_locales'):
-        importGeckoLocales(params, files)
+        import_locales(params, files)
 
     files['manifest.json'] = createManifest(params, files)
     if type == 'chrome':
         fixTranslationsForCWS(files)
 
     if devenv:
         import buildtools
         import random
         files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js'), relpath='devenvPoller__.js')
         files['devenvVersion__'] = str(random.random())
 
         if metadata.has_option('general', 'testScripts'):
             files['qunit/index.html'] = createScriptPage(
                 params, 'testIndex.html.tmpl', ('general', 'testScripts')
             )
 
     zipdata = files.zipToString()
     signature = None
     pubkey = None
     if keyFile != None:
         signature = signBinary(zipdata, keyFile)
         pubkey = getPublicKey(keyFile)
     writePackage(outFile, pubkey, signature, zipdata)