Issue 5477 - Import everything from imported locales

packagerChrome.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import errno
 import io
 import json
 import os
 import re
 from StringIO import StringIO
@@ skipping 172 matching lines @@
         ).encode('utf-8')
 
 
 def toJson(data):
     return json.dumps(
         data, ensure_ascii=False, sort_keys=True,
         indent=2, separators=(',', ': ')
     ).encode('utf-8') + '\n'
 
 
-def importGeckoLocales(params, files):
+def set_full_dict(data, key, value, source, stringID):
+    """Sets data[key] to the desired {'message':value}, adds all additional
+    info from the source dict"""
+    data[key] = {'messages': value}
+    for k, v in source[stringID].items():
+        data[key].setdefault(k, v)
+
+
+def set_message_only(data, key, value, *args):
+    """only set {'message': value} in data-dictionary, API-consitent with
+    set_full_dict()"""
+    data[key] = {'message': value}
+
+
+def import_locales(params, files):
     import localeTools
 
     # FIXME: localeTools doesn't use real Chrome locales, it uses dash as
     # separator instead.
     convert_locale_code = lambda code: code.replace('-', '_')
 
     # We need to map Chrome locales to Gecko locales. Start by mapping Chrome
     # locales to themselves, merely with the dash as separator.
     locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocales}
 
@@ skipping 16 matching lines @@
             fileName, keys = item
             parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
             sourceFile = os.path.join(os.path.dirname(item.source), *parts)
             incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
             if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
                 continue
 
             data = json.loads(files[targetFile].decode('utf-8'))
 
             try:
+                # .json and other formats provide translations differently and
+                # / or provide additional information like e.g. "placeholders".
+                # We want to adhere to that / preserve the addtional info
                 if sourceFile.endswith('.json'):
                     with io.open(sourceFile, 'r', encoding='utf-8') as handle:
-                        sourceData = {k: v['message'] for k, v in json.load(handle).iteritems()}
+                        sourceData = json.load(handle)
+
+                    get_value = lambda x: x[stringID]['message']

[Sebastian Noack, 2017/08/17 16:32:34]

PEP-8 discourages assignment of lambda functions to variables, and suggests to
use def instead.This also causes an E731 with flake8, however, it might be
ignored for this file, because of legacy code that already uses this practice.

When turning get_value() into a proper function defined here, you also might
want to define set_translation() the same way for consistency.

[tlucas, 2017/08/18 07:34:15]

Done.

+                    set_translation = set_full_dict
                 else:
                     sourceData = localeTools.readFile(sourceFile)
+                    get_value = lambda x: x[stringID]
+                    set_translation = set_message_only
 
                 # Resolve wildcard imports
                 if keys == '*' or keys == '=*':
                     importList = sourceData.keys()
                     importList = filter(lambda k: not k.startswith('_'), importList)
                     if keys == '=*':
                         importList = map(lambda k: '=' + k, importList)
                     keys = ' '.join(importList)
 
                 for stringID in keys.split():
                     noMangling = False
                     if stringID.startswith('='):
                         stringID = stringID[1:]
                         noMangling = True
 
                     if stringID in sourceData:
                         if noMangling:
                             key = re.sub(r'\W', '_', stringID)
                         else:
                             key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID)
                         if key in data:
                             print 'Warning: locale string %s defined multiple times' % key
 
                         # Remove access keys
-                        value = sourceData[stringID]
+                        value = get_value(sourceData)
                         match = re.search(r'^(.*?)\s*\(&.\)$', value)
                         if match:
                             value = match.group(1)
                         else:
                             index = value.find('&')
                             if index >= 0:
                                 value = value[0:index] + value[index + 1:]
-                        data[key] = {'message': value}
+
+                        set_translation(data, key, value, sourceData, stringID)
             except Exception as e:
                 print 'Warning: error importing locale data from %s: %s' % (sourceFile, e)
 
             files[targetFile] = toJson(data)
 
 
 def truncate(text, length_limit):
     if len(text) <= length_limit:
         return text
     return text[:length_limit - 1].rstrip() + u'\u2026'
@@ skipping 95 matching lines @@
     if metadata.has_section('convert_js'):
         convertJS(params, files)
 
     if metadata.has_section('preprocess'):
         files.preprocess(
             [f for f, _ in metadata.items('preprocess')],
             {'needsExt': True}
         )
 
     if metadata.has_section('import_locales'):
-        importGeckoLocales(params, files)
+        import_locales(params, files)
 
     files['manifest.json'] = createManifest(params, files)
     if type == 'chrome':
         fixTranslationsForCWS(files)
 
     if devenv:
         import buildtools
         import random
         files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js'), relpath='devenvPoller__.js')
         files['devenvVersion__'] = str(random.random())
 
         if metadata.has_option('general', 'testScripts'):
             files['qunit/index.html'] = createScriptPage(
                 params, 'testIndex.html.tmpl', ('general', 'testScripts')
             )
 
     zipdata = files.zipToString()
     signature = None
     pubkey = None
     if keyFile != None:
         signature = signBinary(zipdata, keyFile)
         pubkey = getPublicKey(keyFile)
     writePackage(outFile, pubkey, signature, zipdata)