Issue 5535 - Replace our module system with webpack

packagerChrome.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import errno
+import glob
 import io
 import json
 import os
 import re
-import shutil
 from StringIO import StringIO
 import struct
 import subprocess
 import sys
-import tempfile
-
-from ensure_dependencies import read_deps
+
 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
                       getTemplate, Files)
 
 defaultLocale = 'en_US'
 
 
 def getIgnoredFiles(params):
     return {'store.description'}
 
 
@@ skipping 107 matching lines @@
     # Normalize JSON structure
     licenseComment = re.compile(r'/\*.*?\*/', re.S)
     data = json.loads(re.sub(licenseComment, '', manifest, 1))
     if '_dummy' in data:
         del data['_dummy']
     manifest = json.dumps(data, sort_keys=True, indent=2)
 
     return manifest.encode('utf-8')
 
 
+def toJson(data):
+    return json.dumps(
+        data, ensure_ascii=False, sort_keys=True,
+        indent=2, separators=(',', ': ')
+    ).encode('utf-8') + '\n'
+
+
 def create_bundles(params, files):
     base_extension_path = params['baseDir']
     info_templates = {
         'chrome': 'chromeInfo.js.tmpl',
         'edge': 'edgeInfo.js.tmpl',
         'gecko-webext': 'geckoInfo.js.tmpl'
     }
-    info_module = None
 
     # Historically we didn't use relative paths when requiring modules, so in
     # order for webpack to know where to find them we need to pass in a list of
     # resolve paths. Going forward we should always use relative paths, once we
     # do that consistently this can be removed. See issues 5760, 5761 and 5762.
-    resolve_paths = ' '.join(
-        [os.path.join(base_extension_path, dir, 'lib')
-         for dir in ['', 'adblockpluscore', 'adblockplusui']]
-    )
-
-    temp_dir = tempfile.mkdtemp()
-    try:
-        info_module = os.path.join(temp_dir, 'info.js')
-        template = getTemplate(info_templates[params['type']])
-        with open(info_module, 'w') as info_file:
-            info_file.write(
-                template.render(
-                    basename=params['metadata'].get('general', 'basename'),
-                    version=params['metadata'].get('general', 'version')
-                ).encode('utf-8')
-            )

[Wladimir Palant, 2017/09/26 12:06:13]

Writing a temporary file here should be avoidable if we define a custom loader
for it (see https://webpack.js.org/concepts/loaders/).

-
-        for item in params['metadata'].items('bundles'):
-            name, value = item
-            base_item_path = os.path.dirname(item.source)
-
-            bundle_file = os.path.relpath(os.path.join(base_item_path, name),
-                                          base_extension_path)
-            entry_files = [
-                os.path.join(
-                    '.',
-                    os.path.relpath(os.path.join(base_item_path, module_path),
-                                    base_extension_path)

[Wladimir Palant, 2017/09/26 12:06:13]

Please use absolute paths here, this should make things much more
straightforward.

[kzar, 2017/10/02 18:48:32]

Good idea, Done. (I tried to do the same with bundle_file while at it but
webpack insisted on a relative path there.)

-                )
-                for module_path in value.split()
-            ]
-            subprocess.check_call(
-                ['npm', 'run-script', 'webpack', '--silent'],
-                cwd=os.path.dirname(__file__),
-                env={
-                    'EXTENSION_PATH': base_extension_path,
-                    'ENTRY_POINTS': ' '.join(entry_files),
-                    'OUTPUT_PATH': temp_dir,
-                    'BUNDLE_NAME': bundle_file,
-                    'RESOLVE_PATHS': resolve_paths,
-                    'INFO_PATH': info_module,
-                    'PATH': os.environ['PATH']
-                }
-            )

[Wladimir Palant, 2017/09/26 12:06:13]

Writing to temporary files shouldn't be necessary, see
https://webpack.js.org/api/node/#custom-file-systems on how to compile bundles
in memory. This implies calling the webpack module directly rather than using
the CLI however.

-            for file_name in [bundle_file, bundle_file + '.map']:
-                with open(os.path.join(temp_dir, file_name), 'r') as f:
-                    files[file_name] = f.read()
-    finally:
-        shutil.rmtree(temp_dir)
-
-
-def toJson(data):
-    return json.dumps(
-        data, ensure_ascii=False, sort_keys=True,
-        indent=2, separators=(',', ': ')
-    ).encode('utf-8') + '\n'
-
-
-def import_string_webext(data, key, source):
-    """Import a single translation from the source dictionary into data"""
-    data[key] = source
-
-
-def import_string_gecko(data, key, value):
-    """Import Gecko-style locales into data.
-
-    Only sets {'message': value} in the data-dictionary, after stripping
-    undesired Gecko-style access keys.
-    """
-    match = re.search(r'^(.*?)\s*\(&.\)$', value)
-    if match:
-        value = match.group(1)
-    else:
-        index = value.find('&')
-        if index >= 0:
-            value = value[0:index] + value[index + 1:]
-
-    data[key] = {'message': value}
+    resolve_paths = [os.path.join(base_extension_path, dir, 'lib')
+                     for dir in ['', 'adblockpluscore', 'adblockplusui']]
+
+    info_template = getTemplate(info_templates[params['type']])
+    info_module = info_template.render(
+        basename=params['metadata'].get('general', 'basename'),
+        version=params['metadata'].get('general', 'version')
+    ).encode('utf-8')
+
+    configuration = {
+        'bundles': [],
+        'extension_path': base_extension_path,
+        'info_module': info_module,
+        'resolve_paths': resolve_paths,
+    }
+
+    for item in params['metadata'].items('bundles'):
+        name, value = item
+        base_item_path = os.path.dirname(item.source)
+
+        bundle_file = os.path.relpath(os.path.join(base_item_path, name),
+                                      base_extension_path)
+        entry_files = [os.path.join(base_item_path, module_path)
+                       for module_path in value.split()]
+        configuration['bundles'].append({
+            'bundle_name': bundle_file,
+            'entry_points': entry_files,
+        })
+
+    cmd = ['node', os.path.join(os.path.dirname(__file__), 'webpack_runner.js')]
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                               stdin=subprocess.PIPE)
+    output = process.communicate(input=toJson(configuration))[0]
+    if process.returncode != 0:
+        raise subprocess.CalledProcessError(process.returncode, cmd=cmd)
+
+    bundles = json.loads(output)
+    for bundle in bundles:
+        files[bundle] = bundles[bundle].encode('utf-8')
 
 
 def import_locales(params, files):
-    import localeTools
-
-    # FIXME: localeTools doesn't use real Chrome locales, it uses dash as
-    # separator instead.
-    convert_locale_code = lambda code: code.replace('-', '_')
-
-    # We need to map Chrome locales to Gecko locales. Start by mapping Chrome
-    # locales to themselves, merely with the dash as separator.
-    locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocales}
-
-    # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
-    for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems():
-        locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale
-
-    # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
-    reverse_mapping = {v: k for k, v in locale_mapping.iteritems()}
-    for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems():
-        if crowdin_locale in reverse_mapping:
-            locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale
-
-    for target, source in locale_mapping.iteritems():
-        targetFile = '_locales/%s/messages.json' % target
-        if not targetFile in files:
-            continue
-
-        for item in params['metadata'].items('import_locales'):
-            fileName, keys = item
-            parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
-            sourceFile = os.path.join(os.path.dirname(item.source), *parts)
-            incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
-            if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
-                continue
-
-            data = json.loads(files[targetFile].decode('utf-8'))
+    for item in params['metadata'].items('import_locales'):
+        filename, keys = item
+        for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
+                                                 *filename.split('/'))):
+            locale = sourceFile.split(os.path.sep)[-2]
+            targetFile = os.path.join('_locales', locale, 'messages.json')
+            data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
 
             try:
-                # The WebExtensions (.json) and Gecko format provide
-                # translations differently and/or provide additional
-                # information like e.g. "placeholders". We want to adhere to
-                # that and preserve the addtional info.
-                if sourceFile.endswith('.json'):
-                    with io.open(sourceFile, 'r', encoding='utf-8') as handle:
-                        sourceData = json.load(handle)
-                    import_string = import_string_webext
-                else:
-                    sourceData = localeTools.readFile(sourceFile)
-                    import_string = import_string_gecko
+                with io.open(sourceFile, 'r', encoding='utf-8') as handle:
+                    sourceData = json.load(handle)
 
                 # Resolve wildcard imports
-                if keys == '*' or keys == '=*':
+                if keys == '*':
                     importList = sourceData.keys()
                     importList = filter(lambda k: not k.startswith('_'), importList)
-                    if keys == '=*':
-                        importList = map(lambda k: '=' + k, importList)
                     keys = ' '.join(importList)
 
                 for stringID in keys.split():
-                    noMangling = False
-                    if stringID.startswith('='):
-                        stringID = stringID[1:]
-                        noMangling = True
-
                     if stringID in sourceData:
-                        if noMangling:
-                            key = re.sub(r'\W', '_', stringID)
-                        else:
-                            key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID)
-                        if key in data:
-                            print 'Warning: locale string %s defined multiple times' % key
-
-                        import_string(data, key, sourceData[stringID])
+                        if stringID in data:
+                            print ('Warning: locale string {} defined multiple'
+                                   ' times').format(stringID)
+
+                        data[stringID] = sourceData[stringID]
             except Exception as e:
                 print 'Warning: error importing locale data from %s: %s' % (sourceFile, e)
 
             files[targetFile] = toJson(data)
 
 
 def truncate(text, length_limit):
     if len(text) <= length_limit:
         return text
     return text[:length_limit - 1].rstrip() + u'\u2026'
 
 
-def fixTranslationsForCWS(files):
-    # Chrome Web Store requires messages used in manifest.json to be present in
-    # all languages. It also enforces length limits for extension names and
-    # descriptions.
+def fix_translations_for_chrome(files):
     defaults = {}
     data = json.loads(files['_locales/%s/messages.json' % defaultLocale])
     for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
         name = match.group(1)
         defaults[name] = data[name]
 
     limits = {}
     manifest = json.loads(files['manifest.json'])
     for key, limit in (('name', 45), ('description', 132), ('short_name', 12)):
         match = re.search(r'__MSG_(\S+)__', manifest.get(key, ''))
         if match:
             limits[match.group(1)] = limit
 
-    for filename in files:
-        if not filename.startswith('_locales/') or not filename.endswith('/messages.json'):
+    for path in list(files):
+        match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
+        if not match:
             continue
 
-        data = json.loads(files[filename])
-        for name, info in defaults.iteritems():
-            data.setdefault(name, info)
-        for name, limit in limits.iteritems():
-            if name in data:
-                data[name]['message'] = truncate(data[name]['message'], limit)
-        files[filename] = toJson(data)
+        # The Chrome Web Store requires messages used in manifest.json to
+        # be present in all languages, and enforces length limits on
+        # extension name and description.
+        is_latam, is_mexican, filename = match.groups()
+        if filename == 'messages.json':
+            data = json.loads(files[path])
+            for name, info in defaults.iteritems():
+                data.setdefault(name, info)
+            for name, limit in limits.iteritems():
+                info = data.get(name)
+                if info:
+                    info['message'] = truncate(info['message'], limit)
+            files[path] = toJson(data)
+
+        # Chrome combines Latin American dialects of Spanish into es-419.
+        if is_latam:
+            data = files.pop(path)
+            if is_mexican:
+                files['_locales/es_419/' + filename] = data
 
 
 def signBinary(zipdata, keyFile):
     from Crypto.Hash import SHA
     from Crypto.PublicKey import RSA
     from Crypto.Signature import PKCS1_v1_5
 
     try:
         with open(keyFile, 'rb') as file:
             key = RSA.importKey(file.read())
@@ skipping 59 matching lines @@
         files.preprocess(
             [f for f, _ in metadata.items('preprocess')],
             {'needsExt': True}
         )
 
     if metadata.has_section('import_locales'):
         import_locales(params, files)
 
     files['manifest.json'] = createManifest(params, files)
     if type == 'chrome':
-        fixTranslationsForCWS(files)
+        fix_translations_for_chrome(files)
 
     if devenv:
         import buildtools
         import random
         files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js'), relpath='devenvPoller__.js')
         files['devenvVersion__'] = str(random.random())
 
         if metadata.has_option('general', 'testScripts'):
             files['qunit/index.html'] = createScriptPage(
                 params, 'testIndex.html.tmpl', ('general', 'testScripts')
             )
 
     zipdata = files.zipToString()
     signature = None
     pubkey = None
     if keyFile != None:
         signature = signBinary(zipdata, keyFile)
         pubkey = getPublicKey(keyFile)
     writePackage(outFile, pubkey, signature, zipdata)