Issue 5535 - Replace our module system with webpack

packagerChrome.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import errno
 import io
 import json
 import os
 import re
+import shutil
 from StringIO import StringIO
 import struct
+import subprocess
 import sys
-import collections
+import tempfile
 
+from ensure_dependencies import read_deps
 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
                       getTemplate, Files)
 
 defaultLocale = 'en_US'
 
 
 def getIgnoredFiles(params):
     return {'store.description'}
 
 
@@ skipping 107 matching lines @@
     # Normalize JSON structure
     licenseComment = re.compile(r'/\*.*?\*/', re.S)
     data = json.loads(re.sub(licenseComment, '', manifest, 1))
     if '_dummy' in data:
         del data['_dummy']
     manifest = json.dumps(data, sort_keys=True, indent=2)
 
     return manifest.encode('utf-8')
 
 
-def convertJS(params, files):
-    output_files = collections.OrderedDict()
-    args = {}
+def create_bundles(params, files):
+    base_extension_path = params['baseDir']
+    info_templates = {
+        'chrome': 'chromeInfo.js.tmpl',
+        'edge': 'edgeInfo.js.tmpl',
+        'gecko-webext': 'geckoInfo.js.tmpl'
+    }
+    info_module = None
 
-    for item in params['metadata'].items('convert_js'):
-        name, value = item
-        filename, arg = re.search(r'^(.*?)(?:\[(.*)\])?$', name).groups()
-        if arg is None:
-            output_files[filename] = (value.split(), item.source)
-        else:
-            args.setdefault(filename, {})[arg] = value
+    # Historically we didn't use relative paths when requiring modules, so in
+    # order for webpack to know where to find them we need to pass in a list of
+    # resolve paths. Going forward we should always use relative paths, once we
+    # do that consistently this can be removed. See issues 5760, 5761 and 5762.
+    resolve_paths = ' '.join(
+        [os.path.join(base_extension_path, dir, 'lib')
+         for dir in ['', 'adblockpluscore', 'adblockplusui']]
+    )
 
-    template = getTemplate('modules.js.tmpl')
+    temp_dir = tempfile.mkdtemp()
+    try:
+        info_module = os.path.join(temp_dir, 'info.js')
+        template = getTemplate(info_templates[params['type']])
+        with open(info_module, 'w') as info_file:
+            info_file.write(
+                template.render(
+                    basename=params['metadata'].get('general', 'basename'),
+                    version=params['metadata'].get('general', 'version')
+                ).encode('utf-8')
+            )

[Wladimir Palant, 2017/09/26 12:06:13]

Writing a temporary file here should be avoidable if we define a custom loader
for it (see https://webpack.js.org/concepts/loaders/).

 
-    for filename, (input_files, origin) in output_files.iteritems():
-        if '/' in filename and not files.isIncluded(filename):
-            continue
+        for item in params['metadata'].items('bundles'):
+            name, value = item
+            base_item_path = os.path.dirname(item.source)
 
-        current_args = args.get(filename, {})
-        current_args['autoload'] = [module for module in
-                                    current_args.get('autoload', '').split(',')
-                                    if module != '']
-
-        base_dir = os.path.dirname(origin)
-        modules = []
-
-        for input_filename in input_files:
-            module_name = os.path.splitext(os.path.basename(input_filename))[0]
-            prefix = os.path.basename(os.path.dirname(input_filename))
-            if prefix != 'lib':
-                module_name = '{}_{}'.format(prefix, module_name)
-            with open(os.path.join(base_dir, input_filename), 'r') as file:
-                modules.append((module_name, file.read().decode('utf-8')))
-            files.pop(input_filename, None)
-
-        files[filename] = template.render(
-            args=current_args,
-            basename=params['metadata'].get('general', 'basename'),
-            modules=modules,
-            type=params['type'],
-            version=params['metadata'].get('general', 'version')
-        ).encode('utf-8')
+            bundle_file = os.path.relpath(os.path.join(base_item_path, name),
+                                          base_extension_path)
+            entry_files = [
+                os.path.join(
+                    '.',
+                    os.path.relpath(os.path.join(base_item_path, module_path),
+                                    base_extension_path)

[Wladimir Palant, 2017/09/26 12:06:13]

Please use absolute paths here, this should make things much more
straightforward.

[kzar, 2017/10/02 18:48:32]

Good idea, Done. (I tried to do the same with bundle_file while at it but
webpack insisted on a relative path there.)

+                )
+                for module_path in value.split()
+            ]
+            subprocess.check_call(
+                ['npm', 'run-script', 'webpack', '--silent'],
+                cwd=os.path.dirname(__file__),
+                env={
+                    'EXTENSION_PATH': base_extension_path,
+                    'ENTRY_POINTS': ' '.join(entry_files),
+                    'OUTPUT_PATH': temp_dir,
+                    'BUNDLE_NAME': bundle_file,
+                    'RESOLVE_PATHS': resolve_paths,
+                    'INFO_PATH': info_module,
+                    'PATH': os.environ['PATH']
+                }
+            )

[Wladimir Palant, 2017/09/26 12:06:13]

Writing to temporary files shouldn't be necessary, see
https://webpack.js.org/api/node/#custom-file-systems on how to compile bundles
in memory. This implies calling the webpack module directly rather than using
the CLI however.

+            for file_name in [bundle_file, bundle_file + '.map']:
+                with open(os.path.join(temp_dir, file_name), 'r') as f:
+                    files[file_name] = f.read()
+    finally:
+        shutil.rmtree(temp_dir)
 
 
 def toJson(data):
     return json.dumps(
         data, ensure_ascii=False, sort_keys=True,
         indent=2, separators=(',', ': ')
     ).encode('utf-8') + '\n'
 
 
 def import_string_webext(data, key, source):
@@ skipping 188 matching lines @@
         'metadata': metadata,
     }
 
     mapped = metadata.items('mapping') if metadata.has_section('mapping') else []
     files = Files(getPackageFiles(params), getIgnoredFiles(params),
                   process=lambda path, data: processFile(path, data, params))
 
     files.readMappedFiles(mapped)
     files.read(baseDir, skip=[opt for opt, _ in mapped])
 
-    if metadata.has_section('convert_js'):
-        convertJS(params, files)
+    if metadata.has_section('bundles'):
+        create_bundles(params, files)
 
     if metadata.has_section('preprocess'):
         files.preprocess(
             [f for f, _ in metadata.items('preprocess')],
             {'needsExt': True}
         )
 
     if metadata.has_section('import_locales'):
         import_locales(params, files)
 
@@ skipping 12 matching lines @@
                 params, 'testIndex.html.tmpl', ('general', 'testScripts')
             )
 
     zipdata = files.zipToString()
     signature = None
     pubkey = None
     if keyFile != None:
         signature = signBinary(zipdata, keyFile)
         pubkey = getPublicKey(keyFile)
     writePackage(outFile, pubkey, signature, zipdata)