Issue 5535 - Replace our module system with webpack

packagerChrome.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import errno
+import glob
 import io
 import json
 import os
 import re
-import shutil
 from StringIO import StringIO
 import struct
 import subprocess
 import sys
-import tempfile
 
 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
                       getTemplate, Files)
 
 defaultLocale = 'en_US'
 
 
 def getIgnoredFiles(params):
     return {'store.description'}
 
@@ skipping 108 matching lines @@
     # Normalize JSON structure
     licenseComment = re.compile(r'/\*.*?\*/', re.S)
     data = json.loads(re.sub(licenseComment, '', manifest, 1))
     if '_dummy' in data:
         del data['_dummy']
     manifest = json.dumps(data, sort_keys=True, indent=2)
 
     return manifest.encode('utf-8')
 
 
+def toJson(data):
+    return json.dumps(
+        data, ensure_ascii=False, sort_keys=True,
+        indent=2, separators=(',', ': ')
+    ).encode('utf-8') + '\n'
+
+
 def create_bundles(params, files):
     base_extension_path = params['baseDir']
     info_templates = {
         'chrome': 'chromeInfo.js.tmpl',
         'edge': 'edgeInfo.js.tmpl',
         'gecko-webext': 'geckoInfo.js.tmpl'
     }
-    info_module = None
-
-    # Once we use relative paths when requiring modules we can remove these,
-    # but in the mean time Webpack needs to know where to look.
-    resolve_paths = ' '.join(['./lib',
-                              './adblockpluscore/lib',
-                              './adblockplusui/lib'])

[Sebastian Noack, 2017/09/22 19:28:02]

Mind filing a follow up issue (and referring to it here), to replace absolute
paths across adblockpluschrome/ablockplusui/andblockpluscore?

For the time being, I'd prefer to look in 'lib' + '*/lib', rather than
hard-coding these paths. This however, would not work because 'buildtools/lib'
which includes conflicting scripts, which are only used in legacy Gecko
extensions. But we should remove support for legacy Gecko builds (as well as for
Safari builds) from buildtools, anyway, regardless of this change. Can you file
an issue for that as well?

[Sebastian Noack, 2017/09/22 22:21:12]

Never mind, these issues have already been filed, apparently:
https://issues.adblockplus.org/ticket/5751
https://issues.adblockplus.org/ticket/5752

[kzar, 2017/09/23 20:26:36]

OK since we're now considering removing buildtools/lib a blocker for this issue
I figured we could do that. I ran into two problems though. Firstly devenv.*/lib
directories were included, which broke everything. So I reused the
ensure_dependencies logic to only check directories which match our
dependencies. Second problem I hit was that elemHideEmulation started resolving
to adblockplus/lib/elemHideEmulation.js which broke everything too.

So now the logic says check the lib directory of all our immediate dependencies
unless they're called "adblockplus". I do wonder if it's worth the trouble
however, perhaps hardcoding them was better after all?

[Sebastian Noack, 2017/09/23 20:49:01]

It seems you missed this part?
Dang. FWIW, we also want to get rid of the "adblockplus" dependency. This
repository is deprecated now, and even before, since "adblockpluscore" moved to
a separate repository, we only kept the dependency on "adblockplus" as temporary
hack to import strings until we moved them where they belong. There should be an
issue for that, otherwise we should file one! However, since the issue tracker
is currently down, I cannot look into it.

Anyway, I do not necessarily want to block this review by an endless lists of
cleanups that are overdue, in particular since the code here is a temporary
workaround anyway. Feel free to go back to your initial approach just
whitelsiting the relevant paths, for now, and remove the dependency on #5751.
(Still I would love to see, #5751 and #5752 move forward independently.)

[kzar, 2017/09/24 09:47:53]

Oh, I forgot to mention that I tried to file the issues but the tracker was
down. I've done that now see #5760, #5761 and #5762.
Yea I agree, Done.

-    try:
-        temp_dir = tempfile.mkdtemp()

[Sebastian Noack, 2017/09/22 19:28:02]

This should be outside of the try-finally block. Otherwise, if it fails, the
code in the finally block would fail as well.

[kzar, 2017/09/23 20:26:35]

Done.

-        template = getTemplate(info_templates[params['type']])
-        with tempfile.NamedTemporaryFile(delete=False,

[Sebastian Noack, 2017/09/22 19:28:02]

Do we need to use NamedTemporaryFile here, or can't we just hard-code the
filename, since we are inside a temporary folder anyway?

[kzar, 2017/09/23 20:26:35]

Done. (But I wonder if the file name is more likely to clash now.)

-                                         dir=temp_dir) as info_file:
-            info_file.write(
-                template.render(
-                    basename=params['metadata'].get('general', 'basename'),
-                    version=params['metadata'].get('general', 'version')
-                ).encode('utf-8')
-            )
-            info_module = info_file.name
-
-        for item in params['metadata'].items('bundles'):
-            name, value = item
-            base_item_path = os.path.dirname(item.source)
-
-            bundle_file = os.path.relpath(os.path.join(base_item_path, name),
-                                          base_extension_path)
-            entry_files = [
-                os.path.join(
-                    '.',
-                    os.path.relpath(os.path.join(base_item_path, module_path),
-                                    base_extension_path)
-                )
-                for module_path in value.split()
-            ]
-            subprocess.check_call(
-                ['npm', 'run-script', 'webpack', '--silent'],
-                cwd=os.path.dirname(__file__),
-                env={
-                    'EXTENSION_PATH': base_extension_path,
-                    'ENTRY_POINTS': ' '.join(entry_files),
-                    'OUTPUT_PATH': temp_dir,
-                    'BUNDLE_NAME': bundle_file,
-                    'RESOLVE_PATHS': resolve_paths,
-                    'INFO_PATH': info_module,
-                    'PATH': os.environ['PATH']
-                }
-            )
-            for file_name in [bundle_file, bundle_file + '.map']:
-                with open(os.path.join(temp_dir, file_name), 'r') as f:
-                    files[file_name] = f.read()
-    finally:
-        shutil.rmtree(temp_dir)
-
-
-def toJson(data):
-    return json.dumps(
-        data, ensure_ascii=False, sort_keys=True,
-        indent=2, separators=(',', ': ')
-    ).encode('utf-8') + '\n'
-
-
-def import_string_webext(data, key, source):
-    """Import a single translation from the source dictionary into data"""
-    data[key] = source
-
-
-def import_string_gecko(data, key, value):
-    """Import Gecko-style locales into data.
-
-    Only sets {'message': value} in the data-dictionary, after stripping
-    undesired Gecko-style access keys.
-    """
-    match = re.search(r'^(.*?)\s*\(&.\)$', value)
-    if match:
-        value = match.group(1)
-    else:
-        index = value.find('&')
-        if index >= 0:
-            value = value[0:index] + value[index + 1:]
-
-    data[key] = {'message': value}
+
+    # Historically we didn't use relative paths when requiring modules, so in
+    # order for webpack to know where to find them we need to pass in a list of
+    # resolve paths. Going forward we should always use relative paths, once we
+    # do that consistently this can be removed. See issues 5760, 5761 and 5762.
+    resolve_paths = [os.path.join(base_extension_path, dir, 'lib')
+                     for dir in ['', 'adblockpluscore', 'adblockplusui']]
+
+    info_template = getTemplate(info_templates[params['type']])
+    info_module = info_template.render(
+        basename=params['metadata'].get('general', 'basename'),
+        version=params['metadata'].get('general', 'version')
+    ).encode('utf-8')
+
+    configuration = {
+        'bundles': [],
+        'extension_path': base_extension_path,
+        'info_module': info_module,
+        'resolve_paths': resolve_paths,
+    }
+
+    for item in params['metadata'].items('bundles'):
+        name, value = item
+        base_item_path = os.path.dirname(item.source)
+
+        bundle_file = os.path.relpath(os.path.join(base_item_path, name),
+                                      base_extension_path)
+        entry_files = [os.path.join(base_item_path, module_path)
+                       for module_path in value.split()]
+        configuration['bundles'].append({
+            'bundle_name': bundle_file,
+            'entry_points': entry_files,
+        })
+
+    cmd = ['node', os.path.join(os.path.dirname(__file__), 'webpack_runner.js')]
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                               stdin=subprocess.PIPE)
+    output = process.communicate(input=toJson(configuration))[0]
+    if process.returncode != 0:
+        raise subprocess.CalledProcessError(process.returncode, cmd=cmd)
+
+    bundles = json.loads(output)
+    for bundle in bundles:
+        files[bundle] = bundles[bundle].encode('utf-8')
 
 
 def import_locales(params, files):
-    import localeTools
-
-    # FIXME: localeTools doesn't use real Chrome locales, it uses dash as
-    # separator instead.
-    convert_locale_code = lambda code: code.replace('-', '_')
-
-    # We need to map Chrome locales to Gecko locales. Start by mapping Chrome
-    # locales to themselves, merely with the dash as separator.
-    locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocales}
-
-    # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
-    for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems():
-        locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale
-
-    # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
-    reverse_mapping = {v: k for k, v in locale_mapping.iteritems()}
-    for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems():
-        if crowdin_locale in reverse_mapping:
-            locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale
-
-    for target, source in locale_mapping.iteritems():
-        targetFile = '_locales/%s/messages.json' % target
-        if not targetFile in files:
-            continue
-
-        for item in params['metadata'].items('import_locales'):
-            fileName, keys = item
-            parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
-            sourceFile = os.path.join(os.path.dirname(item.source), *parts)
-            incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
-            if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
-                continue
-
-            data = json.loads(files[targetFile].decode('utf-8'))
+    for item in params['metadata'].items('import_locales'):
+        filename, keys = item
+        for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
+                                                 *filename.split('/'))):
+            locale = sourceFile.split(os.path.sep)[-2]
+            targetFile = os.path.join('_locales', locale, 'messages.json')
+            data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
 
             try:
-                # The WebExtensions (.json) and Gecko format provide
-                # translations differently and/or provide additional
-                # information like e.g. "placeholders". We want to adhere to
-                # that and preserve the addtional info.
-                if sourceFile.endswith('.json'):
-                    with io.open(sourceFile, 'r', encoding='utf-8') as handle:
-                        sourceData = json.load(handle)
-                    import_string = import_string_webext
-                else:
-                    sourceData = localeTools.readFile(sourceFile)
-                    import_string = import_string_gecko
+                with io.open(sourceFile, 'r', encoding='utf-8') as handle:
+                    sourceData = json.load(handle)
 
                 # Resolve wildcard imports
-                if keys == '*' or keys == '=*':
+                if keys == '*':
                     importList = sourceData.keys()
                     importList = filter(lambda k: not k.startswith('_'), importList)
-                    if keys == '=*':
-                        importList = map(lambda k: '=' + k, importList)
                     keys = ' '.join(importList)
 
                 for stringID in keys.split():
-                    noMangling = False
-                    if stringID.startswith('='):
-                        stringID = stringID[1:]
-                        noMangling = True
-
                     if stringID in sourceData:
-                        if noMangling:
-                            key = re.sub(r'\W', '_', stringID)
-                        else:
-                            key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID)
-                        if key in data:
-                            print 'Warning: locale string %s defined multiple times' % key
-
-                        import_string(data, key, sourceData[stringID])
+                        if stringID in data:
+                            print ('Warning: locale string {} defined multiple'
+                                   ' times').format(stringID)
+
+                        data[stringID] = sourceData[stringID]
             except Exception as e:
                 print 'Warning: error importing locale data from %s: %s' % (sourceFile, e)
 
             files[targetFile] = toJson(data)
 
 
 def truncate(text, length_limit):
     if len(text) <= length_limit:
         return text
     return text[:length_limit - 1].rstrip() + u'\u2026'
 
 
-def fixTranslationsForCWS(files):
-    # Chrome Web Store requires messages used in manifest.json to be present in
-    # all languages. It also enforces length limits for extension names and
-    # descriptions.
+def fix_translations_for_chrome(files):
     defaults = {}
     data = json.loads(files['_locales/%s/messages.json' % defaultLocale])
     for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
         name = match.group(1)
         defaults[name] = data[name]
 
     limits = {}
     manifest = json.loads(files['manifest.json'])
     for key, limit in (('name', 45), ('description', 132), ('short_name', 12)):
         match = re.search(r'__MSG_(\S+)__', manifest.get(key, ''))
         if match:
             limits[match.group(1)] = limit
 
-    for filename in files:
-        if not filename.startswith('_locales/') or not filename.endswith('/messages.json'):
+    for path in list(files):
+        match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
+        if not match:
             continue
 
-        data = json.loads(files[filename])
-        for name, info in defaults.iteritems():
-            data.setdefault(name, info)
-        for name, limit in limits.iteritems():
-            if name in data:
-                data[name]['message'] = truncate(data[name]['message'], limit)
-        files[filename] = toJson(data)
+        # The Chrome Web Store requires messages used in manifest.json to
+        # be present in all languages, and enforces length limits on
+        # extension name and description.
+        is_latam, is_mexican, filename = match.groups()
+        if filename == 'messages.json':
+            data = json.loads(files[path])
+            for name, info in defaults.iteritems():
+                data.setdefault(name, info)
+            for name, limit in limits.iteritems():
+                info = data.get(name)
+                if info:
+                    info['message'] = truncate(info['message'], limit)
+            files[path] = toJson(data)
+
+        # Chrome combines Latin American dialects of Spanish into es-419.
+        if is_latam:
+            data = files.pop(path)
+            if is_mexican:
+                files['_locales/es_419/' + filename] = data
 
 
 def signBinary(zipdata, keyFile):
     from Crypto.Hash import SHA
     from Crypto.PublicKey import RSA
     from Crypto.Signature import PKCS1_v1_5
 
     try:
         with open(keyFile, 'rb') as file:
             key = RSA.importKey(file.read())
@@ skipping 59 matching lines @@
         files.preprocess(
             [f for f, _ in metadata.items('preprocess')],
             {'needsExt': True}
         )
 
     if metadata.has_section('import_locales'):
         import_locales(params, files)
 
     files['manifest.json'] = createManifest(params, files)
     if type == 'chrome':
-        fixTranslationsForCWS(files)
+        fix_translations_for_chrome(files)
 
     if devenv:
         import buildtools
         import random
         files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js'), relpath='devenvPoller__.js')
         files['devenvVersion__'] = str(random.random())
 
         if metadata.has_option('general', 'testScripts'):
             files['qunit/index.html'] = createScriptPage(
                 params, 'testIndex.html.tmpl', ('general', 'testScripts')
             )
 
     zipdata = files.zipToString()
     signature = None
     pubkey = None
     if keyFile != None:
         signature = signBinary(zipdata, keyFile)
         pubkey = getPublicKey(keyFile)
     writePackage(outFile, pubkey, signature, zipdata)