Issue 5383 - Add tests for the Chrome and Firefox packagers

packagerChrome.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import errno
+import glob
 import io
 import json
 import os
 import re
-from StringIO import StringIO
 import struct
+import subprocess
 import sys
-import collections
+import random
 
 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
                       getTemplate, Files)
 
 defaultLocale = 'en_US'
 
 
 def getIgnoredFiles(params):
     return {'store.description'}
 
@@ skipping 20 matching lines @@
 
 def makeIcons(files, filenames):
     icons = {}
     for filename in filenames:
         try:
             magic, width, height = struct.unpack_from('>8s8xii',
                                                       files[filename])
         except struct.error:
             magic = None
         if magic != '\x89PNG\r\n\x1a\n':
-            raise TypeError(filename + ' is no valid PNG.')

[Sebastian Noack, 2017/09/21 19:09:03]

The definition of TypeError doesn't seem to apply here:
https://docs.python.org/3/library/exceptions.html#TypeError

This would rather be a ValueError. But for consistency, I would just go with
Exception, which is what buildtools is raising in other scenarios like this.

[tlucas, 2017/09/22 09:02:18]

Done.

+            raise Exception(filename + ' is no valid PNG.')
         if(width != height):
             print >>sys.stderr, 'Warning: %s size is %ix%i, icon should be square' % (filename, width, height)
         icons[width] = filename
     return icons
 
 
 def createScriptPage(params, template_name, script_option):
     template = getTemplate(template_name, autoEscape=True)
     return template.render(
         basename=params['metadata'].get('general', 'basename'),
@@ skipping 69 matching lines @@
     # Normalize JSON structure
     licenseComment = re.compile(r'/\*.*?\*/', re.S)
     data = json.loads(re.sub(licenseComment, '', manifest, 1))
     if '_dummy' in data:
         del data['_dummy']
     manifest = json.dumps(data, sort_keys=True, indent=2)
 
     return manifest.encode('utf-8')
 
 
-def convertJS(params, files):
-    output_files = collections.OrderedDict()
-    args = {}
-
-    for item in params['metadata'].items('convert_js'):
-        name, value = item
-        filename, arg = re.search(r'^(.*?)(?:\[(.*)\])?$', name).groups()
-        if arg is None:
-            output_files[filename] = (value.split(), item.source)
-        else:
-            args.setdefault(filename, {})[arg] = value
-
-    template = getTemplate('modules.js.tmpl')
-
-    for filename, (input_files, origin) in output_files.iteritems():
-        if '/' in filename and not files.isIncluded(filename):
-            continue
-
-        current_args = args.get(filename, {})
-        current_args['autoload'] = [module for module in
-                                    current_args.get('autoload', '').split(',')
-                                    if module != '']
-
-        base_dir = os.path.dirname(origin)
-        modules = []
-
-        for input_filename in input_files:
-            module_name = os.path.splitext(os.path.basename(input_filename))[0]
-            prefix = os.path.basename(os.path.dirname(input_filename))
-            if prefix != 'lib':
-                module_name = '{}_{}'.format(prefix, module_name)
-            with open(os.path.join(base_dir, input_filename), 'r') as file:
-                modules.append((module_name, file.read().decode('utf-8')))
-            files.pop(input_filename, None)
-
-        files[filename] = template.render(
-            args=current_args,
-            basename=params['metadata'].get('general', 'basename'),
-            modules=modules,
-            type=params['type'],
-            version=params['metadata'].get('general', 'version')
-        ).encode('utf-8')
-
-
 def toJson(data):
     return json.dumps(
         data, ensure_ascii=False, sort_keys=True,
         indent=2, separators=(',', ': ')
     ).encode('utf-8') + '\n'
 
 
-def import_string_webext(data, key, source):
-    """Import a single translation from the source dictionary into data"""
-    data[key] = source
-
-
-def import_string_gecko(data, key, value):
-    """Import Gecko-style locales into data.
-
-    Only sets {'message': value} in the data-dictionary, after stripping
-    undesired Gecko-style access keys.
-    """
-    match = re.search(r'^(.*?)\s*\(&.\)$', value)
-    if match:
-        value = match.group(1)
-    else:
-        index = value.find('&')
-        if index >= 0:
-            value = value[0:index] + value[index + 1:]
-
-    data[key] = {'message': value}
+def create_bundles(params, files):
+    base_extension_path = params['baseDir']
+    info_templates = {
+        'chrome': 'chromeInfo.js.tmpl',
+        'edge': 'edgeInfo.js.tmpl',
+        'gecko': 'geckoInfo.js.tmpl'
+    }
+
+    # Historically we didn't use relative paths when requiring modules, so in
+    # order for webpack to know where to find them we need to pass in a list of
+    # resolve paths. Going forward we should always use relative paths, once we
+    # do that consistently this can be removed. See issues 5760, 5761 and 5762.
+    resolve_paths = [os.path.join(base_extension_path, dir, 'lib')
+                     for dir in ['', 'adblockpluscore', 'adblockplusui']]
+
+    info_template = getTemplate(info_templates[params['type']])
+    info_module = info_template.render(
+        basename=params['metadata'].get('general', 'basename'),
+        version=params['metadata'].get('general', 'version')
+    ).encode('utf-8')
+
+    configuration = {
+        'bundles': [],
+        'extension_path': base_extension_path,
+        'info_module': info_module,
+        'resolve_paths': resolve_paths,
+    }
+
+    for item in params['metadata'].items('bundles'):
+        name, value = item
+        base_item_path = os.path.dirname(item.source)
+
+        bundle_file = os.path.relpath(os.path.join(base_item_path, name),
+                                      base_extension_path)
+        entry_files = [os.path.join(base_item_path, module_path)
+                       for module_path in value.split()]
+        configuration['bundles'].append({
+            'bundle_name': bundle_file,
+            'entry_points': entry_files,
+        })
+
+    cmd = ['node', os.path.join(os.path.dirname(__file__), 'webpack_runner.js')]
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                               stdin=subprocess.PIPE)
+    output = process.communicate(input=toJson(configuration))[0]
+    if process.returncode != 0:
+        raise subprocess.CalledProcessError(process.returncode, cmd=cmd)
+    output = json.loads(output)
+
+    # Clear the mapping for any files included in a bundle, to avoid them being
+    # duplicated in the build.
+    for to_ignore in output['included']:
+        files.pop(to_ignore, None)
+
+    for bundle in output['files']:
+        files[bundle] = output['files'][bundle].encode('utf-8')
 
 
 def import_locales(params, files):
-    import localeTools
-
-    # FIXME: localeTools doesn't use real Chrome locales, it uses dash as
-    # separator instead.
-    convert_locale_code = lambda code: code.replace('-', '_')
-
-    # We need to map Chrome locales to Gecko locales. Start by mapping Chrome
-    # locales to themselves, merely with the dash as separator.
-    locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocales}
-
-    # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
-    for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems():
-        locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale
-
-    # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
-    reverse_mapping = {v: k for k, v in locale_mapping.iteritems()}
-    for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems():
-        if crowdin_locale in reverse_mapping:
-            locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale
-
-    for target, source in locale_mapping.iteritems():
-        targetFile = '_locales/%s/messages.json' % target
-        if not targetFile in files:
-            continue
-
-        for item in params['metadata'].items('import_locales'):
-            fileName, keys = item
-            parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
-            sourceFile = os.path.join(os.path.dirname(item.source), *parts)
-            incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
-            if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
-                continue
-
-            data = json.loads(files[targetFile].decode('utf-8'))
+    for item in params['metadata'].items('import_locales'):
+        filename, keys = item
+        for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
+                                                 *filename.split('/'))):
+            locale = sourceFile.split(os.path.sep)[-2]
+            targetFile = os.path.join('_locales', locale, 'messages.json')
+            data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
 
             try:
-                # The WebExtensions (.json) and Gecko format provide
-                # translations differently and/or provide additional
-                # information like e.g. "placeholders". We want to adhere to
-                # that and preserve the addtional info.
-                if sourceFile.endswith('.json'):
-                    with io.open(sourceFile, 'r', encoding='utf-8') as handle:
-                        sourceData = json.load(handle)
-                    import_string = import_string_webext
-                else:
-                    sourceData = localeTools.readFile(sourceFile)
-                    import_string = import_string_gecko
+                with io.open(sourceFile, 'r', encoding='utf-8') as handle:
+                    sourceData = json.load(handle)
 
                 # Resolve wildcard imports
-                if keys == '*' or keys == '=*':
+                if keys == '*':
                     importList = sourceData.keys()
                     importList = filter(lambda k: not k.startswith('_'), importList)
-                    if keys == '=*':
-                        importList = map(lambda k: '=' + k, importList)
                     keys = ' '.join(importList)
 
                 for stringID in keys.split():
-                    noMangling = False
-                    if stringID.startswith('='):
-                        stringID = stringID[1:]
-                        noMangling = True
-
                     if stringID in sourceData:
-                        if noMangling:
-                            key = re.sub(r'\W', '_', stringID)
-                        else:
-                            key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID)
-                        if key in data:
-                            print 'Warning: locale string %s defined multiple times' % key
-
-                        import_string(data, key, sourceData[stringID])
+                        if stringID in data:
+                            print ('Warning: locale string {} defined multiple'
+                                   ' times').format(stringID)
+
+                        data[stringID] = sourceData[stringID]
             except Exception as e:
                 print 'Warning: error importing locale data from %s: %s' % (sourceFile, e)
 
             files[targetFile] = toJson(data)
 
 
 def truncate(text, length_limit):
     if len(text) <= length_limit:
         return text
     return text[:length_limit - 1].rstrip() + u'\u2026'
 
 
-def fixTranslationsForCWS(files):
-    # Chrome Web Store requires messages used in manifest.json to be present in
-    # all languages. It also enforces length limits for extension names and
-    # descriptions.
+def fix_translations_for_chrome(files):
     defaults = {}
     data = json.loads(files['_locales/%s/messages.json' % defaultLocale])
     for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
         name = match.group(1)
         defaults[name] = data[name]
 
     limits = {}
     manifest = json.loads(files['manifest.json'])
     for key, limit in (('name', 45), ('description', 132), ('short_name', 12)):
         match = re.search(r'__MSG_(\S+)__', manifest.get(key, ''))
         if match:
             limits[match.group(1)] = limit
 
-    for filename in files:
-        if not filename.startswith('_locales/') or not filename.endswith('/messages.json'):
+    for path in list(files):
+        match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
+        if not match:
             continue
 
-        data = json.loads(files[filename])
-        for name, info in defaults.iteritems():
-            data.setdefault(name, info)
-        for name, limit in limits.iteritems():
-            if name in data:
-                data[name]['message'] = truncate(data[name]['message'], limit)
-        files[filename] = toJson(data)
+        # The Chrome Web Store requires messages used in manifest.json to
+        # be present in all languages, and enforces length limits on
+        # extension name and description.
+        is_latam, is_mexican, filename = match.groups()
+        if filename == 'messages.json':
+            data = json.loads(files[path])
+            for name, info in defaults.iteritems():
+                data.setdefault(name, info)
+            for name, limit in limits.iteritems():
+                info = data.get(name)
+                if info:
+                    info['message'] = truncate(info['message'], limit)
+            files[path] = toJson(data)
+
+        # Chrome combines Latin American dialects of Spanish into es-419.
+        if is_latam:
+            data = files.pop(path)
+            if is_mexican:
+                files['_locales/es_419/' + filename] = data
 
 
 def signBinary(zipdata, keyFile):
     from Crypto.Hash import SHA
     from Crypto.PublicKey import RSA
     from Crypto.Signature import PKCS1_v1_5
 
     try:
         with open(keyFile, 'rb') as file:
             key = RSA.importKey(file.read())
@@ skipping 18 matching lines @@
         file = open(outputFile, 'wb')
     else:
         file = outputFile
     if pubkey != None and signature != None:
         file.write(struct.pack('<4sIII', 'Cr24', 2, len(pubkey), len(signature)))
         file.write(pubkey)
         file.write(signature)
     file.write(zipdata)
 
 
+def add_devenv_requirements(files, metadata, params):
+    files.read(
+        os.path.join(os.path.dirname(__file__), 'chromeDevenvPoller__.js'),
+        relpath='devenvPoller__.js',
+    )
+    files['devenvVersion__'] = str(random.random())
+
+    if metadata.has_option('general', 'testScripts'):
+        files['qunit/index.html'] = createScriptPage(
+            params, 'testIndex.html.tmpl', ('general', 'testScripts')
+        )
+
+
 def createBuild(baseDir, type='chrome', outFile=None, buildNum=None, releaseBuild=False, keyFile=None, devenv=False):
     metadata = readMetadata(baseDir, type)
     version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum)
 
     if outFile == None:
-        if type == 'gecko-webext':
+        if type == 'gecko':
             file_extension = 'xpi'
         else:
             file_extension = 'crx' if keyFile else 'zip'
         outFile = getDefaultFileName(metadata, version, file_extension)
 
     params = {
         'type': type,
         'baseDir': baseDir,
         'releaseBuild': releaseBuild,
         'version': version,
         'devenv': devenv,
         'metadata': metadata,
     }
 
     mapped = metadata.items('mapping') if metadata.has_section('mapping') else []
     files = Files(getPackageFiles(params), getIgnoredFiles(params),
                   process=lambda path, data: processFile(path, data, params))
 
     files.readMappedFiles(mapped)
     files.read(baseDir, skip=[opt for opt, _ in mapped])
 
-    if metadata.has_section('convert_js'):
-        convertJS(params, files)
+    if metadata.has_section('bundles'):
+        create_bundles(params, files)
 
     if metadata.has_section('preprocess'):
         files.preprocess(
             [f for f, _ in metadata.items('preprocess')],
             {'needsExt': True}
         )
 
     if metadata.has_section('import_locales'):
         import_locales(params, files)
 
     files['manifest.json'] = createManifest(params, files)
     if type == 'chrome':
-        fixTranslationsForCWS(files)
+        fix_translations_for_chrome(files)
 
     if devenv:
-        import buildtools
-        import random
-        files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js'), relpath='devenvPoller__.js')
-        files['devenvVersion__'] = str(random.random())
-
-        if metadata.has_option('general', 'testScripts'):
-            files['qunit/index.html'] = createScriptPage(
-                params, 'testIndex.html.tmpl', ('general', 'testScripts')
-            )
+        add_devenv_requirements(files, metadata, params)
 
     zipdata = files.zipToString()
     signature = None
     pubkey = None
     if keyFile != None:
         signature = signBinary(zipdata, keyFile)
         pubkey = getPublicKey(keyFile)
     writePackage(outFile, pubkey, signature, zipdata)