Issue 6291 - add ManifoldJS packaging for Edge

packagerEdge.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import base64
-import hashlib
-import json
-import mimetypes
 import os
-import zipfile
+import shutil
+from StringIO import StringIO
+import subprocess
+import tempfile
+from xml.etree import ElementTree
+from zipfile import ZipFile
 
 import packager
 import packagerChrome
 
-# Files and directories expected inside of the .APPX archive.
-MANIFEST = 'AppxManifest.xml'
-CONTENT_TYPES = '[Content_Types].xml'
-BLOCKMAP = 'AppxBlockMap.xml'
-EXTENSION_DIR = 'Extension'
+MANIFEST = 'appxmanifest.xml'
 ASSETS_DIR = 'Assets'
 
-# Size of uncompressed block in the APPX block map.
-BLOCKSIZE = 64 * 1024
-
 defaultLocale = packagerChrome.defaultLocale
 
 
 def _get_template_for(filename):
     return packager.getTemplate('edge/{}.tmpl'.format(filename))
 
 
-def _lfh_size(filename):
-    """Compute the size of zip local file header for `filename`."""
-    try:
-        filename = filename.encode('utf-8')
-    except UnicodeDecodeError:
-        pass  # filename is already a byte string.
-    return zipfile.sizeFileHeader + len(filename)
+def register_xml_namespaces(manifest_path):
+    """Register namespaces of the given file, in order to preserve defaults."""
+    with open(manifest_path, 'r') as fp:
+        ns = dict([node for _, node in ElementTree.iterparse(
+            fp, events=['start-ns'])])
+    for prefix, uri in ns.items():
+        ElementTree.register_namespace(prefix, uri)
+
+    return ns
 
 
-def _make_blockmap_entry(filename, data):
-    blocks = [data[i:i + BLOCKSIZE] for i in range(0, len(data), BLOCKSIZE)]
-    return {
-        'name': filename.replace('/', '\\'),
-        'size': len(data),
-        'lfh_size': _lfh_size(filename),
-        'blocks': [
-            {'hash': base64.b64encode(hashlib.sha256(block).digest())}
-            for block in blocks
-        ],
+def update_appx_manifest(manifest_path, base_dir, files, metadata,
+                         release_build):
+    namespaces = register_xml_namespaces(manifest_path)
+
+    def traverse(current_elem, overwrite):
+        if isinstance(overwrite, dict):
+            for key, value in overwrite.items():
+                if isinstance(key, tuple):
+                    prefix, element = key
+                    next_elem = current_elem.find(
+                        '{{{}}}{}'.format(namespaces[prefix], element))
+                    traverse(next_elem, value)
+                else:
+                    current_elem.attrib.update(overwrite)
+        else:
+            current_elem.text = overwrite
+
+    v_min, v_max = metadata.get('compat', 'windows').split('/')
+
+    filenames = []
+    if metadata.has_section('appx_assets'):

[Sebastian Noack, 2018/07/25 19:18:41]

This check seems redundant as we fail anyway below where we try to access
assets[...] if the appx_assets section doesn't exist.

[tlucas, 2018/08/08 09:35:55]

Done.

+        for name, path in metadata.items('appx_assets'):
+            path = os.path.join(base_dir, path)
+            icon_path = '{}/{}'.format(ASSETS_DIR, name)
+
+            files.read(path, icon_path)
+            filenames.append(icon_path)
+
+    assets = packagerChrome.makeIcons(files, filenames)
+
+    overwrite = {
+        ('', 'Identity'): {
+            'Name': packager.get_app_id(release_build, metadata),
+            'Publisher': metadata.get('general', 'publisher_id'),
+        },
+        ('', 'Properties'): {
+            ('', 'PublisherDisplayName'): metadata.get('general', 'author'),
+            ('', 'Logo'): assets[50],
+        },
+        ('', 'Dependencies'): {
+            ('', 'TargetDeviceFamily'): {
+                'MaxVersionTested': v_max,
+                'MinVersion': v_min,
+            },
+        },
+        ('', 'Applications'): {
+            ('', 'Application'): {
+                ('uap', 'VisualElements'): {
+                    'Square150x150Logo': assets[150],
+                    'Square44x44Logo': assets[44],
+                },
+            },
+        },
     }
 
+    tree = ElementTree.parse(manifest_path)
+    root = tree.getroot()
 
-def create_appx_blockmap(files):
-    """Create APPX blockmap for the list of files."""
-    # We don't support AppxBlockmap.xml generation for compressed zip files at
-    # the moment. The only way to reliably calculate the compressed size of
-    # each 64k chunk in the zip file is to override the relevant parts of
-    # `zipfile` library. We have chosen to not do it so we produce an
-    # uncompressed zip file that is later repackaged by Windows Store with
-    # compression.
-    template = _get_template_for(BLOCKMAP)
-    files = [_make_blockmap_entry(n, d) for n, d in files.items()]
-    return template.render(files=files).encode('utf-8')
+    traverse(root, overwrite)
 
-
-def load_translation(files, locale):
-    """Load translation strings for locale from files."""
-    path = '{}/_locales/{}/messages.json'.format(EXTENSION_DIR, locale)
-    return json.loads(files[path])
-
-
-def get_appx_version(metadata, build_num):
-    """Get the version number for usage in AppxManifest.xml.
-
-    As required by the Windows Store, the returned version string has four
-    components, where the 3rd component is replaced with the build number
-    if available, and the 4th component is always zero (e.g. 1.2.1000.0).
-    """
-    components = metadata.get('general', 'version').split('.')[:3]
-    components.extend(['0'] * (4 - len(components)))
-    if build_num:
-        components[2] = build_num
-    return '.'.join(components)
-
-
-def create_appx_manifest(params, files, build_num, release_build):
-    """Create AppxManifest.xml."""
-    params = dict(params)
-    metadata = params['metadata']
-    w = params['windows_version'] = {}
-    w['min'], w['max'] = metadata.get('compat', 'windows').split('/')
-    params['version'] = get_appx_version(metadata, build_num)
-
-    metadata_suffix = 'release' if release_build else 'devbuild'
-    app_extension_id = 'extension_id_' + metadata_suffix
-    if metadata.has_option('general', app_extension_id):
-        params['app_extension_id'] = metadata.get('general', app_extension_id)
-    else:
-        params['app_extension_id'] = 'EdgeExtension'
-
-    params['app_id'] = packager.get_app_id(release_build, metadata)
-
-    translation = load_translation(files, defaultLocale)
-    name_key = 'name' if release_build else 'name_devbuild'
-    params['display_name'] = translation[name_key]['message']
-    params['description'] = translation['description']['message']
-
-    for size in ['44', '50', '150']:
-        path = '{}/logo_{}.png'.format(ASSETS_DIR, size)
-        if path not in files:
-            raise KeyError(path + ' is not found in files')
-        params['logo_' + size] = path.replace('/', '\\')
-
-    template = _get_template_for(MANIFEST)
-    return template.render(params).encode('utf-8')
-
-
-def move_files_to_extension(files):
-    """Move all files into `Extension` folder for APPX packaging."""
-    # We sort the files to ensure that 'Extension/xyz' is moved before 'xyz'.
-    # If 'xyz' is moved first, it would overwrite 'Extension/xyz' and its
-    # original content would be lost.
-    names = sorted(files.keys(), key=len, reverse=True)
-    for filename in names:
-        files['{}/{}'.format(EXTENSION_DIR, filename)] = files.pop(filename)
-
-
-def create_content_types_map(filenames):
-    """Create [Content_Types].xml -- a mime type map."""
-    params = {'defaults': {}, 'overrides': {}}
-    overrides = {
-        BLOCKMAP: 'application/vnd.ms-appx.blockmap+xml',
-        MANIFEST: 'application/vnd.ms-appx.manifest+xml',
-    }
-    types = mimetypes.MimeTypes()
-    types.add_type('application/octet-stream', '.otf')
-    for filename in filenames:
-        ext = os.path.splitext(filename)[1]
-        if ext:
-            content_type = types.guess_type(filename, strict=False)[0]
-            if content_type is not None:
-                params['defaults'][ext[1:]] = content_type
-        if filename in overrides:
-            params['overrides']['/' + filename] = overrides[filename]
-    content_types_template = _get_template_for(CONTENT_TYPES)
-    return content_types_template.render(params).encode('utf-8')
+    tree.write(manifest_path, encoding='utf-8', xml_declaration=True)
 
 
 def createBuild(baseDir, type='edge', outFile=None,  # noqa: preserve API.
                 buildNum=None, releaseBuild=False, keyFile=None,
                 devenv=False):
 
     metadata = packager.readMetadata(baseDir, type)
     version = packager.getBuildVersion(baseDir, metadata, releaseBuild,
                                        buildNum)
 
@@ skipping 26 matching lines @@
         files.preprocess(metadata.options('preprocess'), {'needsExt': True})
 
     if metadata.has_section('import_locales'):
         packagerChrome.import_locales(params, files)
 
     files['manifest.json'] = packagerChrome.createManifest(params, files)
 
     if devenv:
         packagerChrome.add_devenv_requirements(files, metadata, params)
 
-    move_files_to_extension(files)
+    zipped = StringIO()
+    files.zip(zipped)
 
-    if metadata.has_section('appx_assets'):
-        for name, path in metadata.items('appx_assets'):
-            path = os.path.join(baseDir, path)
-            files.read(path, '{}/{}'.format(ASSETS_DIR, name))
+    zipped.seek(0)
 
-    files[MANIFEST] = create_appx_manifest(params, files,
-                                           buildNum, releaseBuild)
-    files[BLOCKMAP] = create_appx_blockmap(files)
-    files[CONTENT_TYPES] = create_content_types_map(files.keys() + [BLOCKMAP])
+    if devenv:
+        shutil.copyfileobj(zipped, outfile)
+        return
 
-    files.zip(outfile, compression=zipfile.ZIP_STORED)
+    tmp_dir = tempfile.mkdtemp('adblockplus_package')
+    try:
+        src_dir = os.path.join(tmp_dir, 'src')
+        ext_dir = os.path.join(tmp_dir, 'ext')
+
+        with ZipFile(zipped, 'r') as zip_file:
+            zip_file.extractall(src_dir)
+
+        cmd_env = os.environ.copy()
+        cmd_env['SRC_FOLDER'] = src_dir
+        cmd_env['EXT_FOLDER'] = ext_dir
+
+        manifold_folder = os.path.join(ext_dir, 'MSGname', 'edgeextension')
+        manifest_folder = os.path.join(manifold_folder, 'manifest')
+        asset_folder = os.path.join(manifest_folder, ASSETS_DIR)
+
+        # prepare the extension with manifoldjs
+        cmd = ['npm', 'run', '--silent', 'build-edge']
+        subprocess.check_call(cmd, env=cmd_env, cwd=os.path.dirname(__file__))
+
+        # update incomplete appxmanifest
+        intermediate_manifest = os.path.join(manifest_folder, MANIFEST)
+        update_appx_manifest(intermediate_manifest, baseDir, files, metadata,
+                             releaseBuild)
+
+        # cleanup placeholders, copy actual images
+        shutil.rmtree(asset_folder)
+        os.mkdir(asset_folder)
+        if metadata.has_section('appx_assets'):
+            for name, path in metadata.items('appx_assets'):
+                path = os.path.join(baseDir, path)
+                target = os.path.join(asset_folder, name)
+                shutil.copyfile(path, target)
+
+        # package app with manifoldjs
+        cmd = ['npm', 'run', '--silent', 'package-edge']
+
+        subprocess.check_call(cmd, env=cmd_env, cwd=os.path.dirname(__file__))
+
+        package = os.path.join(manifold_folder, 'package',
+                               'edgeExtension.appx')
+
+        shutil.copyfile(package, outfile)
+    finally:
+        shutil.rmtree(tmp_dir, ignore_errors=True)