Issue 6291 - add ManifoldJS packaging for Edge

packagerEdge.py

Index: packagerEdge.py
diff --git a/packagerEdge.py b/packagerEdge.py
index a86ffc45d44c532a24319eb809530a4f956f644d..ae54891d2bccf23d5580dca63c36a76aecb4276a 100644
--- a/packagerEdge.py
+++ b/packagerEdge.py
@@ -2,20 +2,19 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import base64
-import hashlib
 import json
-import mimetypes
 import os
-import zipfile
+import shutil
+from StringIO import StringIO
+import subprocess
+import tempfile
+from zipfile import ZipFile
 
 import packager
 import packagerChrome
 
 # Files and directories expected inside of the .APPX archive.
-MANIFEST = 'AppxManifest.xml'
-CONTENT_TYPES = '[Content_Types].xml'
-BLOCKMAP = 'AppxBlockMap.xml'
+MANIFEST = 'appxmanifest.xml'
 EXTENSION_DIR = 'Extension'
 ASSETS_DIR = 'Assets'
 
@@ -29,41 +28,6 @@ def _get_template_for(filename):
     return packager.getTemplate('edge/{}.tmpl'.format(filename))
 
 
-def _lfh_size(filename):
-    """Compute the size of zip local file header for `filename`."""
-    try:
-        filename = filename.encode('utf-8')
-    except UnicodeDecodeError:
-        pass  # filename is already a byte string.
-    return zipfile.sizeFileHeader + len(filename)
-
-
-def _make_blockmap_entry(filename, data):
-    blocks = [data[i:i + BLOCKSIZE] for i in range(0, len(data), BLOCKSIZE)]
-    return {
-        'name': filename.replace('/', '\\'),
-        'size': len(data),
-        'lfh_size': _lfh_size(filename),
-        'blocks': [
-            {'hash': base64.b64encode(hashlib.sha256(block).digest())}
-            for block in blocks
-        ],
-    }
-
-
-def create_appx_blockmap(files):
-    """Create APPX blockmap for the list of files."""
-    # We don't support AppxBlockmap.xml generation for compressed zip files at
-    # the moment. The only way to reliably calculate the compressed size of
-    # each 64k chunk in the zip file is to override the relevant parts of
-    # `zipfile` library. We have chosen to not do it so we produce an
-    # uncompressed zip file that is later repackaged by Windows Store with
-    # compression.
-    template = _get_template_for(BLOCKMAP)
-    files = [_make_blockmap_entry(n, d) for n, d in files.items()]
-    return template.render(files=files).encode('utf-8')
-
-
 def load_translation(files, locale):
     """Load translation strings for locale from files."""
     path = '{}/_locales/{}/messages.json'.format(EXTENSION_DIR, locale)
@@ -126,27 +90,6 @@ def move_files_to_extension(files):
         files['{}/{}'.format(EXTENSION_DIR, filename)] = files.pop(filename)
 
 
-def create_content_types_map(filenames):
-    """Create [Content_Types].xml -- a mime type map."""
-    params = {'defaults': {}, 'overrides': {}}
-    overrides = {
-        BLOCKMAP: 'application/vnd.ms-appx.blockmap+xml',
-        MANIFEST: 'application/vnd.ms-appx.manifest+xml',
-    }
-    types = mimetypes.MimeTypes()
-    types.add_type('application/octet-stream', '.otf')
-    for filename in filenames:
-        ext = os.path.splitext(filename)[1]
-        if ext:
-            content_type = types.guess_type(filename, strict=False)[0]
-            if content_type is not None:
-                params['defaults'][ext[1:]] = content_type
-        if filename in overrides:
-            params['overrides']['/' + filename] = overrides[filename]
-    content_types_template = _get_template_for(CONTENT_TYPES)
-    return content_types_template.render(params).encode('utf-8')
-
-
 def createBuild(baseDir, type='edge', outFile=None,  # noqa: preserve API.
                 buildNum=None, releaseBuild=False, keyFile=None,
                 devenv=False):
@@ -198,9 +141,64 @@ def createBuild(baseDir, type='edge', outFile=None,  # noqa: preserve API.
             path = os.path.join(baseDir, path)
             files.read(path, '{}/{}'.format(ASSETS_DIR, name))
 
-    files[MANIFEST] = create_appx_manifest(params, files,
-                                           buildNum, releaseBuild)
-    files[BLOCKMAP] = create_appx_blockmap(files)
-    files[CONTENT_TYPES] = create_content_types_map(files.keys() + [BLOCKMAP])
+    if not devenv:

[tlucas, 2018/07/11 12:18:03]

It turned out that devenvs for Edge don't need the AppxManifest.xml file to be
present, same applies to AppxBlockMap.xml and [Content_Types].xml (see changes
in test_packagerWebExt.py) @Ollie, i verified that on my own Windows-PC, i'd
appreciate another verification though.

[Oleksandr, 2018/07/11 14:23:46]

I can confirm that for devenv only the `Extension` part is used. All the other
stuff is redundant.

+        files[MANIFEST] = create_appx_manifest(params, files,

[Sebastian Noack, 2018/07/11 16:26:47]

As discussed before, I would not override the APPX manifest, but rather parse
the one generated by ManifoldJS, and merely add the fields it cannot generate
automatically. This seems to be the workflow suggested in the documentation, and
will be more robust against future changes by Microsoft.

[tlucas, 2018/07/12 09:31:55]

Done (new function update_appx_manifest)

+                                               buildNum, releaseBuild)
+
+    zipped = StringIO()
+    files.zip(zipped)
+
+    zipped.seek(0)
+
+    if devenv:
+        shutil.copyfileobj(zipped, outfile)

[Sebastian Noack, 2018/07/11 16:26:46]

Don't we have to extract the contents for the devenv?

[tlucas, 2018/07/12 09:31:54]

Yes - but that's done in build.py, with all 3 extensions packed before.

+        return
+
+    tmp_dir = tempfile.mkdtemp('adblockplus_package')
+    src_dir = os.path.join(tmp_dir, 'src')
+    ext_dir = os.path.join(tmp_dir, 'ext')
 
-    files.zip(outfile, compression=zipfile.ZIP_STORED)
+    with ZipFile(zipped, 'r') as zip_file:
+        zip_file.extractall(src_dir)
+
+    try:

[Sebastian Noack, 2018/07/11 16:26:46]

This try-block should probably start right after the call to mkdtemp().

[tlucas, 2018/07/12 09:31:54]

Done.

+        cmd_env = os.environ.copy()
+        cmd_env['SRC_FOLDER'] = src_dir
+        cmd_env['EXT_FOLDER'] = ext_dir
+
+        manifold_folder = os.path.join(ext_dir, 'MSGname', 'edgeextension')
+        manifest_folder = os.path.join(manifold_folder, 'manifest')
+        asset_folder = os.path.join(manifest_folder, 'Assets')
+
+        # prepare the extension with manifoldjs
+        cmd = ['npm', 'run', '--silent', 'build-edge']
+        subprocess.check_call(cmd, env=cmd_env, cwd=os.path.dirname(__file__))
+
+        # overwrite incomplete appxmanifest
+        intermediate_manifest = os.path.join(manifest_folder, MANIFEST)
+        shutil.copyfile(os.path.join(src_dir, MANIFEST), intermediate_manifest)
+
+        # cleanup placeholders, copy actual images
+        shutil.rmtree(asset_folder)
+        os.mkdir(asset_folder)
+        if metadata.has_section('appx_assets'):
+            for name, path in metadata.items('appx_assets'):
+                path = os.path.join(baseDir, path)
+                target = os.path.join(asset_folder, name)
+                shutil.copyfile(path, target)
+
+        # package app with manifoldjs
+        cmd = ['npm', 'run', '--silent', 'package-edge']
+
+        subprocess.check_call(cmd, env=cmd_env, cwd=os.path.dirname(__file__))
+
+        package = os.path.join(manifold_folder, 'package',
+                               'edgeExtension.appx')
+
+        if isinstance(outfile, basestring):
+            shutil.copyfile(package, outfile)
+        else:
+            with open(package, 'rb') as fp:

[Sebastian Noack, 2018/07/11 16:26:46]

This is essentially dead code now, since only in case of devenv, a file-like
object is passed in. So I wonder whether we should just assert that assumption,
rather than having dead/untested code in here?

  assert isinstance(outfile, basestring)
  shutil.copyfile(package, outfile)

[tlucas, 2018/07/12 09:31:55]

That's not an assumption, at this point 'outfile' will definitely be a string.

Removed dead code accordingly.

+                shutil.copyfileobj(fp, outfile)
+    finally:
+        shutil.rmtree(tmp_dir, ignore_errors=True)