Issue 6291 - add ManifoldJS packaging for Edge

packagerEdge.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import base64
-import hashlib
+import datetime
 import json
-import mimetypes
 import os
-import zipfile
+import shutil
+import subprocess
 
 import packager
 import packagerChrome
 
 # Files and directories expected inside of the .APPX archive.
-MANIFEST = 'AppxManifest.xml'
+MANIFEST = 'appxmanifest.xml'

[Sebastian Noack, 2018/07/09 12:35:20]

How come that this changed?

[tlucas, 2018/07/09 13:23:56]

manifoldjs only accepts "AppxManifest.xml" to be the results of it's packaging
process; but not to read from it. (I.e. it raises:

[error] manifoldjs  : Failed to package the Microsoft Edge Extension Platform
app.
                      The specified path does not contain a valid app manifest
file.
                      ENOENT: no such file or directory, open
'/tmp/tmpKYWE1Madblockplus_package/edgeextension/manifest/appxmanifest.xml')

 CONTENT_TYPES = '[Content_Types].xml'
 BLOCKMAP = 'AppxBlockMap.xml'
 EXTENSION_DIR = 'Extension'
 ASSETS_DIR = 'Assets'
 
 # Size of uncompressed block in the APPX block map.
 BLOCKSIZE = 64 * 1024
 
 defaultLocale = packagerChrome.defaultLocale
 
 
 def _get_template_for(filename):
     return packager.getTemplate('edge/{}.tmpl'.format(filename))
 
 
-def _lfh_size(filename):
-    """Compute the size of zip local file header for `filename`."""
-    try:
-        filename = filename.encode('utf-8')
-    except UnicodeDecodeError:
-        pass  # filename is already a byte string.
-    return zipfile.sizeFileHeader + len(filename)
-
-
-def _make_blockmap_entry(filename, data):
-    blocks = [data[i:i + BLOCKSIZE] for i in range(0, len(data), BLOCKSIZE)]
-    return {
-        'name': filename.replace('/', '\\'),
-        'size': len(data),
-        'lfh_size': _lfh_size(filename),
-        'blocks': [
-            {'hash': base64.b64encode(hashlib.sha256(block).digest())}
-            for block in blocks
-        ],
-    }
-
-
-def create_appx_blockmap(files):
-    """Create APPX blockmap for the list of files."""
-    # We don't support AppxBlockmap.xml generation for compressed zip files at
-    # the moment. The only way to reliably calculate the compressed size of
-    # each 64k chunk in the zip file is to override the relevant parts of
-    # `zipfile` library. We have chosen to not do it so we produce an
-    # uncompressed zip file that is later repackaged by Windows Store with
-    # compression.
-    template = _get_template_for(BLOCKMAP)
-    files = [_make_blockmap_entry(n, d) for n, d in files.items()]
-    return template.render(files=files).encode('utf-8')
-
-
 def load_translation(files, locale):
     """Load translation strings for locale from files."""
     path = '{}/_locales/{}/messages.json'.format(EXTENSION_DIR, locale)
     return json.loads(files[path])
 
 
 def get_appx_version(metadata, build_num):
     """Get the version number for usage in AppxManifest.xml.
 
     As required by the Windows Store, the returned version string has four
@@ skipping 42 matching lines @@
 def move_files_to_extension(files):
     """Move all files into `Extension` folder for APPX packaging."""
     # We sort the files to ensure that 'Extension/xyz' is moved before 'xyz'.
     # If 'xyz' is moved first, it would overwrite 'Extension/xyz' and its
     # original content would be lost.
     names = sorted(files.keys(), key=len, reverse=True)
     for filename in names:
         files['{}/{}'.format(EXTENSION_DIR, filename)] = files.pop(filename)
 
 
-def create_content_types_map(filenames):
-    """Create [Content_Types].xml -- a mime type map."""
-    params = {'defaults': {}, 'overrides': {}}
-    overrides = {
-        BLOCKMAP: 'application/vnd.ms-appx.blockmap+xml',
-        MANIFEST: 'application/vnd.ms-appx.manifest+xml',
-    }
-    types = mimetypes.MimeTypes()
-    types.add_type('application/octet-stream', '.otf')
-    for filename in filenames:
-        ext = os.path.splitext(filename)[1]
-        if ext:
-            content_type = types.guess_type(filename, strict=False)[0]
-            if content_type is not None:
-                params['defaults'][ext[1:]] = content_type
-        if filename in overrides:
-            params['overrides']['/' + filename] = overrides[filename]
-    content_types_template = _get_template_for(CONTENT_TYPES)
-    return content_types_template.render(params).encode('utf-8')
-
-
 def createBuild(baseDir, type='edge', outFile=None,  # noqa: preserve API.
                 buildNum=None, releaseBuild=False, keyFile=None,
                 devenv=False):
 
     metadata = packager.readMetadata(baseDir, type)
     version = packager.getBuildVersion(baseDir, metadata, releaseBuild,
                                        buildNum)
 
     outfile = outFile or packager.getDefaultFileName(metadata, version, 'appx')
 
@@ skipping 33 matching lines @@
 
     move_files_to_extension(files)
 
     if metadata.has_section('appx_assets'):
         for name, path in metadata.items('appx_assets'):
             path = os.path.join(baseDir, path)
             files.read(path, '{}/{}'.format(ASSETS_DIR, name))
 
     files[MANIFEST] = create_appx_manifest(params, files,
                                            buildNum, releaseBuild)
-    files[BLOCKMAP] = create_appx_blockmap(files)
-    files[CONTENT_TYPES] = create_content_types_map(files.keys() + [BLOCKMAP])

[Sebastian Noack, 2018/07/09 12:35:20]

Any reason you didn't remove the BLOCKMAP and CONTENT_TYPES variables as well as
the create_appx_blockmap() and create_content_types_map() functions?

[tlucas, 2018/07/09 13:23:57]

Done.

+    tmp_dir = files.to_tmp_folder('edgeextension/manifest')
+    try:
+        # mock a generationInfo.json, as i would be created by manifoldjs

[Sebastian Noack, 2018/07/09 12:35:19]

Why do we have to do that manually, now where we use manifoldjs?

[Sebastian Noack, 2018/07/09 12:35:20]

Typo: i -> it

[tlucas, 2018/07/09 13:23:57]

We don't have to.

For packaging the extension (lines 157 / 161), manifoldjs requires the
generationInfo.json to be present.

In order to generate this, we could a) invoke manifoldjs 2 times, first for
building, a.k.a copying the extension's source files around, along with some
placeholder images and then overwrite the resulting manifest, additionally copy
the icons we actually want - this is basically what we already have in our
files-object; finally start the actual packaging, or b) what i implemented here.

for that typo: done.

+        generation_info = packager.getTemplate('generationInfo.json.tmpl')
+        with open(os.path.join(tmp_dir, 'edgeextension',
+                               'generationInfo.json'),
+                  'w') as fp:
+            fp.write(generation_info.render(
+                {'generation_time': datetime.datetime.now()}))
 
-    files.zip(outfile, compression=zipfile.ZIP_STORED)
+        # package app with manifoldjs
+        cmd = ['npm', 'run', '--silent', 'package-edge']
+
+        cmd_env = os.environ.copy()
+        cmd_env['SRC_FOLDER'] = tmp_dir
+        subprocess.check_call(cmd, env=cmd_env, cwd=os.path.dirname(__file__))
+
+        package = os.path.join(tmp_dir, 'edgeextension', 'package',
+                               'edgeExtension.appx')
+
+        if isinstance(outfile, basestring):
+            shutil.copyfile(package, outfile)
+        else:
+            with open(package, 'rb') as fp:

[Sebastian Noack, 2018/07/09 12:35:20]

Where is this code path coming from? It seems previously we were assuming
outfile to always be a path to write the build to.

[tlucas, 2018/07/09 13:23:57]

The tests, or creating a devenv may pass a StringIO-object to this function (for
reference, the implementation in packagerChrome is similar:

https://hg.adblockplus.org/buildtools/file/tip/packagerChrome.py#l338 )

+                outfile.write(fp.read())
+    finally:
+        shutil.rmtree(tmp_dir, ignore_errors=True)