Issue 4028 - Add support for Edge extensions to buildtools

packagerEdge.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import base64
+import hashlib
+import mimetypes
+import os
+import zipfile
+
+import packager
+import packagerChrome
+
+MANIFEST = 'AppxManifest.xml'
+CONTENT_TYPES = '[Content_Types].xml'
+BLOCKMAP = 'AppxBlockMap.xml'
+BLOCKSIZE = 64 * 1024
+
+
+def _get_template_for(filename):
+    return packager.getTemplate('edge/{}.tmpl'.format(filename))
+
+
+def _lfh_size(filename):
+    """Compute the size of zip local file header for `filename`."""
+    try:
+        filename = filename.encode('utf-8')
+    except UnicodeDecodeError:
+        pass  # filename is already a byte string.
+    return zipfile.sizeFileHeader + len(filename)
+
+
+def _make_blockmap_entry(filename, data):
+    blocks = [data[i:i + BLOCKSIZE] for i in range(0, len(data), BLOCKSIZE)]
+    return {
+        'name': filename.replace('/', '\\'),
+        'size': len(data),
+        'lfh_size': _lfh_size(filename),
+        'blocks': [
+            {'hash': base64.b64encode(hashlib.sha256(block).digest())}

[Sebastian Noack, 2016/07/11 16:29:45]

Wrapping the value into a dict seems unnecessary.

[Vasily Kuznetsov, 2016/07/12 09:38:09]

The dictionary might also contain the `compressed_size` key. Right now it's not
used since compression is not implemented, but since we're planning to add the
compression soon (#4149), removing the code related to compressed size from
here, template and tests seems excessive. As we've agreed over IRC, I'm leaving
this as is.

+            for block in blocks
+        ]
+    }
+
+
+def create_appx_blockmap(files):
+    """Create APPX blockmap for the list of files."""
+    template = _get_template_for(BLOCKMAP)
+    files = [_make_blockmap_entry(n, d) for n, d in files.items()]
+    return template.render(files=files).encode('utf-8')
+
+
+def create_appx_manifest(params, files):
+    """Create AppxManifest.xml."""
+    template = _get_template_for(MANIFEST)
+    params = dict(params)
+    metadata = params['metadata']
+    params['package_identity'] = dict(metadata.items('package_identity'))
+    w = params['windows_version'] = {}
+    w['min'], w['max'] = metadata.get('compat', 'windows').split('/')
+    params.update(metadata.items('general'))
+    for size in ['44', '50', '150']:
+        path = 'Assets/logo_{}.png'.format(size)
+        if path not in files:
+            raise KeyError('{} is not found in files'.format(path))

[Sebastian Noack, 2016/07/11 16:29:45]

Use the + operator here?

[Vasily Kuznetsov, 2016/07/12 09:38:09]

Done.

+        params['logo_' + size] = path.replace('/', '\\')
+    return template.render(params).encode('utf-8')
+
+
+def move_files_to_extension(files):
+    """Move all files into `Extension` folder for APPX packaging."""
+    # We sort the files to ensure that 'Extension/xyz' is moved before 'xyz'.
+    # If 'xyz' is moved first, it would overwrite 'Extension/xyz' and its
+    # original content would be lost.
+    names = sorted(files.keys(), key=len, reverse=True)
+    for filename in names:
+        files['Extension/' + filename] = files.pop(filename)
+
+
+def create_content_types_map(files):
+    """Create [Content_Types].xml -- a mime type map."""
+    # We always have a manifest and a blockmap, so we include those by default.
+    params = {
+        'defaults': {
+            'xml': 'text/xml'

[Sebastian Noack, 2016/07/11 16:29:44]

Isn't that overridden below?

[Vasily Kuznetsov, 2016/07/12 09:38:08]

Indeed it is, but since I'm not sure how the content map is interpreted, I was
thinking that it would be safer to have a default here too.

[Sebastian Noack, 2016/07/12 21:15:25]

I mean, isn't the value here overridden by the logic below, since there is
always an *.xml file in the list of files? And in the theoretical scenario,
there wouldn't be any *.xml file we wouldn't need to add that default anyway.

[Vasily Kuznetsov, 2016/07/13 10:35:31]

My thinking was that we are normally adding the content-types file before the
blockmap but it needs to have a record for blockmap. So if there are no other
XML files, the default for XML will not be there and it might not be processed
right.

Anyway, this assumption that the blockmap will be added later is not a good
thing. I rewrote this function to make things more explicit. See how you like it
now.

[Sebastian Noack, 2016/07/13 16:29:22]

Do we even need to consider AppxBlockMap.xml and AppxManifest.xml for the
defaults, since they have their mimetype explicitly specified below? I guess
not. So the overrides could be hard-coded in the template and defaults can be
initialized with an empty dict, significantly simplifying this implementation.

[Vasily Kuznetsov, 2016/07/13 17:36:45]

Yeah, this is possible, but we'll later have other overrides, for example for
the files related to signing the package. I guess we can always hardcode all
overrides but I mildly dislike that in this case we'd have half of mime type
information in the template and the other half computed in the code.

[Sebastian Noack, 2016/07/13 17:42:39]

The package is signed by the Windows Store. In fact, we have to upload an
unsigned package. So I guess we can ignore this theoretical scenario. Having
dynamic logic in code and static content in the template sounds reasonable to
me.

+        },
+        'overrides': {
+            '/AppxBlockMap.xml': 'application/vnd.ms-appx.blockmap+xml',
+            '/AppxManifest.xml': 'application/vnd.ms-appx.manifest+xml'
+        }
+    }
+    for filename in files:
+        if '.' in filename[1:]:
+            ext = filename.split('.')[-1]

[Sebastian Noack, 2016/07/11 16:29:45]

Use os.path.splitext?

[Vasily Kuznetsov, 2016/07/12 09:38:10]

Done.

+            content_type = mimetypes.guess_type(filename, strict=False)
+            # The return value can be a string or a tuple:

[Sebastian Noack, 2016/07/11 16:29:44]

Can it? I think it is always a tuple, or do I miss something in the docs?

[Vasily Kuznetsov, 2016/07/12 09:38:09]

"The return value is a tuple (type, encoding) where type is None if the type
can’t be guessed (missing or unknown suffix) or a string of the form
'type/subtype', usable for a MIME content-type header."

Actually you're right, it makes much more sense to interpret the "or" as
belonging to "where type is None...." clause rather than to "is a tuple ..."
clause. I was really annoyed at this ridiculous API but actually it's my reading
comprehension that's the problem.

Fixed.

+            # https://docs.python.org/2/library/mimetypes.html#mimetypes.guess_type
+            if isinstance(content_type, tuple):
+                content_type = content_type[0]
+            if content_type is not None:
+                params['defaults'][ext] = content_type
+    content_types_template = _get_template_for(CONTENT_TYPES)
+    return content_types_template.render(params).encode('utf-8')
+
+
+def createBuild(baseDir, type='edge', outFile=None,  # noqa: preserve API.
+                buildNum=None, releaseBuild=False, keyFile=None,
+                devenv=False):
+
+    metadata = packager.readMetadata(baseDir, type)
+    version = packager.getBuildVersion(baseDir, metadata, releaseBuild,
+                                       buildNum)
+
+    outfile = outFile or packager.getDefaultFileName(metadata, version, 'appx')
+
+    params = {
+        'type': type,
+        'baseDir': baseDir,
+        'releaseBuild': releaseBuild,
+        'version': version,
+        'devenv': devenv,
+        'metadata': metadata,
+    }
+
+    files = packager.Files(packagerChrome.getPackageFiles(params),
+                           packagerChrome.getIgnoredFiles(params))
+
+    if metadata.has_section('mapping'):
+        mapped = metadata.items('mapping')
+        files.readMappedFiles(mapped)
+        files.read(baseDir, skip=[filename for filename, _ in mapped])
+    else:
+        files.read(baseDir)
+
+    if metadata.has_section('convert_js'):
+        packagerChrome.convertJS(params, files)
+
+    if metadata.has_section('preprocess'):
+        files.preprocess(metadata.options('preprocess'), {'needsExt': True})
+
+    if metadata.has_section('import_locales'):
+        packagerChrome.importGeckoLocales(params, files)
+
+    files['manifest.json'] = packagerChrome.createManifest(params, files)
+
+    move_files_to_extension(files)
+
+    if metadata.has_section('appx_assets'):
+        for name, path in metadata.items('appx_assets'):
+            path = os.path.join(baseDir, path)
+            files.read(path, 'Assets/' + name)
+
+    files[MANIFEST] = create_appx_manifest(params, files)
+    files[CONTENT_TYPES] = create_content_types_map(files)
+
+    # We don't support AppxBlockmap.xml generation for compressed zip files at
+    # the moment. The only way to reliably calculate the compressed size of
+    # each 64k chunk in the zip file is to override the relevant parts of
+    # `zipfile` library. We have chosen to not do it for now, so we produce
+    # an uncompressed zip file.
+    files[BLOCKMAP] = create_appx_blockmap(files)
+
+    # TODO: Implement AppxBlockmap.xml generation for compressed zip files.
+    # https://issues.adblockplus.org/ticket/4149
+    files.zip(outfile, compress=False)