Issue 4028 - Add support for Edge extensions to buildtools

packagerEdge.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import base64
 import hashlib
+import json
 import mimetypes
 import os
 import zipfile
 
 import packager
 import packagerChrome
 
+# Files and directories expected inside of the .APPX archive.
 MANIFEST = 'AppxManifest.xml'
 CONTENT_TYPES = '[Content_Types].xml'
 BLOCKMAP = 'AppxBlockMap.xml'
+EXTENSION_DIR = 'Extension'
+ASSETS_DIR = 'Assets'
+
+# Size of uncompressed block in the APPX block map.
 BLOCKSIZE = 64 * 1024
+
+defaultLocale = packagerChrome.defaultLocale
 
 
 def _get_template_for(filename):
     return packager.getTemplate('edge/{}.tmpl'.format(filename))
 
 
 def _lfh_size(filename):
     """Compute the size of zip local file header for `filename`."""
     try:
         filename = filename.encode('utf-8')
     except UnicodeDecodeError:
         pass  # filename is already a byte string.
     return zipfile.sizeFileHeader + len(filename)
 
 
 def _make_blockmap_entry(filename, data):
     blocks = [data[i:i + BLOCKSIZE] for i in range(0, len(data), BLOCKSIZE)]
     return {
         'name': filename.replace('/', '\\'),
         'size': len(data),
         'lfh_size': _lfh_size(filename),
         'blocks': [
             {'hash': base64.b64encode(hashlib.sha256(block).digest())}

[Sebastian Noack, 2016/07/11 16:29:45]

Wrapping the value into a dict seems unnecessary.

[Vasily Kuznetsov, 2016/07/12 09:38:09]

The dictionary might also contain the `compressed_size` key. Right now it's not
used since compression is not implemented, but since we're planning to add the
compression soon (#4149), removing the code related to compressed size from
here, template and tests seems excessive. As we've agreed over IRC, I'm leaving
this as is.

             for block in blocks
         ]
     }
 
 
 def create_appx_blockmap(files):
     """Create APPX blockmap for the list of files."""
+    # We don't support AppxBlockmap.xml generation for compressed zip files at
+    # the moment. The only way to reliably calculate the compressed size of
+    # each 64k chunk in the zip file is to override the relevant parts of
+    # `zipfile` library. We have chosen to not do it so we produce an
+    # uncompressed zip file that is later repackaged by Windows Store with
+    # compression.
     template = _get_template_for(BLOCKMAP)
     files = [_make_blockmap_entry(n, d) for n, d in files.items()]
     return template.render(files=files).encode('utf-8')
 
 
-def create_appx_manifest(params, files):
+def load_translation(files, locale):
+    """Load translation strings for locale from files."""
+    path = '{}/_locales/{}/messages.json'.format(EXTENSION_DIR, locale)
+    return json.loads(files[path])
+
+
+def pad_version(version):
+    """Make sure version number has 4 groups of digits."""
+    groups = (version.split('.') + ['0', '0', '0'])[:4]
+    return '.'.join(groups)
+
+
+def create_appx_manifest(params, files, release_build=False):
     """Create AppxManifest.xml."""
-    template = _get_template_for(MANIFEST)
     params = dict(params)
     metadata = params['metadata']
-    params['package_identity'] = dict(metadata.items('package_identity'))
     w = params['windows_version'] = {}
     w['min'], w['max'] = metadata.get('compat', 'windows').split('/')
     params.update(metadata.items('general'))
+    params['version'] = pad_version(params['version'])
+
+    translation = load_translation(files, defaultLocale)
+    name_key = 'name' if release_build else 'name_devbuild'
+    params['display_name'] = translation[name_key]['message']
+    params['description'] = translation['description']['message']
+
     for size in ['44', '50', '150']:
-        path = 'Assets/logo_{}.png'.format(size)
+        path = '{}/logo_{}.png'.format(ASSETS_DIR, size)
         if path not in files:
-            raise KeyError('{} is not found in files'.format(path))

[Sebastian Noack, 2016/07/11 16:29:45]

Use the + operator here?

[Vasily Kuznetsov, 2016/07/12 09:38:09]

Done.

+            raise KeyError(path + 'is not found in files')
         params['logo_' + size] = path.replace('/', '\\')
+
+    template = _get_template_for(MANIFEST)
     return template.render(params).encode('utf-8')
 
 
 def move_files_to_extension(files):
     """Move all files into `Extension` folder for APPX packaging."""
     # We sort the files to ensure that 'Extension/xyz' is moved before 'xyz'.
     # If 'xyz' is moved first, it would overwrite 'Extension/xyz' and its
     # original content would be lost.
     names = sorted(files.keys(), key=len, reverse=True)
     for filename in names:
-        files['Extension/' + filename] = files.pop(filename)
+        files['{}/{}'.format(EXTENSION_DIR, filename)] = files.pop(filename)
 
 
-def create_content_types_map(files):
+def create_content_types_map(filenames):
     """Create [Content_Types].xml -- a mime type map."""
-    # We always have a manifest and a blockmap, so we include those by default.
-    params = {
-        'defaults': {
-            'xml': 'text/xml'

[Sebastian Noack, 2016/07/11 16:29:44]

Isn't that overridden below?

[Vasily Kuznetsov, 2016/07/12 09:38:08]

Indeed it is, but since I'm not sure how the content map is interpreted, I was
thinking that it would be safer to have a default here too.

[Sebastian Noack, 2016/07/12 21:15:25]

I mean, isn't the value here overridden by the logic below, since there is
always an *.xml file in the list of files? And in the theoretical scenario,
there wouldn't be any *.xml file we wouldn't need to add that default anyway.

[Vasily Kuznetsov, 2016/07/13 10:35:31]

My thinking was that we are normally adding the content-types file before the
blockmap but it needs to have a record for blockmap. So if there are no other
XML files, the default for XML will not be there and it might not be processed
right.

Anyway, this assumption that the blockmap will be added later is not a good
thing. I rewrote this function to make things more explicit. See how you like it
now.

[Sebastian Noack, 2016/07/13 16:29:22]

Do we even need to consider AppxBlockMap.xml and AppxManifest.xml for the
defaults, since they have their mimetype explicitly specified below? I guess
not. So the overrides could be hard-coded in the template and defaults can be
initialized with an empty dict, significantly simplifying this implementation.

[Vasily Kuznetsov, 2016/07/13 17:36:45]

Yeah, this is possible, but we'll later have other overrides, for example for
the files related to signing the package. I guess we can always hardcode all
overrides but I mildly dislike that in this case we'd have half of mime type
information in the template and the other half computed in the code.

[Sebastian Noack, 2016/07/13 17:42:39]

The package is signed by the Windows Store. In fact, we have to upload an
unsigned package. So I guess we can ignore this theoretical scenario. Having
dynamic logic in code and static content in the template sounds reasonable to
me.

-        },
-        'overrides': {
-            '/AppxBlockMap.xml': 'application/vnd.ms-appx.blockmap+xml',
-            '/AppxManifest.xml': 'application/vnd.ms-appx.manifest+xml'
-        }
+    params = {'defaults': {}, 'overrides': {}}
+    overrides = {
+        BLOCKMAP: 'application/vnd.ms-appx.blockmap+xml',
+        MANIFEST: 'application/vnd.ms-appx.manifest+xml'
     }
-    for filename in files:
-        if '.' in filename[1:]:
-            ext = filename.split('.')[-1]

[Sebastian Noack, 2016/07/11 16:29:45]

Use os.path.splitext?

[Vasily Kuznetsov, 2016/07/12 09:38:10]

Done.

-            content_type = mimetypes.guess_type(filename, strict=False)
-            # The return value can be a string or a tuple:

[Sebastian Noack, 2016/07/11 16:29:44]

Can it? I think it is always a tuple, or do I miss something in the docs?

[Vasily Kuznetsov, 2016/07/12 09:38:09]

"The return value is a tuple (type, encoding) where type is None if the type
can’t be guessed (missing or unknown suffix) or a string of the form
'type/subtype', usable for a MIME content-type header."

Actually you're right, it makes much more sense to interpret the "or" as
belonging to "where type is None...." clause rather than to "is a tuple ..."
clause. I was really annoyed at this ridiculous API but actually it's my reading
comprehension that's the problem.

Fixed.

-            # https://docs.python.org/2/library/mimetypes.html#mimetypes.guess_type
-            if isinstance(content_type, tuple):
-                content_type = content_type[0]
+    for filename in filenames:
+        ext = os.path.splitext(filename)[1]
+        if ext:
+            content_type = mimetypes.guess_type(filename, strict=False)[0]
             if content_type is not None:
-                params['defaults'][ext] = content_type
+                params['defaults'][ext[1:]] = content_type
+        if filename in overrides:
+            params['overrides']['/' + filename] = overrides[filename]
     content_types_template = _get_template_for(CONTENT_TYPES)
     return content_types_template.render(params).encode('utf-8')
 
 
 def createBuild(baseDir, type='edge', outFile=None,  # noqa: preserve API.
                 buildNum=None, releaseBuild=False, keyFile=None,
                 devenv=False):
 
     metadata = packager.readMetadata(baseDir, type)
     version = packager.getBuildVersion(baseDir, metadata, releaseBuild,
@@ skipping 29 matching lines @@
     if metadata.has_section('import_locales'):
         packagerChrome.importGeckoLocales(params, files)
 
     files['manifest.json'] = packagerChrome.createManifest(params, files)
 
     move_files_to_extension(files)
 
     if metadata.has_section('appx_assets'):
         for name, path in metadata.items('appx_assets'):
             path = os.path.join(baseDir, path)
-            files.read(path, 'Assets/' + name)
+            files.read(path, '{}/{}'.format(ASSETS_DIR, name))
 
-    files[MANIFEST] = create_appx_manifest(params, files)
-    files[CONTENT_TYPES] = create_content_types_map(files)
+    files[MANIFEST] = create_appx_manifest(params, files, releaseBuild)
+    files[BLOCKMAP] = create_appx_blockmap(files)
+    files[CONTENT_TYPES] = create_content_types_map(files.keys() + [BLOCKMAP])
 
-    # We don't support AppxBlockmap.xml generation for compressed zip files at
-    # the moment. The only way to reliably calculate the compressed size of
-    # each 64k chunk in the zip file is to override the relevant parts of
-    # `zipfile` library. We have chosen to not do it for now, so we produce
-    # an uncompressed zip file.
-    files[BLOCKMAP] = create_appx_blockmap(files)
-
-    # TODO: Implement AppxBlockmap.xml generation for compressed zip files.
-    # https://issues.adblockplus.org/ticket/4149
-    files.zip(outfile, compress=False)
+    files.zip(outfile, compression=zipfile.ZIP_STORED)