Issue 4028 - Add support for Edge extensions to buildtools

packagerEdge.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import base64
 import hashlib
+import json
+import mimetypes
 import os
 import zipfile
 
 import packager
 import packagerChrome
 
+# Files and directories expected inside of the .APPX archive.
 MANIFEST = 'AppxManifest.xml'
 CONTENT_TYPES = '[Content_Types].xml'
 BLOCKMAP = 'AppxBlockMap.xml'
-BLOCKSIZE = 65536

[Wladimir Palant, 2016/06/17 09:53:46]

Maybe more obvious if hexadecimal - 0x10000?

[Vasily Kuznetsov, 2016/06/17 17:53:26]

I'm not sure if it's actually more obvious. Because what we would like to say is
64k. So maybe just `64 * 1024` ?

+EXTENSION_DIR = 'Extension'
+ASSETS_DIR = 'Assets'
 
-# Size of the fixed part of Zip local file header.
-# See https://en.wikipedia.org/wiki/Zip_(file_format)#File_headers
-LFH_FIXED_SIZE = 30

[Wladimir Palant, 2016/06/17 09:53:46]

zipfile.sizeFileHeader?

[Vasily Kuznetsov, 2016/06/17 17:53:27]

Oh, I didn't know about this constant. It's not mentioned in the documentation,
but I guess if they didn't want me to use it, they would have put an underscore
on it.

Done.

+# Size of uncompressed block in the APPX block map.
+BLOCKSIZE = 64 * 1024
+
+defaultLocale = packagerChrome.defaultLocale
 
 
 def _get_template_for(filename):
     return packager.getTemplate('edge/{}.tmpl'.format(filename))
 
 
-def _SHA256(block):
-    h = hashlib.new('sha256')
-    h.update(block)
-    return base64.b64encode(h.digest())
+def _lfh_size(filename):
+    """Compute the size of zip local file header for `filename`."""
+    try:
+        filename = filename.encode('utf-8')
+    except UnicodeDecodeError:
+        pass  # filename is already a byte string.
+    return zipfile.sizeFileHeader + len(filename)
 
 
 def _make_blockmap_entry(filename, data):
     blocks = [data[i:i + BLOCKSIZE] for i in range(0, len(data), BLOCKSIZE)]
     return {
         'name': filename.replace('/', '\\'),
-        'size': str(len(data)),
-        'lfh_size': str(LFH_FIXED_SIZE + len(filename)),

[Wladimir Palant, 2016/06/17 09:53:46]

Assumption here: filename is ASCII. Maybe enforce that with an assert?

[Vasily Kuznetsov, 2016/06/17 17:53:26]

If it's not ASCII, it gets encoded as utf-8. I implemented this option too, just
in case -- it seems better than failing. So now we don't need an assert.

-        'blocks': [{'hash': _SHA256(block), 'compressed_size': str(len(block))}
-                   for block in blocks]

[Wladimir Palant, 2016/06/17 09:53:46]

No point stringifying all the numbers, the representation of these numbers is up
to the template.

[Vasily Kuznetsov, 2016/06/17 17:53:27]

Done.

+        'size': len(data),
+        'lfh_size': _lfh_size(filename),
+        'blocks': [
+            {'hash': base64.b64encode(hashlib.sha256(block).digest())}
+            for block in blocks
+        ]
     }
 
 
-def createAppxBlockmap(files):
+def create_appx_blockmap(files):
     """Create APPX blockmap for the list of files."""
+    # We don't support AppxBlockmap.xml generation for compressed zip files at
+    # the moment. The only way to reliably calculate the compressed size of
+    # each 64k chunk in the zip file is to override the relevant parts of
+    # `zipfile` library. We have chosen to not do it so we produce an
+    # uncompressed zip file that is later repackaged by Windows Store with
+    # compression.
     template = _get_template_for(BLOCKMAP)
     files = [_make_blockmap_entry(n, d) for n, d in files.items()]
     return template.render(files=files).encode('utf-8')
 
 
-def createAppxManifest(params):
+def load_translation(files, locale):
+    """Load translation strings for locale from files."""
+    path = '{}/_locales/{}/messages.json'.format(EXTENSION_DIR, locale)
+    return json.loads(files[path])
+
+
+def pad_version(version):
+    """Make sure version number has 4 groups of digits."""
+    groups = (version.split('.') + ['0', '0', '0'])[:4]
+    return '.'.join(groups)
+
+
+def create_appx_manifest(params, files, release_build=False):
     """Create AppxManifest.xml."""
-    template = _get_template_for(MANIFEST)
     params = dict(params)
     metadata = params['metadata']
-    params['package_identity'] = dict(metadata.items('package_identity'))
+    w = params['windows_version'] = {}
+    w['min'], w['max'] = metadata.get('compat', 'windows').split('/')
     params.update(metadata.items('general'))
+    params['version'] = pad_version(params['version'])
+
+    translation = load_translation(files, defaultLocale)
+    name_key = 'name' if release_build else 'name_devbuild'
+    params['display_name'] = translation[name_key]['message']
+    params['description'] = translation['description']['message']
+
     for size in ['44', '50', '150']:
-        params['logo_{}'.format(size)] = 'Assets\\logo_{}.png'.format(size)

[Wladimir Palant, 2016/06/17 09:53:46]

We need to verify that these files actually exist. Or maybe explicitly only
package specific asset files so that a missing logo_50.png entry under [assets]
will produce an error.

[Vasily Kuznetsov, 2016/06/17 17:53:27]

I started checking that they exist.

+        path = '{}/logo_{}.png'.format(ASSETS_DIR, size)
+        if path not in files:
+            raise KeyError(path + 'is not found in files')
+        params['logo_' + size] = path.replace('/', '\\')
+
+    template = _get_template_for(MANIFEST)
     return template.render(params).encode('utf-8')
 
 
-def convertToAppx(files):
+def move_files_to_extension(files):
     """Move all files into `Extension` folder for APPX packaging."""
+    # We sort the files to ensure that 'Extension/xyz' is moved before 'xyz'.
+    # If 'xyz' is moved first, it would overwrite 'Extension/xyz' and its
+    # original content would be lost.
     names = sorted(files.keys(), key=len, reverse=True)

[Wladimir Palant, 2016/06/17 09:53:45]

Why sort here? If the point is creating a copy of files.keys() that is an
iterator in Python 3 - that can be done easier.

[Vasily Kuznetsov, 2016/06/17 17:53:26]

Sorting is to make sure that moving 'foo' to 'Extension/foo' doesn't overwrite
an already existing 'Extension/foo' (which should be moved to
'Extension/Extension/foo'). If we start with the files that have the longest
paths, this cannot happen.

[Wladimir Palant, 2016/06/29 19:33:44]

This makes sense but it's also non-obvious. You better add a comment on it,
otherwise somebody else might remove the seemingly unnecessary sorting here
later.

[Vasily Kuznetsov, 2016/07/01 19:51:27]

Done

     for filename in names:
-        files['Extension/' + filename] = files[filename]
-        del files[filename]
-    content_types_template = _get_template_for(CONTENT_TYPES)
-    files[CONTENT_TYPES] = content_types_template.render().encode('utf-8')

[Wladimir Palant, 2016/06/17 09:53:46]

This function does two things - prefix file names and add a content types file
(but not the other XML files). I don't see how they belong together, also
"convertToAppx" doesn't really reflect the purpose here.

[Vasily Kuznetsov, 2016/06/17 17:53:27]

Yeah, makes sense. I broke it up.

+        files['{}/{}'.format(EXTENSION_DIR, filename)] = files.pop(filename)
 
 
-class Files(packager.Files):
-    """Files subclass that zips without compression."""
-
-    # We don't support AppxBlockmap.xml generation for compressed zip files at
-    # the moment. The only way to reliably calculate the compressed size of
-    # each 64k chunk in the zip file is to override the relevant parts of
-    # `zipfile` library. We have chosen to not do it for now, so zip() below
-    # doesn't perform any compression.
-
-    def zip(self, outFile, sortKey=None):

[Vasily Kuznetsov, 2016/06/13 12:57:31]

This can eventually be replaced with a compressing version:
https://issues.adblockplus.org/ticket/4149

[Wladimir Palant, 2016/06/17 09:53:46]

Maybe add a TODO comment on that, so it's obvious?

[Vasily Kuznetsov, 2016/06/17 17:53:27]

Done.

-        """Pack files into zip archive producing matching appx block map."""
-        zf = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_STORED)
-        for name in sorted(self, key=sortKey):
-            zf.writestr(name, self[name])
-        zf.writestr(BLOCKMAP, createAppxBlockmap(self))
-        zf.close()
+def create_content_types_map(filenames):
+    """Create [Content_Types].xml -- a mime type map."""
+    params = {'defaults': {}, 'overrides': {}}
+    overrides = {
+        BLOCKMAP: 'application/vnd.ms-appx.blockmap+xml',
+        MANIFEST: 'application/vnd.ms-appx.manifest+xml'
+    }
+    for filename in filenames:
+        ext = os.path.splitext(filename)[1]
+        if ext:
+            content_type = mimetypes.guess_type(filename, strict=False)[0]
+            if content_type is not None:
+                params['defaults'][ext[1:]] = content_type
+        if filename in overrides:
+            params['overrides']['/' + filename] = overrides[filename]
+    content_types_template = _get_template_for(CONTENT_TYPES)
+    return content_types_template.render(params).encode('utf-8')
 
 
-def createBuild(baseDir, type='edge', outFile=None, buildNum=None,

[Vasily Kuznetsov, 2016/06/13 12:57:31]

This function has been copied from `packagerChrome` and modified to produce edge
packages. There's in general a lot of code duplication between `createBuild`
functions in all packagers and I chose to not do anything about it for now to
keep the number of unrelated changes low.

-                releaseBuild=False, keyFile=None, devenv=False):
+def createBuild(baseDir, type='edge', outFile=None,  # noqa: preserve API.
+                buildNum=None, releaseBuild=False, keyFile=None,
+                devenv=False):
 
     metadata = packager.readMetadata(baseDir, type)
     version = packager.getBuildVersion(baseDir, metadata, releaseBuild,
                                        buildNum)
 
-    if outFile is None:
-        outFile = packager.getDefaultFileName(metadata, version, 'appx')
+    outfile = outFile or packager.getDefaultFileName(metadata, version, 'appx')
 
     params = {
         'type': type,
         'baseDir': baseDir,
         'releaseBuild': releaseBuild,
         'version': version,
         'devenv': devenv,
         'metadata': metadata,
     }
 
-    files = Files(packagerChrome.getPackageFiles(params),
-                  packagerChrome.getIgnoredFiles(params))
+    files = packager.Files(packagerChrome.getPackageFiles(params),
+                           packagerChrome.getIgnoredFiles(params))
 
     if metadata.has_section('mapping'):
         mapped = metadata.items('mapping')
         files.readMappedFiles(mapped)
         files.read(baseDir, skip=[filename for filename, _ in mapped])
     else:
         files.read(baseDir)
 
     if metadata.has_section('convert_js'):
         packagerChrome.convertJS(params, files)
 
     if metadata.has_section('preprocess'):
         files.preprocess(metadata.options('preprocess'), {'needsExt': True})
 
     if metadata.has_section('import_locales'):
         packagerChrome.importGeckoLocales(params, files)
 
     files['manifest.json'] = packagerChrome.createManifest(params, files)
 
-    convertToAppx(files)
+    move_files_to_extension(files)
 
     if metadata.has_section('appx_assets'):
         for name, path in metadata.items('appx_assets'):
             path = os.path.join(baseDir, path)
-            files.read(path, 'Assets/{}'.format(name))
+            files.read(path, '{}/{}'.format(ASSETS_DIR, name))
 
-    files[MANIFEST] = createAppxManifest(params)
-    files.zip(outFile)
+    files[MANIFEST] = create_appx_manifest(params, files, releaseBuild)
+    files[BLOCKMAP] = create_appx_blockmap(files)
+    files[CONTENT_TYPES] = create_content_types_map(files.keys() + [BLOCKMAP])
+
+    files.zip(outfile, compression=zipfile.ZIP_STORED)