Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: packagerEdge.py

Issue 29345751: Issue 4028 - Add support for Edge extensions to buildtools (Closed)
Left Patch Set: Address review comments on patch set 2 Created July 1, 2016, 7:47 p.m.
Right Patch Set: Address Windows Store issues with blockmap and devbuild display name Created Oct. 14, 2016, 12:07 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « packagerChrome.py ('k') | templates/Info.plist.tmpl » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # This Source Code Form is subject to the terms of the Mozilla Public 1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this 2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
5 import base64 5 import base64
6 import hashlib 6 import hashlib
7 import json
8 import mimetypes
7 import os 9 import os
8 import zipfile 10 import zipfile
9 11
10 import packager 12 import packager
11 import packagerChrome 13 import packagerChrome
12 14
15 # Files and directories expected inside of the .APPX archive.
13 MANIFEST = 'AppxManifest.xml' 16 MANIFEST = 'AppxManifest.xml'
14 CONTENT_TYPES = '[Content_Types].xml' 17 CONTENT_TYPES = '[Content_Types].xml'
15 BLOCKMAP = 'AppxBlockMap.xml' 18 BLOCKMAP = 'AppxBlockMap.xml'
19 EXTENSION_DIR = 'Extension'
20 ASSETS_DIR = 'Assets'
21
22 # Size of uncompressed block in the APPX block map.
16 BLOCKSIZE = 64 * 1024 23 BLOCKSIZE = 64 * 1024
24
25 defaultLocale = packagerChrome.defaultLocale
17 26
18 27
19 def _get_template_for(filename): 28 def _get_template_for(filename):
20 return packager.getTemplate('edge/{}.tmpl'.format(filename)) 29 return packager.getTemplate('edge/{}.tmpl'.format(filename))
21
22
23 def _sha256(block):
24 h = hashlib.new('sha256')
Sebastian Noack 2016/07/05 14:30:37 According to the Python documentation the named co
Vasily Kuznetsov 2016/07/07 16:23:49 Changed the constructor. About one-liner: it seems
Sebastian Noack 2016/07/08 13:59:11 You can just pass the data to the constructor:
Vasily Kuznetsov 2016/07/08 16:47:39 Somehow I missed the section in the documentation
25 h.update(block)
26 return base64.b64encode(h.digest())
27 30
28 31
29 def _lfh_size(filename): 32 def _lfh_size(filename):
30 """Compute the size of zip local file header for `filename`.""" 33 """Compute the size of zip local file header for `filename`."""
31 try: 34 try:
32 filename = filename.encode('utf-8') 35 filename = filename.encode('utf-8')
33 except UnicodeDecodeError: 36 except UnicodeDecodeError:
34 pass # filename is already a byte string. 37 pass # filename is already a byte string.
35 return zipfile.sizeFileHeader + len(filename) 38 return zipfile.sizeFileHeader + len(filename)
36 39
37 40
38 def _make_blockmap_entry(filename, data): 41 def _make_blockmap_entry(filename, data):
39 blocks = [data[i:i + BLOCKSIZE] for i in range(0, len(data), BLOCKSIZE)] 42 blocks = [data[i:i + BLOCKSIZE] for i in range(0, len(data), BLOCKSIZE)]
40 return { 43 return {
41 'name': filename.replace('/', '\\'), 44 'name': filename.replace('/', '\\'),
42 'size': len(data), 45 'size': len(data),
43 'lfh_size': _lfh_size(filename), 46 'lfh_size': _lfh_size(filename),
44 'blocks': [{'hash': _sha256(block), 'compressed_size': len(block)} 47 'blocks': [
Sebastian Noack 2016/07/05 14:30:38 From the documentation it seems we can simply omit
Vasily Kuznetsov 2016/07/07 16:23:50 Done.
45 for block in blocks] 48 {'hash': base64.b64encode(hashlib.sha256(block).digest())}
49 for block in blocks
50 ]
46 } 51 }
47 52
48 53
49 def create_appx_blockmap(files): 54 def create_appx_blockmap(files):
50 """Create APPX blockmap for the list of files.""" 55 """Create APPX blockmap for the list of files."""
56 # We don't support AppxBlockmap.xml generation for compressed zip files at
57 # the moment. The only way to reliably calculate the compressed size of
58 # each 64k chunk in the zip file is to override the relevant parts of
59 # `zipfile` library. We have chosen to not do it so we produce an
60 # uncompressed zip file that is later repackaged by Windows Store with
61 # compression.
51 template = _get_template_for(BLOCKMAP) 62 template = _get_template_for(BLOCKMAP)
52 files = [_make_blockmap_entry(n, d) for n, d in files.items()] 63 files = [_make_blockmap_entry(n, d) for n, d in files.items()]
53 return template.render(files=files).encode('utf-8') 64 return template.render(files=files).encode('utf-8')
54 65
55 66
56 def create_appx_manifest(params, files): 67 def load_translation(files, locale):
68 """Load translation strings for locale from files."""
69 path = '{}/_locales/{}/messages.json'.format(EXTENSION_DIR, locale)
70 return json.loads(files[path])
71
72
73 def pad_version(version):
74 """Make sure version number has 4 groups of digits."""
75 groups = (version.split('.') + ['0', '0', '0'])[:4]
76 return '.'.join(groups)
77
78
79 def create_appx_manifest(params, files, release_build=False):
57 """Create AppxManifest.xml.""" 80 """Create AppxManifest.xml."""
58 template = _get_template_for(MANIFEST)
59 params = dict(params) 81 params = dict(params)
60 metadata = params['metadata'] 82 metadata = params['metadata']
61 params['package_identity'] = dict(metadata.items('package_identity')) 83 w = params['windows_version'] = {}
84 w['min'], w['max'] = metadata.get('compat', 'windows').split('/')
62 params.update(metadata.items('general')) 85 params.update(metadata.items('general'))
86 params['version'] = pad_version(params['version'])
87
88 translation = load_translation(files, defaultLocale)
89 name_key = 'name' if release_build else 'name_devbuild'
90 params['display_name'] = translation[name_key]['message']
91 params['description'] = translation['description']['message']
92
63 for size in ['44', '50', '150']: 93 for size in ['44', '50', '150']:
64 path = 'Assets/logo_{}.png'.format(size) 94 path = '{}/logo_{}.png'.format(ASSETS_DIR, size)
Sebastian Noack 2016/07/05 14:30:38 Where does those files come from? Don't you move e
Vasily Kuznetsov 2016/07/07 16:23:50 These are separately added after everything is mov
65 if path not in files: 95 if path not in files:
66 raise KeyError('{} is not found in files'.format(path)) 96 raise KeyError(path + 'is not found in files')
67 params['logo_{}'.format(size)] = path.replace('/', '\\') 97 params['logo_' + size] = path.replace('/', '\\')
Sebastian Noack 2016/07/05 14:30:38 As per our coding style please use the + operator
Vasily Kuznetsov 2016/07/07 16:23:51 Done.
98
99 template = _get_template_for(MANIFEST)
68 return template.render(params).encode('utf-8') 100 return template.render(params).encode('utf-8')
69 101
70 102
71 def move_files_to_extension(files): 103 def move_files_to_extension(files):
72 """Move all files into `Extension` folder for APPX packaging.""" 104 """Move all files into `Extension` folder for APPX packaging."""
73 # We sort the files to ensure that 'Extension/xyz' is moved before 'xyz'. 105 # We sort the files to ensure that 'Extension/xyz' is moved before 'xyz'.
74 # If 'xyz' is moved first, it would overwrite 'Extension/xyz' and its 106 # If 'xyz' is moved first, it would overwrite 'Extension/xyz' and its
75 # original content would be lost. 107 # original content would be lost.
76 names = sorted(files.keys(), key=len, reverse=True) 108 names = sorted(files.keys(), key=len, reverse=True)
77 for filename in names: 109 for filename in names:
78 files['Extension/' + filename] = files[filename] 110 files['{}/{}'.format(EXTENSION_DIR, filename)] = files.pop(filename)
Sebastian Noack 2016/07/05 14:30:38 If you use files.pop() this would make the next li
Vasily Kuznetsov 2016/07/07 16:23:49 Wow! I didn't realize you can `.pop()` from a dict
79 del files[filename]
80 111
81 112
82 def create_content_types_map(): 113 def create_content_types_map(filenames):
83 """Create [Content_Types].xml -- a mime type map.""" 114 """Create [Content_Types].xml -- a mime type map."""
115 params = {'defaults': {}, 'overrides': {}}
116 overrides = {
117 BLOCKMAP: 'application/vnd.ms-appx.blockmap+xml',
118 MANIFEST: 'application/vnd.ms-appx.manifest+xml'
119 }
120 for filename in filenames:
121 ext = os.path.splitext(filename)[1]
122 if ext:
123 content_type = mimetypes.guess_type(filename, strict=False)[0]
124 if content_type is not None:
125 params['defaults'][ext[1:]] = content_type
126 if filename in overrides:
127 params['overrides']['/' + filename] = overrides[filename]
84 content_types_template = _get_template_for(CONTENT_TYPES) 128 content_types_template = _get_template_for(CONTENT_TYPES)
85 return content_types_template.render().encode('utf-8') 129 return content_types_template.render(params).encode('utf-8')
86
87
88 class Files(packager.Files):
89 """Files subclass that zips without compression."""
90
91 # We don't support AppxBlockmap.xml generation for compressed zip files at
92 # the moment. The only way to reliably calculate the compressed size of
93 # each 64k chunk in the zip file is to override the relevant parts of
94 # `zipfile` library. We have chosen to not do it for now, so zip() below
95 # doesn't perform any compression.
96
97 # TODO: Replace zip() below with a compressing version:
98 # https://issues.adblockplus.org/ticket/4149
99
100 def zip(self, outFile, sortKey=None): # noqa: preserve API.
Sebastian Noack 2016/07/05 14:30:38 The code duplication here isn't great. Perhaps jus
Vasily Kuznetsov 2016/07/07 16:23:50 We'll have to redo this again when we implement ou
101 """Pack files into zip archive producing matching appx block map."""
102 zf = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_STORED)
103 for name in sorted(self, key=sortKey):
104 zf.writestr(name, self[name])
105 zf.writestr(BLOCKMAP, create_appx_blockmap(self))
106 zf.close()
107 130
108 131
109 def createBuild(baseDir, type='edge', outFile=None, # noqa: preserve API. 132 def createBuild(baseDir, type='edge', outFile=None, # noqa: preserve API.
110 buildNum=None, releaseBuild=False, keyFile=None, 133 buildNum=None, releaseBuild=False, keyFile=None,
111 devenv=False): 134 devenv=False):
112 135
113 metadata = packager.readMetadata(baseDir, type) 136 metadata = packager.readMetadata(baseDir, type)
114 version = packager.getBuildVersion(baseDir, metadata, releaseBuild, 137 version = packager.getBuildVersion(baseDir, metadata, releaseBuild,
115 buildNum) 138 buildNum)
116 139
117 outfile = outFile or packager.getDefaultFileName(metadata, version, 'appx') 140 outfile = outFile or packager.getDefaultFileName(metadata, version, 'appx')
118 141
119 params = { 142 params = {
120 'type': type, 143 'type': type,
121 'baseDir': baseDir, 144 'baseDir': baseDir,
122 'releaseBuild': releaseBuild, 145 'releaseBuild': releaseBuild,
123 'version': version, 146 'version': version,
124 'devenv': devenv, 147 'devenv': devenv,
125 'metadata': metadata, 148 'metadata': metadata,
126 } 149 }
127 150
128 files = Files(packagerChrome.getPackageFiles(params), 151 files = packager.Files(packagerChrome.getPackageFiles(params),
129 packagerChrome.getIgnoredFiles(params)) 152 packagerChrome.getIgnoredFiles(params))
130 153
131 if metadata.has_section('mapping'): 154 if metadata.has_section('mapping'):
132 mapped = metadata.items('mapping') 155 mapped = metadata.items('mapping')
133 files.readMappedFiles(mapped) 156 files.readMappedFiles(mapped)
134 files.read(baseDir, skip=[filename for filename, _ in mapped]) 157 files.read(baseDir, skip=[filename for filename, _ in mapped])
135 else: 158 else:
136 files.read(baseDir) 159 files.read(baseDir)
137 160
138 if metadata.has_section('convert_js'): 161 if metadata.has_section('convert_js'):
139 packagerChrome.convertJS(params, files) 162 packagerChrome.convertJS(params, files)
140 163
141 if metadata.has_section('preprocess'): 164 if metadata.has_section('preprocess'):
142 files.preprocess(metadata.options('preprocess'), {'needsExt': True}) 165 files.preprocess(metadata.options('preprocess'), {'needsExt': True})
143 166
144 if metadata.has_section('import_locales'): 167 if metadata.has_section('import_locales'):
145 packagerChrome.importGeckoLocales(params, files) 168 packagerChrome.importGeckoLocales(params, files)
146 169
147 files['manifest.json'] = packagerChrome.createManifest(params, files) 170 files['manifest.json'] = packagerChrome.createManifest(params, files)
148 171
149 move_files_to_extension(files) 172 move_files_to_extension(files)
150 173
151 if metadata.has_section('appx_assets'): 174 if metadata.has_section('appx_assets'):
152 for name, path in metadata.items('appx_assets'): 175 for name, path in metadata.items('appx_assets'):
153 path = os.path.join(baseDir, path) 176 path = os.path.join(baseDir, path)
154 files.read(path, 'Assets/{}'.format(name)) 177 files.read(path, '{}/{}'.format(ASSETS_DIR, name))
155 178
156 files[MANIFEST] = create_appx_manifest(params, files) 179 files[MANIFEST] = create_appx_manifest(params, files, releaseBuild)
157 files[CONTENT_TYPES] = create_content_types_map() 180 files[BLOCKMAP] = create_appx_blockmap(files)
181 files[CONTENT_TYPES] = create_content_types_map(files.keys() + [BLOCKMAP])
158 182
159 files.zip(outfile) 183 files.zip(outfile, compression=zipfile.ZIP_STORED)
LEFTRIGHT

Powered by Google App Engine
This is Rietveld