Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: tests/test_packagerWebExt.py

Issue 29501558: Issue 5383 - Add tests for the Chrome and Firefox packagers (Closed)
Patch Set: Rebasing against current master (502:7e896c368056) Created Oct. 17, 2017, 12:39 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tests/test_packagerEdge.py ('k') | tox.ini » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 import os
6 import shutil
7 import zipfile
8 import json
9 import re
10 from struct import unpack
11 import difflib
12
13 import pytest
14 from Crypto.Hash import SHA
15 from Crypto.PublicKey import RSA
16 from Crypto.Signature import PKCS1_v1_5
17 from xml.etree import ElementTree
18
19
20 from buildtools import packager
21 from buildtools.packagerChrome import defaultLocale
22 from buildtools.build import processArgs
23
24 LOCALES_MODULE = {
25 'test.Foobar': {
26 'message': 'Ensuring dict-copy from modules for $domain$',
27 'description': 'test description',
28 'placeholders': {'content': '$1', 'example': 'www.adblockplus.org'}
29 }
30 }
31
32 ALL_LANGUAGES = ['en_US', 'de', 'it']
33
34 MESSAGES_EN_US = json.dumps({
35 'name': {'message': 'Adblock Plus'},
36 'name_devbuild': {'message': 'devbuild-marker'},
37 'description': {
38 'message': 'Adblock Plus is the most popular ad blocker ever, '
39 'and also supports websites by not blocking '
40 'unobstrusive ads by default (configurable).'
41 },
42 })
43
44
45 class Content(object):
46 """Base class for a unified ZipFile / Directory interface.
47
48 Base class for providing a unified context manager interface for
49 accessing files. This class is subclassed by ZipContent and DirContent,
50 which provide the additional methods "namelist()" and "read(path)".
51 """
52
53 def __enter__(self):
54 return self
55
56 def __exit__(self, exc_type, exc_value, exc_tb):
57 self._close()
58
59
60 class ZipContent(Content):
61 """Provide a unified context manager for ZipFile access.
62
63 Inherits the context manager API from Content.
64 If desired, the specified ZipFile is deleted on exiting the manager.
65 """
66
67 def __init__(self, zip_path, delete_on_close=True):
68 """Construct ZipContent object handling the file <zip_path>.
69
70 The parameter 'delete_on_close' causes the context manager to
71 delete the handled ZipFile (specified by zip_path) if set to
72 True (default).
73 """
74 self._zip_path = zip_path
75 self._zip_file = zipfile.ZipFile(zip_path)
76 self._delete_on_close = delete_on_close
77 super(ZipContent, self).__init__()
78
79 def _close(self):
80 self._zip_file.close()
81 if self._delete_on_close:
82 # if specified, delete the handled file
83 os.remove(self._zip_path)
84
85 def namelist(self):
86 return self._zip_file.namelist()
87
88 def read(self, path):
89 return self._zip_file.read(path)
90
91
92 class DirContent(Content):
93 """Provides a unified context manager for directory access.
94
95 Inherits the context managet API from Content.
96 """
97
98 def __init__(self, path):
99 """Construct a DirContent object handling <path>."""
100 self._path = path
101 super(DirContent, self).__init__()
102
103 def _close(self):
104 pass
105
106 def namelist(self):
107 """Generate a list of filenames."""
108 result = []
109 for parent, directories, files in os.walk(self._path):
110 for filename in files:
111 file_path = os.path.join(parent, filename)
112 rel_path = os.path.relpath(file_path, self._path)
113 result.append(rel_path)
114 return result
115
116 def read(self, path):
117 content = None
118 with open(os.path.join(self._path, path)) as fp:
119 content = fp.read()
120 return content
121
122
123 def copy_metadata(filename, tmpdir):
124 """Copy the used metadata to the used temporary directory."""
125 path = os.path.join(os.path.dirname(__file__), filename)
126 destination = str(tmpdir.join(filename))
127 shutil.copy(path, destination)
128
129
130 def run_webext_build(ext_type, build_opt, srcdir, buildnum=None, keyfile=None):
131 """Run a build process."""
132 if build_opt == 'build':
133 build_args = ['build']
134 elif build_opt == 'release':
135 build_args = ['build', '-r']
136 else:
137 build_args = ['devenv']
138
139 args = ['build.py', '-t', ext_type] + build_args
140
141 if keyfile:
142 args += ['-k', keyfile]
143 if buildnum:
144 args += ['-b', buildnum]
145
146 processArgs(str(srcdir), args)
147
148
149 def locale_files(languages, rootfolder, srcdir):
150 """Generate example locales.
151
152 languages: tuple, list or set of languages to include
153 rootdir: folder-name to create the locale-folders in
154 """
155 for lang in languages:
156 subfolders = rootfolder.split(os.pathsep) + [lang, 'messages.json']
157 json_file = srcdir.ensure(*subfolders)
158 if lang == defaultLocale:
159 json_file.write(MESSAGES_EN_US)
160 else:
161 json_file.write('{}')
162
163
164 def assert_all_locales_present(package, prefix):
165 names = {x for x in package.namelist() if
166 x.startswith(os.path.join(prefix, '_locales'))}
167 assert names == {
168 os.path.join(prefix, '_locales', lang, 'messages.json')
169 for lang in ALL_LANGUAGES
170 }
171
172
173 @pytest.fixture
174 def srcdir(tmpdir):
175 """Source directory for building the package."""
176 for size in ['44', '50', '150']:
177 path = tmpdir.join('chrome', 'icons', 'abp-{}.png'.format(size))
178 path.write(size, ensure=True)
179
180 tmpdir.join('bar.json').write(json.dumps({}))
181 return tmpdir
182
183
184 @pytest.fixture
185 def locale_modules(tmpdir):
186 mod_dir = tmpdir.mkdir('_modules')
187 lang_dir = mod_dir.mkdir('en_US')
188 lang_dir.join('module.json').write(json.dumps(LOCALES_MODULE))
189
190
191 @pytest.fixture
192 def icons(srcdir):
193 icons_dir = srcdir.mkdir('icons')
194 for filename in ['abp-16.png', 'abp-19.png', 'abp-53.png']:
195 shutil.copy(
196 os.path.join(os.path.dirname(__file__), filename),
197 os.path.join(str(icons_dir), filename),
198 )
199
200
201 @pytest.fixture
202 def all_lang_locales(tmpdir):
203 return locale_files(ALL_LANGUAGES, '_locales', tmpdir)
204
205
206 @pytest.fixture
207 def chrome_metadata(tmpdir):
208 filename = 'metadata.chrome'
209 copy_metadata(filename, tmpdir)
210
211
212 @pytest.fixture
213 def gecko_webext_metadata(tmpdir, chrome_metadata):
214 filename = 'metadata.gecko'
215 copy_metadata(filename, tmpdir)
216
217
218 @pytest.fixture
219 def edge_metadata(tmpdir):
220 filename = 'metadata.edge'
221 copy_metadata(filename, tmpdir)
222
223 return packager.readMetadata(str(tmpdir), 'edge')
224
225
226 @pytest.fixture
227 def keyfile(tmpdir):
228 """Test-privatekey for signing chrome release-package."""
229 return os.path.join(os.path.dirname(__file__), 'chrome_rsa.pem')
230
231
232 @pytest.fixture
233 def lib_files(tmpdir):
234 files = packager.Files(['lib'], set())
235 files['ext/a.js'] = 'var bar;'
236 files['lib/b.js'] = 'var foo;'
237
238 tmpdir.mkdir('lib').join('b.js').write(files['lib/b.js'])
239 tmpdir.mkdir('ext').join('a.js').write(files['ext/a.js'])
240
241 return files
242
243
244 def comparable_xml(xml):
245 """Create a nonambiguous representation of a given XML tree."""
246 def get_leafs_string(tree):
247 """Recursively build a string representing all xml leaf-nodes."""
248 root_str = '{}|{}|{}'.format(tree.tag, tree.tail, tree.text).strip()
249 result = []
250
251 if len(tree) > 0:
252 for subtree in tree:
253 for leaf in get_leafs_string(subtree):
254 result.append('{}__{}'.format(root_str, leaf))
255 for k, v in tree.attrib.items():
256 result.append('{}__{}:{}'.format(root_str, k, v))
257 else:
258 result.append(root_str)
259 return result
260
261 # XML data itself shall not be sorted, hence we can safely sort
262 # our string representations in order to produce a valid unified diff.
263 return sorted(get_leafs_string(ElementTree.fromstring(xml)))
264
265
266 def comparable_json(json_string):
267 """Create a nonambiguous representation of a given JSON string."""
268 return json.dumps(
269 json.loads(json_string), sort_keys=True, indent=0
270 ).split('\n')
271
272
273 def assert_manifest_content(manifest, expected_path):
274 extension = os.path.basename(expected_path).split('.')[-1]
275
276 with open(expected_path, 'r') as fp:
277 if extension == 'xml':
278 generated = comparable_xml(manifest)
279 expected = comparable_xml(fp.read())
280 else:
281 generated = comparable_json(manifest)
282 expected = comparable_json(fp.read())
283
284 diff = list(difflib.unified_diff(generated, expected, n=0))
285 assert len(diff) == 0, '\n'.join(diff)
286
287
288 def assert_webpack_bundle(package, prefix, excluded=False):
289 libfoo = package.read(os.path.join(prefix, 'lib/foo.js'))
290 libfoomap = package.read(os.path.join(prefix, 'lib/foo.js.map'))
291
292 assert 'var bar;' in libfoo
293 assert 'webpack:///./ext/a.js' in libfoomap
294
295 assert ('var foo;' in libfoo) != excluded
296 assert ('webpack:///./lib/b.js' in libfoomap) != excluded
297
298
299 def assert_devenv_scripts(package, devenv):
300 manifest = json.loads(package.read('manifest.json'))
301 filenames = package.namelist()
302 scripts = [
303 'ext/common.js',
304 'ext/background.js',
305 ]
306
307 assert ('qunit/index.html' in filenames) == devenv
308 assert ('devenvPoller__.js' in filenames) == devenv
309 assert ('devenvVersion__' in filenames) == devenv
310
311 if devenv:
312 assert '../ext/common.js' in package.read('qunit/index.html')
313 assert '../ext/background.js' in package.read('qunit/index.html')
314
315 assert set(manifest['background']['scripts']) == set(
316 scripts + ['devenvPoller__.js']
317 )
318 else:
319 assert set(manifest['background']['scripts']) == set(scripts)
320
321
322 def assert_base_files(package, platform):
323 filenames = set(package.namelist())
324
325 if platform in {'chrome', 'gecko'}:
326 assert 'bar.json' in filenames
327 assert 'manifest.json' in filenames
328 assert 'lib/foo.js' in filenames
329 assert 'foo/logo_50.png' in filenames
330 assert 'icons/logo_150.png' in filenames
331 else:
332 assert 'AppxManifest.xml' in filenames
333 assert 'AppxBlockMap.xml' in filenames
334 assert '[Content_Types].xml' in filenames
335
336 assert 'Extension/bar.json' in filenames
337 assert 'Extension/lib/foo.js' in filenames
338
339 assert package.read('Assets/logo_44.png') == '44'
340 assert package.read('Extension/icons/abp-44.png') == '44'
341
342
343 def assert_chrome_signature(filename, keyfile):
344 with open(filename, 'r') as fp:
345 content = fp.read()
346
347 _, _, l_pubkey, l_signature = unpack('<4sIII', content[:16])
348 signature = content[16 + l_pubkey: 16 + l_pubkey + l_signature]
349
350 digest = SHA.new()
351 with open(keyfile, 'r') as fp:
352 rsa_key = RSA.importKey(fp.read())
353
354 signer = PKCS1_v1_5.new(rsa_key)
355
356 digest.update(content[16 + l_pubkey + l_signature:])
357 assert signer.verify(digest, signature)
358
359
360 def assert_locale_upfix(package):
361 translations = [
362 json.loads(package.read('_locales/{}/messages.json'.format(lang)))
363 for lang in ALL_LANGUAGES
364 ]
365
366 manifest = package.read('manifest.json')
367
368 # Chrome Web Store requires descriptive translations to be present in
369 # every language.
370 for match in re.finditer(r'__MSG_(\S+)__', manifest):
371 name = match.group(1)
372
373 for other in translations[1:]:
374 assert translations[0][name]['message'] == other[name]['message']
375
376
377 @pytest.mark.usefixtures(
378 'all_lang_locales',
379 'locale_modules',
380 'icons',
381 'lib_files',
382 'chrome_metadata',
383 'gecko_webext_metadata',
384 'edge_metadata',
385 )
386 @pytest.mark.parametrize('platform,dev_build_release,buildnum', [
387 ('chrome', 'build', '1337'),
388 ('chrome', 'devenv', None),
389 ('chrome', 'release', None),
390 ('gecko', 'build', '1337'),
391 ('gecko', 'devenv', None),
392 ('gecko', 'release', None),
393 ('edge', 'build', '1337'),
394 pytest.param('edge', 'devenv', None, marks=pytest.mark.xfail),
395 ('edge', 'release', None),
396 ])
397 def test_build_webext(platform, dev_build_release, keyfile, tmpdir, srcdir,
398 buildnum, capsys):
399 release = dev_build_release == 'release'
400 devenv = dev_build_release == 'devenv'
401
402 if platform == 'chrome' and release:
403 key = keyfile
404 else:
405 key = None
406
407 manifests = {
408 'gecko': [('', 'manifest', 'json')],
409 'chrome': [('', 'manifest', 'json')],
410 'edge': [('', 'AppxManifest', 'xml'),
411 ('Extension', 'manifest', 'json')],
412 }
413
414 filenames = {
415 'gecko': 'adblockplusfirefox-1.2.3{}.xpi',
416 'chrome': 'adblockpluschrome-1.2.3{{}}.{}'.format(
417 {True: 'crx', False: 'zip'}[release]
418 ),
419 'edge': 'adblockplusedge-1.2.3{}.appx',
420 }
421
422 if platform == 'edge':
423 prefix = 'Extension'
424 else:
425 prefix = ''
426
427 run_webext_build(platform, dev_build_release, srcdir, buildnum=buildnum,
428 keyfile=key)
429
430 # The makeIcons() in packagerChrome.py should warn about non-square
431 # icons via stderr.
432 if platform in {'chrome', 'gecko'}:
433 out, err = capsys.readouterr()
434 assert 'icon should be square' in err
435
436 if devenv:
437 content_class = DirContent
438 out_file_path = os.path.join(str(srcdir), 'devenv.' + platform)
439 else:
440 content_class = ZipContent
441
442 if release:
443 add_version = ''
444 else:
445 add_version = '.' + buildnum
446
447 out_file = filenames[platform].format(add_version)
448
449 out_file_path = os.path.abspath(os.path.join(
450 os.path.dirname(__file__), os.pardir, out_file))
451 assert os.path.exists(out_file_path)
452
453 if release and platform == 'chrome':
454 assert_chrome_signature(out_file_path, keyfile)
455
456 with content_class(out_file_path) as package:
457 assert_base_files(package, platform)
458 assert_all_locales_present(package, prefix)
459 assert_webpack_bundle(package, prefix, platform == 'gecko')
460
461 if platform == 'chrome':
462 assert_locale_upfix(package)
463 if platform in {'chrome', 'gecko'}:
464 assert_devenv_scripts(package, devenv)
465
466 for folder, name, ext in manifests[platform]:
467 filename = '{{}}_{}_{}.{{}}'.format(platform, dev_build_release)
468 expected = os.path.join(
469 os.path.dirname(__file__),
470 'expecteddata',
471 filename.format(name, ext),
472 )
473
474 assert_manifest_content(
475 package.read(os.path.join(folder, '{}.{}'.format(name, ext))),
476 expected,
477 )
OLDNEW
« no previous file with comments | « tests/test_packagerEdge.py ('k') | tox.ini » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld