Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: tests/test_packagerWebExt.py

Issue 29501558: Issue 5383 - Add tests for the Chrome and Firefox packagers (Closed)
Left Patch Set: Adding pytest-cov, adhering to comments Created Aug. 14, 2017, 2:19 p.m.
Right Patch Set: Addressing Vasily's comments Created Oct. 22, 2017, 11:11 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « tests/test_packagerEdge.py ('k') | tox.ini » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 import difflib
6 import json
7 import os
8 import re
9 import shutil
10 import zipfile
11 from xml.etree import ElementTree
12 from struct import unpack
13
14 import pytest
15 from Crypto.Signature import PKCS1_v1_5
16 from Crypto.PublicKey import RSA
17 from Crypto.Hash import SHA
18
19 from buildtools import packager
20 from buildtools.packagerChrome import defaultLocale
21 from buildtools.build import processArgs
22
23 LOCALES_MODULE = {
24 'test.Foobar': {
25 'message': 'Ensuring dict-copy from modules for $domain$',
26 'description': 'test description',
27 'placeholders': {'content': '$1', 'example': 'www.adblockplus.org'}
28 }
29 }
30
31 ALL_LANGUAGES = ['en_US', 'de', 'it']
32
33 MESSAGES_EN_US = json.dumps({
34 'name': {'message': 'Adblock Plus'},
35 'name_devbuild': {'message': 'devbuild-marker'},
36 'description': {
37 'message': 'Adblock Plus is the most popular ad blocker ever, '
38 'and also supports websites by not blocking '
39 'unobstrusive ads by default (configurable).'
40 },
41 })
42
43
44 class Content(object):
45 """Base class for a unified ZipFile / Directory interface.
46
47 Base class for providing a unified context manager interface for
48 accessing files. This class is subclassed by ZipContent and DirContent,
49 which provide the additional methods "namelist()" and "read(path)".
50 """
51
52 def __enter__(self):
53 return self
54
55 def __exit__(self, exc_type, exc_value, exc_tb):
56 self._close()
57
58
59 class ZipContent(Content):
60 """Provide a unified context manager for ZipFile access.
61
62 Inherits the context manager API from Content.
63 If desired, the specified ZipFile is deleted on exiting the manager.
64 """
65
66 def __init__(self, zip_path, delete_on_close=True):
67 """Construct ZipContent object handling the file <zip_path>.
68
69 The parameter 'delete_on_close' causes the context manager to
70 delete the handled ZipFile (specified by zip_path) if set to
71 True (default).
72 """
73 self._zip_path = zip_path
74 self._zip_file = zipfile.ZipFile(zip_path)
75 self._delete_on_close = delete_on_close
76 super(ZipContent, self).__init__()
77
78 def _close(self):
79 self._zip_file.close()
80 if self._delete_on_close:
81 # if specified, delete the handled file
82 os.remove(self._zip_path)
83
84 def namelist(self):
85 return self._zip_file.namelist()
86
87 def read(self, path):
88 return self._zip_file.read(path)
89
90
91 class DirContent(Content):
92 """Provides a unified context manager for directory access.
93
94 Inherits the context managet API from Content.
95 """
96
97 def __init__(self, path):
98 """Construct a DirContent object handling <path>."""
99 self._path = path
100 super(DirContent, self).__init__()
101
102 def _close(self):
103 pass
104
105 def namelist(self):
106 """Generate a list of filenames."""
107 result = []
108 for parent, directories, files in os.walk(self._path):
109 for filename in files:
110 file_path = os.path.join(parent, filename)
111 rel_path = os.path.relpath(file_path, self._path)
112 result.append(rel_path)
113 return result
114
115 def read(self, path):
116 content = None
117 with open(os.path.join(self._path, path)) as fp:
118 content = fp.read()
119 return content
120
121
122 def copy_metadata(filename, tmpdir):
123 """Copy the used metadata to the used temporary directory."""
124 path = os.path.join(os.path.dirname(__file__), filename)
125 destination = str(tmpdir.join(filename))
126 shutil.copy(path, destination)
127
128
129 def run_webext_build(ext_type, build_opt, srcdir, keyfile=None):
130 """Run a build process."""
131 cmd_mapping = {
132 'devenv': ['devenv'],
133 'development_build': ['build', '-b', '1337'],
134 'release_build': ['build', '-r'],
135 }
136
137 args = ['build.py', '-t', ext_type] + cmd_mapping[build_opt]
138
139 if keyfile:
140 args += ['-k', keyfile]
141
142 processArgs(str(srcdir), args)
143
144
145 def locale_files(languages, rootfolder, srcdir):
146 """Generate example locales.
147
148 languages: tuple, list or set of languages to include
149 rootdir: folder-name to create the locale-folders in
150 """
151 for lang in languages:
152 subfolders = rootfolder.split(os.pathsep) + [lang, 'messages.json']
153 json_file = srcdir.ensure(*subfolders)
154 if lang == defaultLocale:
155 json_file.write(MESSAGES_EN_US)
156 else:
157 json_file.write('{}')
158
159
160 def assert_all_locales_present(package, prefix):
161 names = {x for x in package.namelist() if
162 x.startswith(os.path.join(prefix, '_locales'))}
163 assert names == {
164 os.path.join(prefix, '_locales', lang, 'messages.json')
165 for lang in ALL_LANGUAGES
166 }
167
168
169 @pytest.fixture
170 def srcdir(tmpdir):
171 """Source directory for building the package."""
172 for size in ['44', '50', '150']:
173 path = tmpdir.join('chrome', 'icons', 'abp-{}.png'.format(size))
174 path.write(size, ensure=True)
175
176 tmpdir.join('bar.json').write(json.dumps({}))
177 return tmpdir
178
179
180 @pytest.fixture
181 def locale_modules(tmpdir):
182 mod_dir = tmpdir.mkdir('_modules')
183 lang_dir = mod_dir.mkdir('en_US')
184 lang_dir.join('module.json').write(json.dumps(LOCALES_MODULE))
185
186
187 @pytest.fixture
188 def icons(srcdir):
189 icons_dir = srcdir.mkdir('icons')
190 for filename in ['abp-16.png', 'abp-19.png', 'abp-53.png']:
191 shutil.copy(
192 os.path.join(os.path.dirname(__file__), filename),
193 os.path.join(str(icons_dir), filename),
194 )
195
196
197 @pytest.fixture
198 def all_lang_locales(tmpdir):
199 return locale_files(ALL_LANGUAGES, '_locales', tmpdir)
200
201
202 @pytest.fixture
203 def chrome_metadata(tmpdir):
204 filename = 'metadata.chrome'
205 copy_metadata(filename, tmpdir)
206
207
208 @pytest.fixture
209 def gecko_webext_metadata(tmpdir, chrome_metadata):
210 filename = 'metadata.gecko'
211 copy_metadata(filename, tmpdir)
212
213
214 @pytest.fixture
215 def edge_metadata(tmpdir):
216 filename = 'metadata.edge'
217 copy_metadata(filename, tmpdir)
218
219 return packager.readMetadata(str(tmpdir), 'edge')
220
221
222 @pytest.fixture
223 def keyfile(tmpdir):
224 """Test-privatekey for signing chrome release-package."""
225 return os.path.join(os.path.dirname(__file__), 'chrome_rsa.pem')
226
227
228 @pytest.fixture
229 def lib_files(tmpdir):
230 files = packager.Files(['lib'], set())
231 files['ext/a.js'] = 'require("./c.js");\nvar bar;'
232 files['lib/b.js'] = 'var foo;'
233 files['ext/c.js'] = 'var this_is_c;'
234
235 tmpdir.mkdir('lib').join('b.js').write(files['lib/b.js'])
236 ext_dir = tmpdir.mkdir('ext')
237 ext_dir.join('a.js').write(files['ext/a.js'])
238 ext_dir.join('c.js').write(files['ext/c.js'])
239
240 return files
241
242
243 def comparable_xml(xml):
244 """Create a nonambiguous representation of a given XML tree.
245
246 Note that this function is not safe against ambiguous tags
247 containing differently distributed children, e.g.:
248
249 '<a><b><c/></b><b><d/></b></a>'
250 vs.
251 '<a><b/><b><c/><d/></b></a>'
252
253 This is irrelevant for comparing the XML used by the tests of this
254 module.
255 """
256 def get_leafs_string(tree):
257 """Recursively build a string representing all xml leaf-nodes."""
258 root_str = '{}|{}|{}'.format(tree.tag, tree.tail, tree.text).strip()
259 result = []
260
261 if len(tree) > 0:
262 for subtree in tree:
263 for leaf in get_leafs_string(subtree):
264 result.append('{}__{}'.format(root_str, leaf))
265 for k, v in tree.attrib.items():
266 result.append('{}__{}:{}'.format(root_str, k, v))
267 else:
268 result.append(root_str)
269 return result
270
271 # XML data itself shall not be sorted, hence we can safely sort
272 # our string representations in order to produce a valid unified diff.
273 return sorted(get_leafs_string(ElementTree.fromstring(xml)))
274
275
276 def comparable_json(json_string):
277 """Create a nonambiguous representation of a given JSON string."""
278 return json.dumps(
279 json.loads(json_string), sort_keys=True, indent=0
280 ).split('\n')
281
282
283 def assert_manifest_content(manifest, expected_path):
284 extension = os.path.basename(expected_path).split('.')[-1]
285
286 with open(expected_path, 'r') as fp:
287 if extension == 'xml':
288 generated = comparable_xml(manifest)
289 expected = comparable_xml(fp.read())
290 else:
291 generated = comparable_json(manifest)
292 expected = comparable_json(fp.read())
293
294 diff = list(difflib.unified_diff(generated, expected, n=0))
295 assert len(diff) == 0, '\n'.join(diff)
296
297
298 def assert_webpack_bundle(package, prefix, excluded=False):
299 libfoo = package.read(os.path.join(prefix, 'lib/foo.js'))
300 libfoomap = package.read(os.path.join(prefix, 'lib/foo.js.map'))
301
302 assert 'var bar;' in libfoo
303 assert 'webpack:///./ext/a.js' in libfoomap
304
305 assert 'var this_is_c;' in libfoo
306 assert 'webpack:///./ext/c.js' in libfoomap
307
308 assert ('var foo;' in libfoo) != excluded
309 assert ('webpack:///./lib/b.js' in libfoomap) != excluded
310
311
312 def assert_devenv_scripts(package, prefix, devenv):
313 manifest = json.loads(package.read(os.path.join(prefix, 'manifest.json')))
314 filenames = package.namelist()
315 scripts = [
316 'ext/common.js',
317 'ext/background.js',
318 ]
319
320 assert (os.path.join(prefix, 'qunit/index.html') in filenames) == devenv
321 assert (os.path.join(prefix, 'devenvPoller__.js') in filenames) == devenv
322 assert (os.path.join(prefix, 'devenvVersion__') in filenames) == devenv
323
324 if devenv:
325 quint_index = package.read(os.path.join(prefix, 'qunit/index.html'))
326 assert '../ext/common.js' in quint_index
327 assert '../ext/background.js' in quint_index
328
329 assert set(manifest['background']['scripts']) == set(
330 scripts + ['devenvPoller__.js']
331 )
332 else:
333 assert set(manifest['background']['scripts']) == set(scripts)
334
335
336 def assert_base_files(package, platform, prefix):
337 filenames = set(package.namelist())
338
339 if platform == 'edge':
340 assert 'AppxManifest.xml' in filenames
341 assert 'AppxBlockMap.xml' in filenames
342 assert '[Content_Types].xml' in filenames
343
344 assert package.read('Assets/logo_44.png') == '44'
345 assert package.read('Extension/icons/abp-44.png') == '44'
346
347 assert os.path.join(prefix, 'bar.json') in filenames
348 assert os.path.join(prefix, 'manifest.json') in filenames
349 assert os.path.join(prefix, 'lib/foo.js') in filenames
350 assert os.path.join(prefix, 'foo/logo_50.png') in filenames
351 assert os.path.join(prefix, 'icons/logo_150.png') in filenames
352
353
354 def assert_chrome_signature(filename, keyfile):
355 with open(filename, 'r') as fp:
356 content = fp.read()
357
358 _, _, l_pubkey, l_signature = unpack('<4sIII', content[:16])
359 signature = content[16 + l_pubkey: 16 + l_pubkey + l_signature]
360
361 digest = SHA.new()
362 with open(keyfile, 'r') as fp:
363 rsa_key = RSA.importKey(fp.read())
364
365 signer = PKCS1_v1_5.new(rsa_key)
366
367 digest.update(content[16 + l_pubkey + l_signature:])
368 assert signer.verify(digest, signature)
369
370
371 def assert_locale_upfix(package):
372 translations = [
373 json.loads(package.read('_locales/{}/messages.json'.format(lang)))
374 for lang in ALL_LANGUAGES
375 ]
376
377 manifest = package.read('manifest.json')
378
379 # Chrome Web Store requires descriptive translations to be present in
380 # every language.
381 for match in re.finditer(r'__MSG_(\S+)__', manifest):
382 name = match.group(1)
383
384 for other in translations[1:]:
385 assert translations[0][name]['message'] == other[name]['message']
386
387
388 @pytest.mark.usefixtures(
389 'all_lang_locales',
390 'locale_modules',
391 'icons',
392 'lib_files',
393 'chrome_metadata',
394 'gecko_webext_metadata',
395 'edge_metadata',
396 )
397 @pytest.mark.parametrize('platform,command', [
398 ('chrome', 'development_build'),
399 ('chrome', 'devenv'),
400 ('chrome', 'release_build'),
401 ('gecko', 'development_build'),
402 ('gecko', 'devenv'),
403 ('gecko', 'release_build'),
404 ('edge', 'development_build'),
405 ('edge', 'devenv'),
406 ('edge', 'release_build'),
407 ])
408 def test_build_webext(platform, command, keyfile, tmpdir, srcdir, capsys):
409 release = command == 'release_build'
410 devenv = command == 'devenv'
411
412 if platform == 'chrome' and release:
413 key = keyfile
414 else:
415 key = None
416
417 manifests = {
418 'gecko': [('', 'manifest', 'json')],
419 'chrome': [('', 'manifest', 'json')],
420 'edge': [('', 'AppxManifest', 'xml'),
421 ('Extension', 'manifest', 'json')],
422 }
423
424 filenames = {
425 'gecko': 'adblockplusfirefox-1.2.3{}.xpi',
426 'chrome': 'adblockpluschrome-1.2.3{{}}.{}'.format(
427 {True: 'crx', False: 'zip'}[release]
428 ),
429 'edge': 'adblockplusedge-1.2.3{}.appx',
430 }
431
432 if platform == 'edge':
433 prefix = 'Extension'
434 else:
435 prefix = ''
436
437 run_webext_build(platform, command, srcdir, keyfile=key)
438
439 # The makeIcons() in packagerChrome.py should warn about non-square
440 # icons via stderr.
441 out, err = capsys.readouterr()
442 assert 'icon should be square' in err
443
444 if devenv:
445 content_class = DirContent
446 out_file_path = os.path.join(str(srcdir), 'devenv.' + platform)
447 else:
448 content_class = ZipContent
449
450 if release:
451 add_version = ''
452 else:
453 add_version = '.1337'
454
455 out_file = filenames[platform].format(add_version)
456
457 out_file_path = os.path.abspath(os.path.join(
458 os.path.dirname(__file__), os.pardir, out_file))
459 assert os.path.exists(out_file_path)
460
461 if release and platform == 'chrome':
462 assert_chrome_signature(out_file_path, keyfile)
463
464 with content_class(out_file_path) as package:
465 assert_base_files(package, platform, prefix)
466 assert_all_locales_present(package, prefix)
467 assert_webpack_bundle(package, prefix, platform == 'gecko')
468
469 if platform == 'chrome':
470 assert_locale_upfix(package)
471
472 assert_devenv_scripts(package, prefix, devenv)
473
474 for folder, name, ext in manifests[platform]:
475 filename = '{{}}_{}_{}.{{}}'.format(platform, command)
476 expected = os.path.join(
477 os.path.dirname(__file__),
478 'expecteddata',
479 filename.format(name, ext),
480 )
481
482 assert_manifest_content(
483 package.read(os.path.join(folder, '{}.{}'.format(name, ext))),
484 expected,
485 )
LEFTRIGHT

Powered by Google App Engine
This is Rietveld