Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: tests/test_packagerWebExt.py

Issue 29501558: Issue 5383 - Add tests for the Chrome and Firefox packagers (Closed)
Patch Set: NO CHANGE Rebasing against #4720 @ PS 4 Created Oct. 18, 2017, 11:18 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tests/test_packagerEdge.py ('k') | tox.ini » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 import os
6 import shutil
7 import zipfile
8 import json
9 import re
10 from struct import unpack
11 import difflib
12
13 import pytest
14 from Crypto.Hash import SHA
15 from Crypto.PublicKey import RSA
16 from Crypto.Signature import PKCS1_v1_5
17 from xml.etree import ElementTree
18
19
20 from buildtools import packager
21 from buildtools.packagerChrome import defaultLocale
22 from buildtools.build import processArgs
23
24 LOCALES_MODULE = {
25 'test.Foobar': {
26 'message': 'Ensuring dict-copy from modules for $domain$',
27 'description': 'test description',
28 'placeholders': {'content': '$1', 'example': 'www.adblockplus.org'}
29 }
30 }
31
32 ALL_LANGUAGES = ['en_US', 'de', 'it']
33
34 MESSAGES_EN_US = json.dumps({
35 'name': {'message': 'Adblock Plus'},
36 'name_devbuild': {'message': 'devbuild-marker'},
37 'description': {
38 'message': 'Adblock Plus is the most popular ad blocker ever, '
39 'and also supports websites by not blocking '
40 'unobstrusive ads by default (configurable).'
41 },
42 })
43
44
45 class Content(object):
46 """Base class for a unified ZipFile / Directory interface.
47
48 Base class for providing a unified context manager interface for
49 accessing files. This class is subclassed by ZipContent and DirContent,
50 which provide the additional methods "namelist()" and "read(path)".
51 """
52
53 def __enter__(self):
54 return self
55
56 def __exit__(self, exc_type, exc_value, exc_tb):
57 self._close()
58
59
60 class ZipContent(Content):
61 """Provide a unified context manager for ZipFile access.
62
63 Inherits the context manager API from Content.
64 If desired, the specified ZipFile is deleted on exiting the manager.
65 """
66
67 def __init__(self, zip_path, delete_on_close=True):
68 """Construct ZipContent object handling the file <zip_path>.
69
70 The parameter 'delete_on_close' causes the context manager to
71 delete the handled ZipFile (specified by zip_path) if set to
72 True (default).
73 """
74 self._zip_path = zip_path
75 self._zip_file = zipfile.ZipFile(zip_path)
76 self._delete_on_close = delete_on_close
77 super(ZipContent, self).__init__()
78
79 def _close(self):
80 self._zip_file.close()
81 if self._delete_on_close:
82 # if specified, delete the handled file
83 os.remove(self._zip_path)
84
85 def namelist(self):
86 return self._zip_file.namelist()
87
88 def read(self, path):
89 return self._zip_file.read(path)
90
91
92 class DirContent(Content):
93 """Provides a unified context manager for directory access.
94
95 Inherits the context managet API from Content.
96 """
97
98 def __init__(self, path):
99 """Construct a DirContent object handling <path>."""
100 self._path = path
101 super(DirContent, self).__init__()
102
103 def _close(self):
104 pass
105
106 def namelist(self):
107 """Generate a list of filenames."""
108 result = []
109 for parent, directories, files in os.walk(self._path):
110 for filename in files:
111 file_path = os.path.join(parent, filename)
112 rel_path = os.path.relpath(file_path, self._path)
113 result.append(rel_path)
114 return result
115
116 def read(self, path):
117 content = None
118 with open(os.path.join(self._path, path)) as fp:
119 content = fp.read()
120 return content
121
122
123 def copy_metadata(filename, tmpdir):
124 """Copy the used metadata to the used temporary directory."""
125 path = os.path.join(os.path.dirname(__file__), filename)
126 destination = str(tmpdir.join(filename))
127 shutil.copy(path, destination)
128
129
130 def run_webext_build(ext_type, build_opt, srcdir, buildnum=None, keyfile=None):
131 """Run a build process."""
132 if build_opt == 'build':
133 build_args = ['build']
134 elif build_opt == 'release':
135 build_args = ['build', '-r']
136 else:
137 build_args = ['devenv']
138
139 args = ['build.py', '-t', ext_type] + build_args
140
141 if keyfile:
142 args += ['-k', keyfile]
143 if buildnum:
144 args += ['-b', buildnum]
145
146 processArgs(str(srcdir), args)
147
148
149 def locale_files(languages, rootfolder, srcdir):
150 """Generate example locales.
151
152 languages: tuple, list or set of languages to include
153 rootdir: folder-name to create the locale-folders in
154 """
155 for lang in languages:
156 subfolders = rootfolder.split(os.pathsep) + [lang, 'messages.json']
157 json_file = srcdir.ensure(*subfolders)
158 if lang == defaultLocale:
159 json_file.write(MESSAGES_EN_US)
160 else:
161 json_file.write('{}')
162
163
164 def assert_all_locales_present(package, prefix):
165 names = {x for x in package.namelist() if
166 x.startswith(os.path.join(prefix, '_locales'))}
167 assert names == {
168 os.path.join(prefix, '_locales', lang, 'messages.json')
169 for lang in ALL_LANGUAGES
170 }
171
172
173 @pytest.fixture
174 def srcdir(tmpdir):
175 """Source directory for building the package."""
176 for size in ['44', '50', '150']:
177 path = tmpdir.join('chrome', 'icons', 'abp-{}.png'.format(size))
178 path.write(size, ensure=True)
179
180 tmpdir.join('bar.json').write(json.dumps({}))
181 return tmpdir
182
183
184 @pytest.fixture
185 def locale_modules(tmpdir):
186 mod_dir = tmpdir.mkdir('_modules')
187 lang_dir = mod_dir.mkdir('en_US')
188 lang_dir.join('module.json').write(json.dumps(LOCALES_MODULE))
189
190
191 @pytest.fixture
192 def icons(srcdir):
193 icons_dir = srcdir.mkdir('icons')
194 for filename in ['abp-16.png', 'abp-19.png', 'abp-53.png']:
195 shutil.copy(
196 os.path.join(os.path.dirname(__file__), filename),
197 os.path.join(str(icons_dir), filename),
198 )
199
200
201 @pytest.fixture
202 def all_lang_locales(tmpdir):
203 return locale_files(ALL_LANGUAGES, '_locales', tmpdir)
204
205
206 @pytest.fixture
207 def chrome_metadata(tmpdir):
208 filename = 'metadata.chrome'
209 copy_metadata(filename, tmpdir)
210
211
212 @pytest.fixture
213 def gecko_webext_metadata(tmpdir, chrome_metadata):
214 filename = 'metadata.gecko'
215 copy_metadata(filename, tmpdir)
216
217
218 @pytest.fixture
219 def edge_metadata(tmpdir):
220 filename = 'metadata.edge'
221 copy_metadata(filename, tmpdir)
222
223 return packager.readMetadata(str(tmpdir), 'edge')
224
225
226 @pytest.fixture
227 def keyfile(tmpdir):
228 """Test-privatekey for signing chrome release-package."""
229 return os.path.join(os.path.dirname(__file__), 'chrome_rsa.pem')
230
231
232 @pytest.fixture
233 def lib_files(tmpdir):
234 files = packager.Files(['lib'], set())
235 files['ext/a.js'] = 'require("./c.js");\nvar bar;'
236 files['lib/b.js'] = 'var foo;'
237 files['ext/c.js'] = 'var this_is_c;'
238
239 tmpdir.mkdir('lib').join('b.js').write(files['lib/b.js'])
240 ext_dir = tmpdir.mkdir('ext')
241 ext_dir.join('a.js').write(files['ext/a.js'])
242 ext_dir.join('c.js').write(files['ext/c.js'])
243
244 return files
245
246
247 def comparable_xml(xml):
248 """Create a nonambiguous representation of a given XML tree."""
249 def get_leafs_string(tree):
250 """Recursively build a string representing all xml leaf-nodes."""
251 root_str = '{}|{}|{}'.format(tree.tag, tree.tail, tree.text).strip()
252 result = []
253
254 if len(tree) > 0:
255 for subtree in tree:
256 for leaf in get_leafs_string(subtree):
257 result.append('{}__{}'.format(root_str, leaf))
258 for k, v in tree.attrib.items():
259 result.append('{}__{}:{}'.format(root_str, k, v))
260 else:
261 result.append(root_str)
262 return result
263
264 # XML data itself shall not be sorted, hence we can safely sort
265 # our string representations in order to produce a valid unified diff.
266 return sorted(get_leafs_string(ElementTree.fromstring(xml)))
267
268
269 def comparable_json(json_string):
270 """Create a nonambiguous representation of a given JSON string."""
271 return json.dumps(
272 json.loads(json_string), sort_keys=True, indent=0
273 ).split('\n')
274
275
276 def assert_manifest_content(manifest, expected_path):
277 extension = os.path.basename(expected_path).split('.')[-1]
278
279 with open(expected_path, 'r') as fp:
280 if extension == 'xml':
281 generated = comparable_xml(manifest)
282 expected = comparable_xml(fp.read())
283 else:
284 generated = comparable_json(manifest)
285 expected = comparable_json(fp.read())
286
287 diff = list(difflib.unified_diff(generated, expected, n=0))
288 assert len(diff) == 0, '\n'.join(diff)
289
290
291 def assert_webpack_bundle(package, prefix, excluded=False):
292 libfoo = package.read(os.path.join(prefix, 'lib/foo.js'))
293 libfoomap = package.read(os.path.join(prefix, 'lib/foo.js.map'))
294
295 assert 'var bar;' in libfoo
296 assert 'webpack:///./ext/a.js' in libfoomap
297
298 assert 'var this_is_c;' in libfoo
299 assert 'webpack:///./ext/c.js' in libfoomap
300
301 assert ('var foo;' in libfoo) != excluded
302 assert ('webpack:///./lib/b.js' in libfoomap) != excluded
303
304
305 def assert_devenv_scripts(package, devenv):
306 manifest = json.loads(package.read('manifest.json'))
307 filenames = package.namelist()
308 scripts = [
309 'ext/common.js',
310 'ext/background.js',
311 ]
312
313 assert ('qunit/index.html' in filenames) == devenv
314 assert ('devenvPoller__.js' in filenames) == devenv
315 assert ('devenvVersion__' in filenames) == devenv
316
317 if devenv:
318 assert '../ext/common.js' in package.read('qunit/index.html')
319 assert '../ext/background.js' in package.read('qunit/index.html')
320
321 assert set(manifest['background']['scripts']) == set(
322 scripts + ['devenvPoller__.js']
323 )
324 else:
325 assert set(manifest['background']['scripts']) == set(scripts)
326
327
328 def assert_base_files(package, platform):
329 filenames = set(package.namelist())
330
331 if platform in {'chrome', 'gecko'}:
332 assert 'bar.json' in filenames
333 assert 'manifest.json' in filenames
334 assert 'lib/foo.js' in filenames
335 assert 'foo/logo_50.png' in filenames
336 assert 'icons/logo_150.png' in filenames
337 else:
338 assert 'AppxManifest.xml' in filenames
339 assert 'AppxBlockMap.xml' in filenames
340 assert '[Content_Types].xml' in filenames
341
342 assert 'Extension/bar.json' in filenames
343 assert 'Extension/lib/foo.js' in filenames
344
345 assert package.read('Assets/logo_44.png') == '44'
346 assert package.read('Extension/icons/abp-44.png') == '44'
347
348
349 def assert_chrome_signature(filename, keyfile):
350 with open(filename, 'r') as fp:
351 content = fp.read()
352
353 _, _, l_pubkey, l_signature = unpack('<4sIII', content[:16])
354 signature = content[16 + l_pubkey: 16 + l_pubkey + l_signature]
355
356 digest = SHA.new()
357 with open(keyfile, 'r') as fp:
358 rsa_key = RSA.importKey(fp.read())
359
360 signer = PKCS1_v1_5.new(rsa_key)
361
362 digest.update(content[16 + l_pubkey + l_signature:])
363 assert signer.verify(digest, signature)
364
365
366 def assert_locale_upfix(package):
367 translations = [
368 json.loads(package.read('_locales/{}/messages.json'.format(lang)))
369 for lang in ALL_LANGUAGES
370 ]
371
372 manifest = package.read('manifest.json')
373
374 # Chrome Web Store requires descriptive translations to be present in
375 # every language.
376 for match in re.finditer(r'__MSG_(\S+)__', manifest):
377 name = match.group(1)
378
379 for other in translations[1:]:
380 assert translations[0][name]['message'] == other[name]['message']
381
382
383 @pytest.mark.usefixtures(
384 'all_lang_locales',
385 'locale_modules',
386 'icons',
387 'lib_files',
388 'chrome_metadata',
389 'gecko_webext_metadata',
390 'edge_metadata',
391 )
392 @pytest.mark.parametrize('platform,dev_build_release,buildnum', [
393 ('chrome', 'build', '1337'),
394 ('chrome', 'devenv', None),
395 ('chrome', 'release', None),
396 ('gecko', 'build', '1337'),
397 ('gecko', 'devenv', None),
398 ('gecko', 'release', None),
399 ('edge', 'build', '1337'),
400 pytest.param('edge', 'devenv', None, marks=pytest.mark.xfail),
401 ('edge', 'release', None),
402 ])
403 def test_build_webext(platform, dev_build_release, keyfile, tmpdir, srcdir,
404 buildnum, capsys):
405 release = dev_build_release == 'release'
406 devenv = dev_build_release == 'devenv'
407
408 if platform == 'chrome' and release:
409 key = keyfile
410 else:
411 key = None
412
413 manifests = {
414 'gecko': [('', 'manifest', 'json')],
415 'chrome': [('', 'manifest', 'json')],
416 'edge': [('', 'AppxManifest', 'xml'),
417 ('Extension', 'manifest', 'json')],
418 }
419
420 filenames = {
421 'gecko': 'adblockplusfirefox-1.2.3{}.xpi',
422 'chrome': 'adblockpluschrome-1.2.3{{}}.{}'.format(
423 {True: 'crx', False: 'zip'}[release]
424 ),
425 'edge': 'adblockplusedge-1.2.3{}.appx',
426 }
427
428 if platform == 'edge':
429 prefix = 'Extension'
430 else:
431 prefix = ''
432
433 run_webext_build(platform, dev_build_release, srcdir, buildnum=buildnum,
434 keyfile=key)
435
436 # The makeIcons() in packagerChrome.py should warn about non-square
437 # icons via stderr.
438 if platform in {'chrome', 'gecko'}:
439 out, err = capsys.readouterr()
440 assert 'icon should be square' in err
441
442 if devenv:
443 content_class = DirContent
444 out_file_path = os.path.join(str(srcdir), 'devenv.' + platform)
445 else:
446 content_class = ZipContent
447
448 if release:
449 add_version = ''
450 else:
451 add_version = '.' + buildnum
452
453 out_file = filenames[platform].format(add_version)
454
455 out_file_path = os.path.abspath(os.path.join(
456 os.path.dirname(__file__), os.pardir, out_file))
457 assert os.path.exists(out_file_path)
458
459 if release and platform == 'chrome':
460 assert_chrome_signature(out_file_path, keyfile)
461
462 with content_class(out_file_path) as package:
463 assert_base_files(package, platform)
464 assert_all_locales_present(package, prefix)
465 assert_webpack_bundle(package, prefix, platform == 'gecko')
466
467 if platform == 'chrome':
468 assert_locale_upfix(package)
469 if platform in {'chrome', 'gecko'}:
470 assert_devenv_scripts(package, devenv)
471
472 for folder, name, ext in manifests[platform]:
473 filename = '{{}}_{}_{}.{{}}'.format(platform, dev_build_release)
474 expected = os.path.join(
475 os.path.dirname(__file__),
476 'expecteddata',
477 filename.format(name, ext),
478 )
479
480 assert_manifest_content(
481 package.read(os.path.join(folder, '{}.{}'.format(name, ext))),
482 expected,
483 )
OLDNEW
« no previous file with comments | « tests/test_packagerEdge.py ('k') | tox.ini » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld