Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: tests/test_packagerWebExt.py

Issue 29501558: Issue 5383 - Add tests for the Chrome and Firefox packagers (Closed)
Patch Set: Rebase against master, renaming gecko-webext to gecko Created Oct. 11, 2017, 3:59 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tests/test_packagerEdge.py ('k') | tox.ini » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 import os
6 import shutil
7 import zipfile
8 import json
9 import re
10 from struct import unpack
11 import difflib
12
13 import pytest
14 from Crypto.Hash import SHA
15 from Crypto.PublicKey import RSA
16 from Crypto.Signature import PKCS1_v1_5
17 from xml.etree import ElementTree
18
19
20 from buildtools import packager
21 from buildtools.packagerChrome import defaultLocale
22 from buildtools.build import processArgs
23
24 LOCALES_MODULE = {
25 'test.Foobar': {
26 'message': 'Ensuring dict-copy from modules for $domain$',
27 'description': 'test description',
28 'placeholders': {'content': '$1', 'example': 'www.adblockplus.org'}
29 }
30 }
31
32 DTD_TEST = ('<!ENTITY access.key "access key(&amp;a)">'
33 '<!ENTITY ampersand "foo &amp;-bar">')
34
35 PROPERTIES_TEST = 'description=very descriptive!'
36
37 ALL_LANGUAGES = ['en_US', 'de', 'it']
38
39
40 MESSAGES_EN_US = json.dumps({
41 'name': {'message': 'Adblock Plus'},
42 'name_devbuild': {'message': 'devbuild-marker'},
43 'description': {
44 'message': 'Adblock Plus is the most popular ad blocker ever, '
45 'and also supports websites by not blocking '
46 'unobstrusive ads by default (configurable).'
47 },
48 })
49
50
51 class Content(object):
52 """Base class for a unified ZipFile / Directory interface.
53
54 Base class for providing a unified context manager interface for
55 accessing files. This class is subclassed by ZipContent and DirContent,
56 which provide the additional methods "namelist()" and "read(path)".
57 """
58
59 def __enter__(self):
60 return self
61
62 def __exit__(self, exc_type, exc_value, exc_tb):
63 self._close()
64
65
66 class ZipContent(Content):
67 """Provides a unified context manager for ZipFile access.
68
69 Inherits the context manager API from Content.
70 If desired, the specified ZipFile is deleted on exiting the manager.
71 """
72
73 def __init__(self, zip_path, delete_on_close=True):
74 """Constructor of ZipContent handling the file <zip_path>.
75
76 The parameter 'delete_on_close' causes the context manager to
77 delete the handled ZipFile (specified by zip_path) if set to
78 True (default).
79 """
80
81 self._zip_path = zip_path
82 self._zip_file = zipfile.ZipFile(zip_path)
83 self._delete_on_close = delete_on_close
84 super(ZipContent, self).__init__()
85
86 def _close(self):
87 self._zip_file.close()
88 if self._delete_on_close:
89 # if specified, delete the handled file
90 os.remove(self._zip_path)
91
92 def namelist(self):
93 return self._zip_file.namelist()
94
95 def read(self, path):
96 return self._zip_file.read(path)
97
98
99 class DirContent(Content):
100 """Provides a unified context manager for directory access.
101
102 Inherits the context managet API from Content.
103 """
104
105 def __init__(self, path):
106 """Constructor of DirContent handling <path>.
107 """
108
109 self._path = path
110 super(DirContent, self).__init__()
111
112 def _close(self):
113 pass
114
115 def namelist(self):
116 """Generate a list of filenames, as expected from ZipFile.nameslist().
117 """
118
119 result = []
120 for parent, directories, files in os.walk(self._path):
121 for filename in files:
122 file_path = os.path.join(parent, filename)
123 rel_path = os.path.relpath(file_path, self._path)
124 result.append(rel_path)
125 return result
126
127 def read(self, path):
128 content = None
129 with open(os.path.join(self._path, path)) as fp:
130 content = fp.read()
131 return content
132
133
134 def copy_metadata(filename, tmpdir):
135 """Copy the used metadata to the used temporary directory."""
136 path = os.path.join(os.path.dirname(__file__), filename)
137 destination = str(tmpdir.join(filename))
138 shutil.copy(path, destination)
139
140
141 def run_webext_build(ext_type, build_opt, srcdir, buildnum=None, keyfile=None):
142 """Run a build process."""
143 if build_opt == 'build':
144 build_args = ['build']
145 elif build_opt == 'release':
146 build_args = ['build', '-r']
147 else:
148 build_args = ['devenv']
149
150 args = ['build.py', '-t', ext_type] + build_args
151
152 if keyfile:
153 args += ['-k', keyfile]
154 if buildnum:
155 args += ['-b', buildnum]
156
157 processArgs(str(srcdir), args)
158
159
160 def locale_files(languages, rootfolder, srcdir):
161 """Generate example locales.
162
163 languages: tuple, list or set of languages to include
164 rootdir: folder-name to create the locale-folders in
165 """
166 for lang in languages:
167 subfolders = rootfolder.split(os.pathsep) + [lang, 'messages.json']
168 json_file = srcdir.ensure(*subfolders)
169 if lang == defaultLocale:
170 json_file.write(MESSAGES_EN_US)
171 else:
172 json_file.write('{}')
173
174
175 def assert_all_locales_present(package, prefix):
176 names = {x for x in package.namelist() if
177 x.startswith(os.path.join(prefix, '_locales'))}
178 assert names == {
179 os.path.join(prefix, '_locales', lang, 'messages.json')
180 for lang in ALL_LANGUAGES
181 }
182
183
184 @pytest.fixture
185 def srcdir(tmpdir):
186 """Source directory for building the package."""
187 for size in ['44', '50', '150']:
188 path = tmpdir.join('chrome', 'icons', 'abp-{}.png'.format(size))
189 path.write(size, ensure=True)
190
191 tmpdir.join('bar.json').write(json.dumps({}))
192 return tmpdir
193
194
195 @pytest.fixture
196 def gecko_import(tmpdir):
197 tmpdir.mkdir('_imp').mkdir('en-US').join('gecko.dtd').write(DTD_TEST)
198
199
200 @pytest.fixture
201 def locale_modules(tmpdir):
202 mod_dir = tmpdir.mkdir('_modules')
203 lang_dir = mod_dir.mkdir('en-US')
204 lang_dir.join('module.json').write(json.dumps(LOCALES_MODULE))
205 lang_dir.join('unit.properties').write(json.dumps(PROPERTIES_TEST))
206
207
208 @pytest.fixture
209 def icons(srcdir):
210 icons_dir = srcdir.mkdir('icons')
211 for filename in ['abp-16.png', 'abp-19.png', 'abp-53.png']:
212 shutil.copy(
213 os.path.join(os.path.dirname(__file__), filename),
214 os.path.join(str(icons_dir), filename),
215 )
216
217
218 @pytest.fixture
219 def all_lang_locales(tmpdir):
220 return locale_files(ALL_LANGUAGES, '_locales', tmpdir)
221
222
223 @pytest.fixture
224 def chrome_metadata(tmpdir):
225 filename = 'metadata.chrome'
226 copy_metadata(filename, tmpdir)
227
228
229 @pytest.fixture
230 def gecko_webext_metadata(tmpdir, chrome_metadata):
231 filename = 'metadata.gecko'
232 copy_metadata(filename, tmpdir)
233
234
235 @pytest.fixture
236 def edge_metadata(tmpdir):
237 filename = 'metadata.edge'
238 copy_metadata(filename, tmpdir)
239
240 return packager.readMetadata(str(tmpdir), 'edge')
241
242
243 @pytest.fixture
244 def keyfile(tmpdir):
245 """Test-privatekey for signing chrome release-package"""
246 return os.path.join(os.path.dirname(__file__), 'chrome_rsa.pem')
247
248
249 @pytest.fixture
250 def lib_files(tmpdir):
251 files = packager.Files(['lib'], set())
252 files['ext/a.js'] = 'var bar;'
253 files['lib/b.js'] = 'var foo;'
254
255 tmpdir.mkdir('lib').join('b.js').write(files['lib/b.js'])
256 tmpdir.mkdir('ext').join('a.js').write(files['ext/a.js'])
257
258 return files
259
260
261 def comparable_xml(xml):
262 """Create a nonambiguous representation of a given XML tree."""
263 def get_leafs_string(tree):
264 """Recursively build a string representing all xml leaf-nodes."""
265 root_str = '{}|{}|{}'.format(tree.tag, tree.tail, tree.text).strip()
266 result = []
267
268 if len(tree) > 0:
269 for subtree in tree:
270 for leaf in get_leafs_string(subtree):
271 result.append('{}__{}'.format(root_str, leaf))
272 for k, v in tree.attrib.items():
273 result.append('{}__{}:{}'.format(root_str, k, v))
274 else:
275 result.append(root_str)
276 return result
277
278 # XML data itself shall not be sorted, hence we can safely sort
279 # our string representations in order to produce a valid unified diff.
280 return sorted(get_leafs_string(ElementTree.fromstring(xml)))
281
282
283 def comparable_json(json_string):
284 """Create a nonambiguous representation of a given JSON string."""
285 return json.dumps(
286 json.loads(json_string), sort_keys=True, indent=0
287 ).split('\n')
288
289
290 def assert_manifest_content(manifest, expected_path):
291 extension = os.path.basename(expected_path).split('.')[-1]
292
293 with open(expected_path, 'r') as fp:
294 if extension == 'xml':
295 generated = comparable_xml(manifest)
296 expected = comparable_xml(fp.read())
297 else:
298 generated = comparable_json(manifest)
299 expected = comparable_json(fp.read())
300
301 diff = list(difflib.unified_diff(generated, expected, n=0))
302 assert len(diff) == 0, '\n'.join(diff)
303
304
305 def assert_gecko_locale_conversion(package, prefix):
306 locale = json.loads(package.read(os.path.join(
307 prefix, '_locales/en_US/messages.json')))
308
309 assert locale.get('test_Foobar', {}) == LOCALES_MODULE['test.Foobar']
310 assert locale.get('access_key', {}) == {'message': 'access key'}
311 assert locale.get('ampersand', {}) == {'message': 'foo -bar'}
312 assert locale.get('_description', {}) == {'message': 'very descriptive!"'}
313
314
315 def assert_convert_js(package, prefix, excluded=False):
316 libfoo = package.read(os.path.join(prefix, 'lib/foo.js'))
317
318 assert 'var bar;' in libfoo
319 assert 'require.modules["ext_a"]' in libfoo
320
321 assert ('var foo;' in libfoo) != excluded
322 assert ('require.modules["b"]' in libfoo) != excluded
323
324
325 def assert_devenv_scripts(package, devenv):
326 manifest = json.loads(package.read('manifest.json'))
327 filenames = package.namelist()
328 scripts = [
329 'ext/common.js',
330 'ext/background.js',
331 ]
332
333 assert ('qunit/index.html' in filenames) == devenv
334 assert ('devenvPoller__.js' in filenames) == devenv
335 assert ('devenvVersion__' in filenames) == devenv
336
337 if devenv:
338 assert '../ext/common.js' in package.read('qunit/index.html')
339 assert '../ext/background.js' in package.read('qunit/index.html')
340
341 assert set(manifest['background']['scripts']) == set(
342 scripts + ['devenvPoller__.js']
343 )
344 else:
345 assert set(manifest['background']['scripts']) == set(scripts)
346
347
348 def assert_base_files(package, platform):
349 filenames = set(package.namelist())
350
351 if platform in {'chrome', 'gecko'}:
352 assert 'bar.json' in filenames
353 assert 'manifest.json' in filenames
354 assert 'lib/foo.js' in filenames
355 assert 'foo/logo_50.png' in filenames
356 assert 'icons/logo_150.png' in filenames
357 else:
358 assert 'AppxManifest.xml' in filenames
359 assert 'AppxBlockMap.xml' in filenames
360 assert '[Content_Types].xml' in filenames
361
362 assert 'Extension/bar.json' in filenames
363 assert 'Extension/lib/foo.js' in filenames
364
365 assert package.read('Assets/logo_44.png') == '44'
366 assert package.read('Extension/icons/abp-44.png') == '44'
367
368
369 def assert_chrome_signature(filename, keyfile):
370 with open(filename, 'r') as fp:
371 content = fp.read()
372
373 _, _, l_pubkey, l_signature = unpack('<4sIII', content[:16])
374 signature = content[16 + l_pubkey: 16 + l_pubkey + l_signature]
375
376 digest = SHA.new()
377 with open(keyfile, 'r') as fp:
378 rsa_key = RSA.importKey(fp.read())
379
380 signer = PKCS1_v1_5.new(rsa_key)
381
382 digest.update(content[16 + l_pubkey + l_signature:])
383 assert signer.verify(digest, signature)
384
385
386 def assert_locale_upfix(package):
387 translations = [
388 json.loads(package.read('_locales/{}/messages.json'.format(lang)))
389 for lang in ALL_LANGUAGES
390 ]
391
392 manifest = package.read('manifest.json')
393
394 # Chrome Web Store requires descriptive translations to be present in
395 # every language.
396 for match in re.finditer(r'__MSG_(\S+)__', manifest):
397 name = match.group(1)
398
399 for other in translations[1:]:
400 assert translations[0][name]['message'] == other[name]['message']
401
402
403 @pytest.mark.usefixtures(
404 'all_lang_locales',
405 'locale_modules',
406 'gecko_import',
407 'icons',
408 'lib_files',
409 'chrome_metadata',
410 'gecko_webext_metadata',
411 'edge_metadata',
412 )
413 @pytest.mark.parametrize('platform,dev_build_release,buildnum', [
414 ('chrome', 'build', '1337'),
415 ('chrome', 'devenv', None),
416 ('chrome', 'release', None),
417 ('gecko', 'build', '1337'),
418 ('gecko', 'devenv', None),
419 ('gecko', 'release', None),
420 ('edge', 'build', '1337'),
421 pytest.param('edge', 'devenv', None, marks=pytest.mark.xfail),
422 ('edge', 'release', None),
423 ])
424 def test_build_webext(platform, dev_build_release, keyfile, tmpdir, srcdir,
425 buildnum, capsys):
426 release = dev_build_release == 'release'
427 devenv = dev_build_release == 'devenv'
428
429 if platform == 'chrome' and release:
430 key = keyfile
431 else:
432 key = None
433
434 manifests = {
435 'gecko': [('', 'manifest', 'json')],
436 'chrome': [('', 'manifest', 'json')],
437 'edge': [('', 'AppxManifest', 'xml'),
438 ('Extension', 'manifest', 'json')],
439 }
440
441 filenames = {
442 'gecko': 'adblockplusfirefox-1.2.3{}.xpi',
443 'chrome': 'adblockpluschrome-1.2.3{{}}.{}'.format(
444 {True: 'crx', False: 'zip'}[release]
445 ),
446 'edge': 'adblockplusedge-1.2.3{}.appx',
447 }
448
449 if platform == 'edge':
450 prefix = 'Extension'
451 else:
452 prefix = ''
453
454 run_webext_build(platform, dev_build_release, srcdir, buildnum=buildnum,
455 keyfile=key)
456
457 # The makeIcons() in packagerChrome.py should warn about non-square
458 # icons via stderr.
459 if platform in {'chrome', 'gecko'}:
460 out, err = capsys.readouterr()
461 assert 'icon should be square' in err
462
463 if devenv:
464 content_class = DirContent
465 out_file_path = os.path.join(str(srcdir), 'devenv.' + platform)
466 else:
467 content_class = ZipContent
468
469 if release:
470 add_version = ''
471 else:
472 add_version = '.' + buildnum
473
474 out_file = filenames[platform].format(add_version)
475
476 out_file_path = os.path.abspath(os.path.join(
477 os.path.dirname(__file__), os.pardir, out_file))
478 assert os.path.exists(out_file_path)
479
480 if release and platform == 'chrome':
481 assert_chrome_signature(out_file_path, keyfile)
482
483 with content_class(out_file_path) as package:
484 assert_base_files(package, platform)
485 assert_all_locales_present(package, prefix)
486 assert_gecko_locale_conversion(package, prefix)
487 assert_convert_js(package, prefix, platform == 'gecko')
488
489 if platform == 'chrome':
490 assert_locale_upfix(package)
491 if platform in {'chrome', 'gecko'}:
492 assert_devenv_scripts(package, devenv)
493
494 for folder, name, ext in manifests[platform]:
495 filename = '{{}}_{}_{}.{{}}'.format(platform, dev_build_release)
496 expected = os.path.join(
497 os.path.dirname(__file__),
498 'expecteddata',
499 filename.format(name, ext),
500 )
501
502 assert_manifest_content(
503 package.read(os.path.join(folder, '{}.{}'.format(name, ext))),
504 expected,
505 )
OLDNEW
« no previous file with comments | « tests/test_packagerEdge.py ('k') | tox.ini » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld