Issue 5383 - Add tests for the Chrome and Firefox packagers

tests/test_packagerWebExt.py

Index: tests/test_packagerWebExt.py
diff --git a/tests/test_packagerWebExt.py b/tests/test_packagerWebExt.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ea271324cdf4f9a4e43cf15f94601ebcfd4c844
--- /dev/null
+++ b/tests/test_packagerWebExt.py
@@ -0,0 +1,274 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+import json
+import re
+from struct import unpack
+
+import pytest
+from Crypto.Hash import SHA
+from Crypto.PublicKey import RSA
+from Crypto.Signature import PKCS1_v1_5
+
+
+from buildtools import packager, packagerChrome
+from buildtools.tests.tools import (DirContent, ZipContent, copy_metadata,
+                                    run_webext_build, assert_manifest_content,
+                                    assert_all_locales_present, locale_files)
+from buildtools.tests.conftest import ALL_LANGUAGES
+
+
+LOCALES_MODULE = {
+    'test.Foobar': {
+        'message': 'Ensuring dict-copy from modules for $domain$',
+        'description': 'test description',
+        'placeholders': {'content': '$1', 'example': 'www.adblockplus.org'}
+    }
+}
+
+DTD_TEST = ('<!ENTITY access.key "access key(&amp;a)">'
+            '<!ENTITY ampersand "foo &amp;-bar">')
+
+PROPERTIES_TEST = 'description=very descriptive!'
+
+
+@pytest.fixture
+def gecko_import(tmpdir):
+    tmpdir.mkdir('_imp').mkdir('en-US').join('gecko.dtd').write(DTD_TEST)
+
+
+@pytest.fixture
+def locale_modules(tmpdir):
+    mod_dir = tmpdir.mkdir('_modules')
+    lang_dir = mod_dir.mkdir('en-US')
+    lang_dir.join('module.json').write(json.dumps(LOCALES_MODULE))
+    lang_dir.join('unit.properties').write(json.dumps(PROPERTIES_TEST))
+
+
+@pytest.fixture
+def icons(srcdir):
+    icons_dir = srcdir.mkdir('icons')
+    for filename in ['abp-16.png', 'abp-19.png', 'abp-53.png']:
+        shutil.copy(
+            os.path.join(os.path.dirname(__file__), filename),
+            os.path.join(str(icons_dir), filename),
+        )
+
+
+@pytest.fixture
+def all_lang_locales(tmpdir):
+    return locale_files(ALL_LANGUAGES, '_locales', tmpdir)
+
+
+@pytest.fixture
+def chrome_metadata(tmpdir):
+    filename = 'metadata.chrome'
+    copy_metadata(filename, tmpdir)
+
+
+@pytest.fixture
+def gecko_webext_metadata(tmpdir, chrome_metadata):
+    filename = 'metadata.gecko-webext'
+    copy_metadata(filename, tmpdir)
+
+
+@pytest.fixture
+def keyfile(tmpdir):
+    """Test-privatekey for signing chrome release-package"""
+    return os.path.join(os.path.dirname(__file__), 'chrome_rsa.pem')
+
+
+@pytest.fixture
+def lib_files(tmpdir):
+    files = packager.Files(['lib'], set())
+    files['ext/a.js'] = 'var bar;'
+    files['lib/b.js'] = 'var foo;'
+
+    tmpdir.mkdir('lib').join('b.js').write(files['lib/b.js'])
+    tmpdir.mkdir('ext').join('a.js').write(files['ext/a.js'])
+
+    return files
+
+
+def assert_gecko_locale_conversion(package):
+    locale = json.loads(package.read('_locales/en_US/messages.json'))
+
+    assert locale.get('test_Foobar', {}) == LOCALES_MODULE['test.Foobar']
+    assert locale.get('access_key', {}) == {'message': 'access key'}
+    assert locale.get('ampersand', {}) == {'message': 'foo -bar'}
+    assert locale.get('_description', {}) == {'message': 'very descriptive!"'}
+
+
+def assert_convert_js(package, excluded=False):
+    libfoo = package.read('lib/foo.js')
+
+    assert 'var bar;' in libfoo
+    assert 'require.modules["ext_a"]' in libfoo
+
+    assert ('var foo;' in libfoo) != excluded
+    assert ('require.modules["b"]' in libfoo) != excluded
+
+
+def assert_devenv_scripts(package, devenv):
+    manifest = json.loads(package.read('manifest.json'))
+    filenames = package.namelist()
+    scripts = [
+            'ext/common.js',
+            'ext/background.js',
+    ]
+
+    if devenv:
+        assert 'qunit/index.html' in filenames
+        assert 'devenvPoller__.js' in filenames
+        assert 'devenvVersion__' in filenames
+
+        assert '../ext/common.js' in package.read('qunit/index.html')
+        assert '../ext/background.js' in package.read('qunit/index.html')
+
+        assert set(manifest['background']['scripts']) == set(
+            scripts + ['devenvPoller__.js']
+        )
+    else:
+        assert 'qunit/index.html' not in filenames
+
+        assert set(manifest['background']['scripts']) == set(scripts)
+
+
+def assert_base_files(package):
+    filenames = set(package.namelist())
+
+    assert 'bar.json' in filenames
+    assert 'manifest.json' in filenames
+    assert 'lib/foo.js' in filenames
+    assert 'foo/logo_50.png' in filenames
+    assert 'icons/logo_150.png' in filenames
+
+
+def assert_chrome_signature(filename, keyfile):
+    with open(filename, 'r') as fp:
+        content = fp.read()
+
+    _, _, l_pubkey, l_signature = unpack('<4sIII', content[:16])
+    signature = content[16 + l_pubkey: 16 + l_pubkey + l_signature]
+
+    digest = SHA.new()
+    with open(keyfile, 'r') as fp:
+        rsa_key = RSA.importKey(fp.read())
+
+    signer = PKCS1_v1_5.new(rsa_key)
+
+    digest.update(content[16 + l_pubkey + l_signature:])
+    assert signer.verify(digest, signature)
+
+
+def assert_locale_upfix(package):
+    translations = [
+        json.loads(package.read('_locales/{}/messages.json'.format(lang)))
+        for lang in ALL_LANGUAGES
+    ]
+
+    manifest = package.read('manifest.json')
+
+    # Chrome Web Store requires descriptive translations to be present in
+    # every language.
+    for match in re.finditer(r'__MSG_(\S+)__', manifest):
+        name = match.group(1)
+
+        for other in translations[1:]:
+            assert translations[0][name]['message'] == other[name]['message']
+
+
+@pytest.mark.usefixtures(
+    'all_lang_locales',
+    'locale_modules',
+    'gecko_import',
+    'icons',
+    'lib_files',
+    'chrome_metadata',
+    'gecko_webext_metadata',
+)
+@pytest.mark.parametrize('platform', ['chrome', 'gecko-webext'])
+@pytest.mark.parametrize('dev_build_release,buildnum', [
+    ('build', None),
+    ('build', '1337'),
+    ('devenv', None),
+    pytest.param('devenv', '1337', marks=pytest.mark.xfail),
+    ('release', None),
+    ('release', '1337'),
+])
+def test_build_webext(platform, dev_build_release, keyfile, tmpdir, srcdir,
+                      buildnum, capsys):
+    release = dev_build_release == 'release'
+    devenv = dev_build_release == 'devenv'

[Sebastian Noack, 2017/09/20 00:58:02]

There are actually 3 scenarios:

1. build.py -t {chrome|gecko-webext|edge} build -r
2. build.py -t {chrome|gecko-webext|edge} build -b <revision>
3. build.py -t {chrome|gecko-webext|edge} devenv

#1 generates release builds, and #3 is for the local development environment,
you seem to handle only these two scenarios.

However, #2 is what sitescripts is invoking, in order to generate our
development builds, this scenario doesn't seem to be handled.

[tlucas, 2017/09/20 08:52:51]

You are correct about #1 and #3 - however, there is no for devenv for edge (yet,
see #4720).

Regarding #2: the run_webext_build (tests/tools.py:110) appends '-b <num>' to
the arguments interpreted by processArgs from build.py, in case buildnum is set
(which is true for every second parameter combination in this test, e.g.
'dev_build_release,buildnum' = ('build', '1337')).

Effectively, a './build.py -t <platform> build -b 1337' is indeed run for all
three platforms - chrome, gecko-webext and edge. Is this what you meant or am i
missing something?

[Sebastian Noack, 2017/09/20 21:36:09]

Sorry, I misinterpreted the logic here. If I understand it correctly now, these
are the command lines you are currently testing, here:

build.py -t chrome build
build.py -t chrome build -b 1337
build.py -t chrome devenv
build.py -t chrome devenv -b 1337
build.py -t chrome release -k chrome_rsa.pem
build.py -t chrome release -k chrome_rsa.pem -b 1337
build.py -t gecko-webext build
build.py -t gecko-webext build -b 1337
build.py -t gecko-webext devenv
build.py -t gecko-webext devenv -b 1337
build.py -t gecko-webext release
build.py -t gecko-webext release -b 1337

This made me wonder how the tests even pass, since the -b option is only
supported by the "build" command, and the "release" command doesn't support the
"gecko-webext" target at all. It turned out that you are actually not calling
the "release" command, which makes sense as we certainly don't want to have the
tests perform parts of the actual release process online, which would fail
anyway if you don't have push access to the downloads repository. At some point
we probably would want to have tests for the release automation though,
potentially involving some monkey patching, but this is out of scope here.
However, it doesn't make much sense either to call the internal API generating
the release build instead (in a quite obfuscated way). Instead you should rather
run "build -r" which does exactly that, i.e. generating the release build.

Furthermore, testing the "build" command with neither the -b nor the -r option
seems redundant, as it is essentially the same as "build -b 1337", just that the
revision for the devbuild is extracted from the repository. This logic, however,
is already covered by the "devenv" command, and more importantly won't work in
the test environment anyway (falling back to 0), since the directory you build
the test extension out of isn't a Mercurial or Git repository. Not to mention
that his is a rather theoretical use case, that we don't have in practice.

So that leaves us with following command lines, that should be tested instead,
for the "chrome" and "gecko-webext" targets:

build.py -t chrome build -r -k chrome_rsa.pem
build.py -t chrome build -b 1337
build.py -t chrome devenv
build.py -t gecko-webext build -r
build.py -t gecko-webext build -b 1337
build.py -t gecko-webext devenv

Respectively, following command lines need to be tested for the "edge" target:

build.py -t edge build -r
build.py -t edge build -b 1337
build.py -t edge devenv

Yes, the "devenv" command doesn't support the "edge" target at the moment. But
this is matter to change. So feel free to mark this test as xfail, for now.

+
+    if platform == 'chrome' and release:
+        key = keyfile
+    else:
+        key = None
+
+    filenames = {
+        'gecko-webext': 'adblockplusfirefox-1.2.3{}.{}',

[Vasily Kuznetsov, 2017/09/19 17:52:27]

I would probably just use the structure of `extensions` nested dict here instead
and inline the extensions. Then the `extensions` dict could be eliminated
together with some code.

[tlucas, 2017/09/20 08:52:52]

Done.

+        'chrome': 'adblockpluschrome-1.2.3{}.{}'
+    }
+
+    extensions = {
+        'gecko-webext': {
+            True: 'xpi',
+            False: 'xpi',
+        },
+        'chrome': {
+            True: 'crx',
+            False: 'zip',
+        },
+    }
+
+    expected_manifest = os.path.join(
+        os.path.dirname(__file__),
+        'expecteddata',
+        'manifest_{}_{}_{}.json'.format(platform, dev_build_release, buildnum),
+    )
+
+    run_webext_build(platform, dev_build_release, srcdir, packagerChrome,
+                     buildnum=buildnum, keyfile=key)
+
+    # The makeIcons() in packagerChrome.py should warn about non-square
+    # icons via stderr.
+    out, err = capsys.readouterr()
+    assert 'icon should be square' in err
+
+    if devenv:
+        content_class = DirContent
+        out_file_path = os.path.join(str(srcdir), 'devenv.' + platform)
+    else:
+        content_class = ZipContent
+
+        if release:
+            add_version = ''
+        else:
+            add_version = '.' + (buildnum or '0')
+        extension = extensions[platform][release]
+
+        out_file = filenames[platform].format(add_version, extension)
+
+        out_file_path = os.path.abspath(os.path.join(
+            os.path.dirname(__file__), os.pardir, out_file))
+        assert os.path.exists(out_file_path)
+
+    if release and platform == 'chrome':
+        assert_chrome_signature(out_file_path, keyfile)
+
+    with content_class(out_file_path) as package:
+        assert_manifest_content(
+            package.read('manifest.json'), expected_manifest
+        )
+        assert_base_files(package)
+        assert_devenv_scripts(package, devenv)
+        assert_all_locales_present(package, '_locales')
+        assert_gecko_locale_conversion(package)
+        assert_convert_js(package, platform == 'gecko-webext')
+
+        if platform == 'chrome':
+            assert_locale_upfix(package)