Issue 5934 - CMS testing automation

cms-dev/tests/test_cms_cmp.py

Index: cms-dev/tests/test_cms_cmp.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/cms-dev/tests/test_cms_cmp.py
@@ -0,0 +1,127 @@
+# This file is part of Adblock Plus <https://adblockplus.org/>,
+# Copyright (C) 2017-present eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import glob
+import os
+import subprocess
+import sys
+
+import pytest
+
+CMSCMP = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'cms_cmp.py')
+
+
+def run_cms_cmp(*args, **kw):
+    return subprocess.check_call([sys.executable, CMSCMP] + list(args), **kw)
+
+
+def hg(*args, **kw):

[tlucas, 2017/11/24 10:57:46]

This is a copy of cms_cmp.hg(), without the call of run_cmd() - is there any
specific reason, why you are not simply importing it?

Imho the tests would also benefit from the beautified error message you print in
cmp_cms.run_cmd().

[Vasily Kuznetsov, 2017/11/28 16:53:37]

I had in mind three arguments for implementing a separate `hg()` function here:
  1. We don't want to use the code under test for running the tests.
  2. `run_cmd()` throws `SystemExit` exception, which normally leads to process
exit.
  3. We need to set `PYTHONPATH` in order to import `cms_cmp.py` and that's kind
of ugly.

However, (1) is not a very strong argument and (2) is based on a faulty
assumption (actually, pytest catches `SystemExit` and reports it  as a test
failure, so the behavior of tests, in which `hg` fails, would not change). (3)
is valid, but should instead be solved by making this a python package (which I
thought was a bit of an overkill).

All in all, I'm leaning towards following your advice as soon as this is made
into a Python package, but perhaps not right now. What do you think?

[tlucas, 2017/12/14 10:30:42]

I agree, right now this is not a very urgent topic and yes, it would be nice to
someday have this as a python package.

+    cmd = ['hg']
+    if 'repo' in kw:
+        cmd += ['-R', kw['repo']]
+        del kw['repo']
+    # Disable default options from local user config.
+    cmd += ['--config', 'defaults.{}='.format(args[0])]
+    return subprocess.check_call(cmd + list(args), **kw)
+
+
+@pytest.fixture(scope='session')
+def website(tmpdir_factory):
+    root = tmpdir_factory.mktemp('website')
+    root.join('foo').write('foo')
+    hg('init', str(root))
+    hg('commit', '-A', '-m', 'x', repo=str(root))
+    return root
+
+
+@pytest.fixture(scope='session')
+def cms(tmpdir_factory):
+    root = tmpdir_factory.mktemp('cms')
+    generate = root.mkdir('cms').mkdir('bin').join('generate_static_pages.py')
+    generate.write('\n'.join([
+        'import sys, shutil',
+        'shutil.copytree(sys.argv[1], sys.argv[2])'
+    ]))
+    hg('init', str(root))
+    hg('commit', '-A', '-m', 'x', repo=str(root))
+    root.join('foo').write('bar')
+    hg('commit', '-A', '-m', 'y', repo=str(root))
+    generate.write(
+        generate.read() +
+        '\nshutil.copy(sys.argv[2] + "/foo", sys.argv[2] + "/bar")'
+    )
+    hg('commit', '-m', 'z', repo=str(root))
+    hg('bookmark', '-r', '0', 'master', repo=str(root))
+    hg('bookmark', '-r', '1', 'yoda', repo=str(root))
+    hg('bookmark', '-r', '2', 'other', repo=str(root))
+    return root
+
+
+def test_same_revision(website, cms, tmpdir):
+    run_cms_cmp('-d', str(tmpdir), '-c', str(cms),
+                '-b', 'master', '-t', 'yoda',
+                str(website))
+
+
+def test_different_revision(website, cms, tmpdir):
+    with pytest.raises(subprocess.CalledProcessError):
+        # Here the websites will be different, so the comparison will fail...
+        run_cms_cmp('-d', str(tmpdir), '-c', str(cms),
+                    '-b', 'master', '-t', 'other',
+                    str(website))
+    # ...but it will succeed if we ignore 'bar'.
+    run_cms_cmp('-d', str(tmpdir), '-c', str(cms),
+                '-b', 'master', '-t', 'other',
+                '-i', 'bar',
+                str(website))
+
+
+def test_bad_python(website, cms, tmpdir):
+    with pytest.raises(subprocess.CalledProcessError):
+        # The run should fail with a broken python...
+        run_cms_cmp('-d', str(tmpdir), '-c', str(cms),
+                    '-t', 'yoda', '-p', 'foobar',
+                    str(website))
+    # ...but succeed with a good one.
+    run_cms_cmp('-d', str(tmpdir), '-c', str(cms),
+                '-t', 'yoda', '-p', sys.executable,
+                str(website))
+
+
+def test_remove_old(website, cms, tmpdir):
+    run_cms_cmp('-d', str(tmpdir), '-c', str(cms), '-t', 'yoda',
+                str(website))
+    # Let's add a file to one of the output directories:
+    d = glob.glob(os.path.join(str(tmpdir), 'website*'))[0]
+    with open(os.path.join(d, 'baz'), 'w') as f:
+        f.write('here')
+    # Now the comparison will fail...
+    with pytest.raises(subprocess.CalledProcessError):
+        run_cms_cmp('-d', str(tmpdir), '-c', str(cms), '-t', 'yoda',
+                    str(website))
+    # ...but it will succeed if we tell it to delete old outputs.
+    run_cms_cmp('-d', str(tmpdir), '-c', str(cms), '-t', 'yoda', '-r',
+                str(website))
+
+
+def test_working_copy(website, cms, tmpdir):
+    with pytest.raises(subprocess.CalledProcessError):
+        # This will fail because surrent version is 'other'.

[tlucas, 2017/11/24 10:57:46]

typo "current"

[Vasily Kuznetsov, 2017/11/28 16:53:37]

Done.

+        run_cms_cmp('-d', str(tmpdir), '-c', str(cms), str(website))
+    generate = cms.join('cms').join('bin').join('generate_static_pages.py')
+    # Remove last line that breaks stuff.
+    generate.write('\n'.join(generate.read().splitlines()[:-1]))
+    # Now it should be better.
+    run_cms_cmp('-d', str(tmpdir), '-c', str(cms), str(website))