Issue 5934 - CMS testing automation

cms-dev/cms_cmp.py

Index: cms-dev/cms_cmp.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/cms-dev/cms_cmp.py
@@ -0,0 +1,211 @@
+#!/bin/env python
+# This file is part of Adblock Plus <https://adblockplus.org/>,
+# Copyright (C) 2017-present eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test CMS by comparing the output of its different revisions."""
+
+from __future__ import print_function, unicode_literals
+
+import argparse
+import difflib
+import filecmp
+import io
+import os
+import shutil
+import subprocess
+import sys
+
+# Path to static generation script.
+GENERATE_PATH = 'cms/bin/generate_static_pages.py'
+
+# Fake revision that indicates "use current working copy".
+WORKING_COPY = '[working-copy]'
+
+
+def run_cmd(*cmd, **kw):
+    """Run a command, print and return its output."""
+    silent = kw.get('silent', False)
+    if 'silent' in kw:
+        del kw['silent']
+    if not silent:
+        print('$', *cmd)
+    try:
+        output = subprocess.check_output(cmd, **kw).decode('utf-8')
+    except subprocess.CalledProcessError:
+        sys.exit('Command invocation failed: {}'.format(' '.join(cmd)))
+    if not silent:
+        for line in output.splitlines():
+            print('>', line)
+    return output
+
+
+def hg(*args, **kw):
+    """Run Mercurial and return its output."""
+    cmd = ['hg']
+    if 'repo' in kw:
+        cmd += ['-R', kw['repo']]
+        del kw['repo']
+    # Disable default options from local user config.
+    cmd += ['--config', 'defaults.{}='.format(args[0])]
+    return run_cmd(*(cmd + list(args)), **kw)
+
+
+def get_current_rev(repo):
+    """Get the revision ids of the working copy in repository."""
+    return hg('id', repo=repo, silent=True).split()
+
+
+def read_file(path):
+    """Read file, return list of strings."""
+    with io.open(path, encoding='utf-8') as f:
+        return f.read().splitlines()
+
+
+def print_diff(one, two):
+    """Print unified diff between two files."""
+    for line in difflib.unified_diff(read_file(one), read_file(two),
+                                     fromfile=one, tofile=two):
+        print(line)
+
+
+def compare_dirs(one, two, ignore=[]):
+    """Compare two directories, return True if same, False if not."""
+
+    def recursive_compare(c):
+        if c.left_only:
+            print('The following file(s)/dir(s) are only in base', c.left)
+            for f in c.left_only:
+                print('-', f)
+            return False
+        if c.right_only:
+            print('The following file(s)/dir(s) are only in test', c.right)
+            for f in c.right_only:
+                print('-', f)
+            return False
+        if c.diff_files:
+            print('The following file(s) are different between', c.left,
+                  'and', c.right)
+            for f in c.diff_files:
+                print('-', f)
+                base = os.path.join(c.left, f)
+                test = os.path.join(c.right, f)
+                print_diff(base, test)
+            return False
+        return all(recursive_compare(sub) for sub in c.subdirs.values())
+
+    print('Comparing', one, 'and', two)
+    comparator = filecmp.dircmp(one, two, ignore=ignore)
+    return recursive_compare(comparator)
+
+
+class Tester(object):
+    """Test runner."""
+
+    def __init__(self, config):
+        self.__dict__.update(config.__dict__)
+
+    def clone_cms(self):
+        """Clone CMS repository for to use for tests."""
+        self.cms_clone = os.path.join(self.dest, 'cms-cmp.cms-clone')
+        print('Cloning CMS to', self.cms_clone)
+        if os.path.exists(self.cms_clone):
+            shutil.rmtree(self.cms_clone)
+        hg('clone', self.cms_repo, self.cms_clone)
+
+    def cms_checkout(self, rev):
+        """Checkout specified revision of CMS.
+
+        Returns revision hash (or working copy marker) and path to where it is.
+        """
+        if rev is WORKING_COPY:
+            print('Using CMS working copy')
+            return WORKING_COPY, self.cms_repo
+        print('Switching CMS to revision:', rev)
+        hg('co', rev, repo=self.cms_clone)
+        return get_current_rev(self.cms_clone)[0], self.cms_clone
+
+    def generate(self, cms_rev, website_path):
+        """Generate the website using specified revision of CMS."""
+        name = os.path.basename(website_path)
+        website_rev = get_current_rev(website_path)[0]
+        cms_rev, cms_path = self.cms_checkout(cms_rev)
+        print('Generating', website_path, 'with CMS revision:', cms_rev)
+        unique_id = '{}-rev-{}-cms-{}'.format(name, website_rev, cms_rev)
+        dst = os.path.join(self.dest, unique_id)
+        if os.path.exists(dst):
+            if self.remove_old or cms_rev == WORKING_COPY:
+                shutil.rmtree(dst)
+            else:
+                print(dst, 'exists, assuming it was generated earlier')
+                return dst
+        env = dict(os.environ)
+        env['PYTHONPATH'] = cms_path
+        generate = os.path.join(cms_path, GENERATE_PATH)
+        run_cmd(self.python, generate, website_path, dst, env=env)
+        return dst
+
+    def run(self):
+        if not os.path.exists(os.path.join(self.cms_repo, GENERATE_PATH)):
+            sys.exit('No cms source found in ' + self.cms_repo)
+        print('Using CMS repository at', self.cms_repo)
+        self.clone_cms()
+        for website_path in self.websites:
+            base = self.generate(self.base_rev, website_path)
+            test = self.generate(self.test_rev, website_path)
+            if not compare_dirs(base, test, ignore=self.ignore):
+                print('Differences found for', website_path)
+                sys.exit(1)
+
+
+def configure():

[tlucas, 2017/11/24 10:57:46]

Since configure()'s only purpose is to specify parameters for Tester, what do
you think about making it a Tester method, called within __init__()?

[Vasily Kuznetsov, 2017/11/28 16:53:37]

I would like to keep `Tester` independent from particulars of option parsing, so
my preference would be to keep the option parsing out of it. You could say that
it's not really independent since it expects an `argparse` namespace in the
constructor, and I would agree with this. Still, I would prefer to go towards
greater separation of option parsing from CMS running and comparisons, perhaps
via passing keyword arguments to `Tester.__init__` instead of an option
namespace.

[tlucas, 2017/12/14 10:30:42]

ACK, your argumentation seems valid :)

+    """Configure the script from arguments."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('websites', metavar='WEBSITE', nargs='+',
+                        help='path of website source to use for comparison')
+    parser.add_argument('-b', '--base-rev', metavar='REVISION',
+                        default='master',
+                        help='base revision of CMS to use as baseline')
+    parser.add_argument('-c', '--cms-repo', metavar='CMS_REPO',
+                        default=os.getcwd(),
+                        help='Location of CMS repository')
+    parser.add_argument('-d', '--dest', metavar='OUT_DIR',
+                        default=os.environ.get('TMPDIR', '/tmp'),
+                        help='directory for storing CMS output')
+    parser.add_argument('-i', '--ignore', metavar='FILENAME', action='append',
+                        default=[],
+                        help='file names to ignore in output comparison')
+    parser.add_argument('-p', '--python', metavar='PYTHON_EXE',
+                        default=sys.executable,
+                        help='python interpreter to run CMS')
+    parser.add_argument('-r', '--remove-old', action='store_true',
+                        help='remove previously generated CMS outputs instead'
+                             'of reusing them')
+    parser.add_argument('-t', '--test-rev', metavar='REVISION',
+                        default=WORKING_COPY,
+                        help='revision of CMS to use for testing (by default '
+                             'currently checked out working copy including '
+                             'uncommited changes will be used)')
+    return parser.parse_args()
+
+
+def main():
+    """Main entry point for the script."""
+    config = configure()
+    tester = Tester(config)
+    tester.run()
+
+
+if __name__ == '__main__':
+    main()