Issue 5934 - CMS testing automation

cms-dev/cms_cmp.py

 #!/bin/env python
 # This file is part of Adblock Plus <https://adblockplus.org/>,
 # Copyright (C) 2017-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ skipping 64 matching lines @@
 
 def print_diff(one, two):
     """Print unified diff between two files."""
     for line in difflib.unified_diff(read_file(one), read_file(two),
                                      fromfile=one, tofile=two):
         print(line)
 
 
 def compare_dirs(one, two, ignore=[]):
     """Compare two directories, return True if same, False if not."""
-
     def recursive_compare(c):
         if c.left_only:
             print('The following file(s)/dir(s) are only in base', c.left)
             for f in c.left_only:
                 print('-', f)
             return False
         if c.right_only:
             print('The following file(s)/dir(s) are only in test', c.right)
             for f in c.right_only:
                 print('-', f)
             return False
         if c.diff_files:
             print('The following file(s) are different between', c.left,
                   'and', c.right)
             for f in c.diff_files:
                 print('-', f)
                 base = os.path.join(c.left, f)
                 test = os.path.join(c.right, f)
                 print_diff(base, test)
             return False
         return all(recursive_compare(sub) for sub in c.subdirs.values())
 
     print('Comparing', one, 'and', two)
     comparator = filecmp.dircmp(one, two, ignore=ignore)
     return recursive_compare(comparator)
 
 
 class Tester(object):
-    """Test runner."""
-
-    def __init__(self, config):
-        self.__dict__.update(config.__dict__)
+    """Test runner.
+
+    Generates one or more websites with two different versions of CMS and
+    compares the results.
+    """
+
+    def __init__(self, website_paths, cms_repo, dest, ignore=[],
+                 python=sys.executable, remove_old=False,
+                 base_rev='master', test_rev=WORKING_COPY):
+        """Create test runner.
+
+        Parameters
+        ----------
+        website_paths : list of strings
+            Paths of the website source folders that are used for comparisons.
+        cms_repo : str
+            Path to CMS repository.
+        dest : str
+            Directory into which CMS outputs will be placed.
+        ignore : list of strings
+            List of file names to ignore in output comparision.
+        python : str
+            Path to the python interpeter.
+        remove_old : bool
+            Remove results of earlier runs of this script? Setting this to
+            `True` will make things slower but will ensure correctness.
+        base_rev : str
+            Revision of CMS to use as a baseline.
+        test_rev : str
+            Revision of CMS to use for testing.
+
+        """
+        self.website_paths = website_paths
+        self.cms_repo = cms_repo
+        self.dest = dest
+        self.ignore = ignore
+        self.python = python
+        self.remove_old = remove_old
+        self.base_rev = base_rev
+        self.test_rev = test_rev
 
     def clone_cms(self):
         """Clone CMS repository for to use for tests."""
         self.cms_clone = os.path.join(self.dest, 'cms-cmp.cms-clone')
         print('Cloning CMS to', self.cms_clone)
         if os.path.exists(self.cms_clone):
             shutil.rmtree(self.cms_clone)
         hg('clone', self.cms_repo, self.cms_clone)
 
     def cms_checkout(self, rev):
@@ skipping 22 matching lines @@
             else:
                 print(dst, 'exists, assuming it was generated earlier')
                 return dst
         env = dict(os.environ)
         env['PYTHONPATH'] = cms_path
         generate = os.path.join(cms_path, GENERATE_PATH)
         run_cmd(self.python, generate, website_path, dst, env=env)
         return dst
 
     def run(self):
+        """Run the comparison."""
         if not os.path.exists(os.path.join(self.cms_repo, GENERATE_PATH)):
             sys.exit('No cms source found in ' + self.cms_repo)
         print('Using CMS repository at', self.cms_repo)
         self.clone_cms()
-        for website_path in self.websites:
+        for website_path in self.website_paths:
             base = self.generate(self.base_rev, website_path)
             test = self.generate(self.test_rev, website_path)
             if not compare_dirs(base, test, ignore=self.ignore):
                 print('Differences found for', website_path)
                 sys.exit(1)
 
 
 def configure():

[tlucas, 2017/11/24 10:57:46]

Since configure()'s only purpose is to specify parameters for Tester, what do
you think about making it a Tester method, called within __init__()?

[Vasily Kuznetsov, 2017/11/28 16:53:37]

I would like to keep `Tester` independent from particulars of option parsing, so
my preference would be to keep the option parsing out of it. You could say that
it's not really independent since it expects an `argparse` namespace in the
constructor, and I would agree with this. Still, I would prefer to go towards
greater separation of option parsing from CMS running and comparisons, perhaps
via passing keyword arguments to `Tester.__init__` instead of an option
namespace.

[tlucas, 2017/12/14 10:30:42]

ACK, your argumentation seems valid :)

     """Configure the script from arguments."""
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('websites', metavar='WEBSITE', nargs='+',
+    parser.add_argument('website_paths', metavar='WEBSITE', nargs='+',
                         help='path of website source to use for comparison')
     parser.add_argument('-b', '--base-rev', metavar='REVISION',
                         default='master',
                         help='base revision of CMS to use as baseline')
     parser.add_argument('-c', '--cms-repo', metavar='CMS_REPO',
                         default=os.getcwd(),
                         help='Location of CMS repository')
     parser.add_argument('-d', '--dest', metavar='OUT_DIR',
                         default=os.environ.get('TMPDIR', '/tmp'),
                         help='directory for storing CMS output')
     parser.add_argument('-i', '--ignore', metavar='FILENAME', action='append',
                         default=[],
                         help='file names to ignore in output comparison')
     parser.add_argument('-p', '--python', metavar='PYTHON_EXE',
                         default=sys.executable,
                         help='python interpreter to run CMS')
     parser.add_argument('-r', '--remove-old', action='store_true',
                         help='remove previously generated CMS outputs instead'
                              'of reusing them')
     parser.add_argument('-t', '--test-rev', metavar='REVISION',
                         default=WORKING_COPY,
                         help='revision of CMS to use for testing (by default '
                              'currently checked out working copy including '
                              'uncommited changes will be used)')
     return parser.parse_args()
 
 
 def main():
-    """Main entry point for the script."""
+    """Parse command line arguments and run the tests.
+
+    Main entry point for the script.
+    """
     config = configure()
-    tester = Tester(config)
+    tester = Tester(**config.__dict__)
     tester.run()
 
 
 if __name__ == '__main__':
     main()