Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: cms-dev/cms_cmp.py

Issue 29588962: Issue 5934 - CMS testing automation (Closed)
Left Patch Set: Created Oct. 25, 2017, 4:48 p.m.
Right Patch Set: Improve documentation and structure following Tristan's comments Created Nov. 28, 2017, 4:11 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « no previous file | cms-dev/tests/test_cms_cmp.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 #!/bin/env python
2 # This file is part of Adblock Plus <https://adblockplus.org/>,
3 # Copyright (C) 2017-present eyeo GmbH
4 #
5 # Adblock Plus is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License version 3 as
7 # published by the Free Software Foundation.
8 #
9 # Adblock Plus is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16
17 """Test CMS by comparing the output of its different revisions."""
18
19 from __future__ import print_function, unicode_literals
20
21 import argparse
22 import difflib
23 import filecmp
24 import io
25 import os
26 import shutil
27 import subprocess
28 import sys
29
30 # Path to static generation script.
31 GENERATE_PATH = 'cms/bin/generate_static_pages.py'
32
33 # Fake revision that indicates "use current working copy".
34 WORKING_COPY = '[working-copy]'
35
36
37 def run_cmd(*cmd, **kw):
38 """Run a command, print and return its output."""
39 silent = kw.get('silent', False)
40 if 'silent' in kw:
41 del kw['silent']
42 if not silent:
43 print('$', *cmd)
44 try:
45 output = subprocess.check_output(cmd, **kw).decode('utf-8')
46 except subprocess.CalledProcessError:
47 sys.exit('Command invocation failed: {}'.format(' '.join(cmd)))
48 if not silent:
49 for line in output.splitlines():
50 print('>', line)
51 return output
52
53
54 def hg(*args, **kw):
55 """Run Mercurial and return its output."""
56 cmd = ['hg']
57 if 'repo' in kw:
58 cmd += ['-R', kw['repo']]
59 del kw['repo']
60 # Disable default options from local user config.
61 cmd += ['--config', 'defaults.{}='.format(args[0])]
62 return run_cmd(*(cmd + list(args)), **kw)
63
64
65 def get_current_rev(repo):
66 """Get the revision ids of the working copy in repository."""
67 return hg('id', repo=repo, silent=True).split()
68
69
70 def read_file(path):
71 """Read file, return list of strings."""
72 with io.open(path, encoding='utf-8') as f:
73 return f.read().splitlines()
74
75
76 def print_diff(one, two):
77 """Print unified diff between two files."""
78 for line in difflib.unified_diff(read_file(one), read_file(two),
79 fromfile=one, tofile=two):
80 print(line)
81
82
83 def compare_dirs(one, two, ignore=[]):
84 """Compare two directories, return True if same, False if not."""
85 def recursive_compare(c):
86 if c.left_only:
87 print('The following file(s)/dir(s) are only in base', c.left)
88 for f in c.left_only:
89 print('-', f)
90 return False
91 if c.right_only:
92 print('The following file(s)/dir(s) are only in test', c.right)
93 for f in c.right_only:
94 print('-', f)
95 return False
96 if c.diff_files:
97 print('The following file(s) are different between', c.left,
98 'and', c.right)
99 for f in c.diff_files:
100 print('-', f)
101 base = os.path.join(c.left, f)
102 test = os.path.join(c.right, f)
103 print_diff(base, test)
104 return False
105 return all(recursive_compare(sub) for sub in c.subdirs.values())
106
107 print('Comparing', one, 'and', two)
108 comparator = filecmp.dircmp(one, two, ignore=ignore)
109 return recursive_compare(comparator)
110
111
112 class Tester(object):
113 """Test runner.
114
115 Generates one or more websites with two different versions of CMS and
116 compares the results.
117 """
118
119 def __init__(self, website_paths, cms_repo, dest, ignore=[],
120 python=sys.executable, remove_old=False,
121 base_rev='master', test_rev=WORKING_COPY):
122 """Create test runner.
123
124 Parameters
125 ----------
126 website_paths : list of strings
127 Paths of the website source folders that are used for comparisons.
128 cms_repo : str
129 Path to CMS repository.
130 dest : str
131 Directory into which CMS outputs will be placed.
132 ignore : list of strings
133 List of file names to ignore in output comparision.
134 python : str
135 Path to the python interpeter.
136 remove_old : bool
137 Remove results of earlier runs of this script? Setting this to
138 `True` will make things slower but will ensure correctness.
139 base_rev : str
140 Revision of CMS to use as a baseline.
141 test_rev : str
142 Revision of CMS to use for testing.
143
144 """
145 self.website_paths = website_paths
146 self.cms_repo = cms_repo
147 self.dest = dest
148 self.ignore = ignore
149 self.python = python
150 self.remove_old = remove_old
151 self.base_rev = base_rev
152 self.test_rev = test_rev
153
154 def clone_cms(self):
155 """Clone CMS repository for to use for tests."""
156 self.cms_clone = os.path.join(self.dest, 'cms-cmp.cms-clone')
157 print('Cloning CMS to', self.cms_clone)
158 if os.path.exists(self.cms_clone):
159 shutil.rmtree(self.cms_clone)
160 hg('clone', self.cms_repo, self.cms_clone)
161
162 def cms_checkout(self, rev):
163 """Checkout specified revision of CMS.
164
165 Returns revision hash (or working copy marker) and path to where it is.
166 """
167 if rev is WORKING_COPY:
168 print('Using CMS working copy')
169 return WORKING_COPY, self.cms_repo
170 print('Switching CMS to revision:', rev)
171 hg('co', rev, repo=self.cms_clone)
172 return get_current_rev(self.cms_clone)[0], self.cms_clone
173
174 def generate(self, cms_rev, website_path):
175 """Generate the website using specified revision of CMS."""
176 name = os.path.basename(website_path)
177 website_rev = get_current_rev(website_path)[0]
178 cms_rev, cms_path = self.cms_checkout(cms_rev)
179 print('Generating', website_path, 'with CMS revision:', cms_rev)
180 unique_id = '{}-rev-{}-cms-{}'.format(name, website_rev, cms_rev)
181 dst = os.path.join(self.dest, unique_id)
182 if os.path.exists(dst):
183 if self.remove_old or cms_rev == WORKING_COPY:
184 shutil.rmtree(dst)
185 else:
186 print(dst, 'exists, assuming it was generated earlier')
187 return dst
188 env = dict(os.environ)
189 env['PYTHONPATH'] = cms_path
190 generate = os.path.join(cms_path, GENERATE_PATH)
191 run_cmd(self.python, generate, website_path, dst, env=env)
192 return dst
193
194 def run(self):
195 """Run the comparison."""
196 if not os.path.exists(os.path.join(self.cms_repo, GENERATE_PATH)):
197 sys.exit('No cms source found in ' + self.cms_repo)
198 print('Using CMS repository at', self.cms_repo)
199 self.clone_cms()
200 for website_path in self.website_paths:
201 base = self.generate(self.base_rev, website_path)
202 test = self.generate(self.test_rev, website_path)
203 if not compare_dirs(base, test, ignore=self.ignore):
204 print('Differences found for', website_path)
205 sys.exit(1)
206
207
208 def configure():
209 """Configure the script from arguments."""
210 parser = argparse.ArgumentParser(description=__doc__)
211 parser.add_argument('website_paths', metavar='WEBSITE', nargs='+',
212 help='path of website source to use for comparison')
213 parser.add_argument('-b', '--base-rev', metavar='REVISION',
214 default='master',
215 help='base revision of CMS to use as baseline')
216 parser.add_argument('-c', '--cms-repo', metavar='CMS_REPO',
217 default=os.getcwd(),
218 help='Location of CMS repository')
219 parser.add_argument('-d', '--dest', metavar='OUT_DIR',
220 default=os.environ.get('TMPDIR', '/tmp'),
221 help='directory for storing CMS output')
222 parser.add_argument('-i', '--ignore', metavar='FILENAME', action='append',
223 default=[],
224 help='file names to ignore in output comparison')
225 parser.add_argument('-p', '--python', metavar='PYTHON_EXE',
226 default=sys.executable,
227 help='python interpreter to run CMS')
228 parser.add_argument('-r', '--remove-old', action='store_true',
229 help='remove previously generated CMS outputs instead'
230 'of reusing them')
231 parser.add_argument('-t', '--test-rev', metavar='REVISION',
232 default=WORKING_COPY,
233 help='revision of CMS to use for testing (by default '
234 'currently checked out working copy including '
235 'uncommited changes will be used)')
236 return parser.parse_args()
237
238
239 def main():
240 """Parse command line arguments and run the tests.
241
242 Main entry point for the script.
243 """
244 config = configure()
245 tester = Tester(**config.__dict__)
246 tester.run()
247
248
249 if __name__ == '__main__':
250 main()
LEFTRIGHT

Powered by Google App Engine
This is Rietveld