Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: eyeo-depup/src/depup.py

Issue 29599579: OffTopic: DependencyUpdater
Left Patch Set: Addressing Dave's comments Created Nov. 13, 2017, 2:32 p.m.
Right Patch Set: Created Nov. 27, 2017, 9:40 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Right: Side by side diff | Download
LEFTRIGHT
(no file at all)
1 #!/usr/bin/env python
2
3 # This file is part of Adblock Plus <https://adblockplus.org/>,
4 # Copyright (C) 2006-present eyeo GmbH
5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation.
9 #
10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17
18 """Prepare a dependency update.
19
20 This script executes the automatable work which needs to be done for a
21 dependency update and provides additional information, i.e. a complete
22 diff of imported changes, as well as related integration notes.
23 """
24
25 from __future__ import print_function, unicode_literals
26
27 import argparse
28 import io
29 import json
30 import logging
31 import os
32 import re
33 import subprocess
34 try:
35 from urllib import urlopen
36 except ImportError:
37 from urllib.request import urlopen
38
39 import jinja2
40
41 from src.vcs import Vcs
42
43 logging.basicConfig()
44 logger = logging.getLogger('eyeo-depup')
45
46
47 class DepUpdate(object):
48 """The main class used to process dependency updates.
49
50 DepUpdate provides an argument parser and subcommand routing for the
51 commands 'diff', 'issue', 'changes' and 'commit'. Each command may require
52 it's own set of parameters and parse them accordingly.
53
54 When parsing is successufl and all required parameters are provided,
55 DepUpdate will execute the corresponding command and will either print the
56 output to STDOUT or to the given filename.
57 """
58
59 VCS_EXECUTABLE = ('hg', '--config', 'defaults.log=', '--config',
60 'defaults.pull=')
61 ISSUE_NUMBER_REGEX = re.compile(r'\b(issue|fixes)\s+(\d+)\b', re.I)
62 NOISSUE_REGEX = re.compile(r'^noissue\b', re.I)
63
64 def __init__(self, *args):
65 """Construct a DepUpdate object.
66
67 During initialization, DepUpdate will invoke the appropriate VCS to
68 fetch a list of changes, parse them and (if not otherwise specified)
69 get the matching revisions from the mirrored repository.
70
71 Parameters: *args - Passed down to the argparse.ArgumentParser instance
72
73 """
74 self._cwd = os.getcwd()
75
76 self._base_revision = None
77 self._parsed_changes = None
78 self.arguments = None
79
80 self._dep_config = None
81
82 default_template = os.path.join(
83 os.path.dirname(os.path.realpath(__file__)), 'templates',
84 'default.trac')
85
86 # Initialize and run the internal argument parser
87 self._make_arguments(default_template, *args)
88
89 # Check if root repository is dirty
90 self.root_repo = Vcs.factory(self._cwd)
91 if not self.root_repo.repo_is_clean():
92 logger.error('Your repository is dirty')
93 exit(1)
94
95 # Initialize the main VCS and the list of changes
96 self._main_vcs = Vcs.factory(os.path.join(self._cwd,
97 self.arguments.dependency))
98 self.changes = self._main_vcs.change_list(self.base_revision,
99 self.arguments.new_revision)
100 if len(self.changes) == 0:
101 self.changes = self._main_vcs.change_list(
102 self.arguments.new_revision,
103 self.base_revision
104 )
105 if len(self.changes) > 0:
106 # reverse mode. Uh-oh.
107 logger.warn('You are trying to downgrade the dependency!')
108
109 self._main_vcs.enhance_changes_information(
110 self.changes,
111 os.path.join(self._mirror_location(),
112 self.arguments.dependency),
113 self.arguments.skip_mirror,
114 )
115
116 def _make_arguments(self, default_template, *args):
117 """Initialize the argument parser and store the arguments."""
118 parser = argparse.ArgumentParser(
119 description=__doc__,
120 formatter_class=argparse.RawDescriptionHelpFormatter)
121
122 # First prepare all basic shared options
123 options_parser = argparse.ArgumentParser(add_help=False)
124 shared = options_parser.add_argument_group()
125 shared.add_argument(
126 'dependency',
127 help=('The dependency to be updated, as specified in the '
128 'dependencies file.')
129 )
130 shared.add_argument(
131 '-r', '--revision', dest='new_revision',
132 help=('The revision to update to. Defaults to the remote '
133 'master bookmark/branch. Must be accessible by the '
134 "dependency's vcs.")
135 )
136 shared.add_argument(
137 '-m', '--mirrored-repository', dest='local_mirror',
138 help=('Path to the local copy of a mirrored repository. '
139 'Used to fetch the corresponding hash. If not '
140 'given, the source parsed from the dependencies file is '
141 'used.')
142 )
143
144 # Shared options for non-commit commands
145 advanced_parser = argparse.ArgumentParser(add_help=False)
146 advanced = advanced_parser.add_argument_group()
147 advanced.add_argument(
148 '-f', '--filename', dest='filename', default=None,
149 help=("When specified, write the subcommand's output to the "
150 'given file, rather than to STDOUT.')
151 )
152 advanced.add_argument(
153 '-s', '--skip-mirror', action='store_true', dest='skip_mirror',
154 help='Do not use any mirror.'
155 )
156 advanced.add_argument(
157 '-l', '--lookup-integration-notes', action='store_true',
158 dest='lookup_inotes', default=False,
159 help=('Search https://issues.adblockplus.org for integration '
160 'notes associated with the included issue IDs. The '
161 'results are written to STDERR. CAUTION: This is a very '
162 'network heavy operation.')
163 )
164
165 subs = parser.add_subparsers(
166 title='Subcommands', dest='action', metavar='[subcommand]',
167 help=('Required, the actual command to be executed. Execute '
168 'run "<subcommand> -h" for more information.')
169 )
170
171 # Add the command and options for creating a diff
172 diff_parser = subs.add_parser(
173 'diff', parents=[options_parser, advanced_parser],
174 help='Create a unified diff of all changes',
175 description=("Invoke the current repository's VCS to generate "
176 'a diff, containing all changes made between two '
177 'revisions.'))
178 diff_parser.add_argument(
179 '-n', '--n-context-lines', dest='unified_lines', type=int,
180 default=16,
181 help=('Number of unified context lines to be added to the '
182 'diff. Defaults to 16 (Used only with -d/--diff).')
183 )
184
185 # Add the command and options for creating an issue body
186 issue_parser = subs.add_parser(
187 'issue', parents=[options_parser, advanced_parser],
188 help='Render an issue body',
189 description=('Render an issue subject and an issue body, '
190 'according to the given template.'))
191 issue_parser.add_argument(
192 '-t', '--template', dest='tmpl_path',
193 default=default_template,
194 help=('The template to use. Defaults to the provided '
195 'default.trac.')
196 )
197
198 # Add the command for printing a list of changes
199 subs.add_parser(
200 'changes', parents=[options_parser, advanced_parser],
201 help='Generate a list of commits between two revisions',
202 description=('Generate a list of commit hashes and commit '
203 "messages between the dependency's current "
204 'revision and a given new revision.'))
205
206 # Add the command for changing and committing a dependency update
207 commit_parser = subs.add_parser(
208 'commit', help='Update and commit a dependency change',
209 parents=[options_parser],
210 description=('Rewrite and commit a dependency file to the new '
211 'revision. WARNING: This actually changes your '
212 "repository's history, use with care!"))
213 commit_parser.add_argument(
214 'issue_number', help=('The issue number, filed on '
215 'https://issues.adblockplus.org'))
216 commit_parser.set_defaults(skip_mirror=False, lookup_inotes=False,
217 filename=None)
218
219 self.arguments = parser.parse_args(args if len(args) > 0 else None)
220
221 @property
222 def dep_config(self):
223 """Provide the dependencies by using ensure_dependencies.read_dep().
224
225 Since this program is meant to be run inside a repository which uses
226 the buildtools' dependency functionalities, we are sure that
227 ensure_dependencies.py and dependencies exist.
228
229 However, ensure_dependencies is currently only compatible with python2.
230 Due to this we explicitly invoke a python2 interpreter to run our
231 dependencies.py, which runs ensure_dependencies.read_deps() and returns
232 the output as JSON data.
233 """
234 if self._dep_config is None:
235 dependencies_script = os.path.join(
236 os.path.dirname(os.path.realpath(__file__)), 'dependencies.py')
237
238 dep_json = subprocess.check_output(
239 ['python2', dependencies_script]
240 ).decode('utf-8')
241 self._dep_config = json.loads(dep_json)
242 return self._dep_config
243
244 @property
245 def base_revision(self):
246 """Provide the current revision of the dependency to be processed."""
247 if self._base_revision is None:
248 for key in ['*', self._main_vcs.EXECUTABLE]:
249 rev = self.dep_config[self.arguments.dependency][key][1]
250 if rev is not None:
251 self._base_revision = rev
252 break
253 return self._base_revision
254
255 def _parse_changes(self, changes):
256 """Parse the changelist to issues / noissues."""
257 issue_ids = set()
258 noissues = []
259 for change in changes:
260 match = self.ISSUE_NUMBER_REGEX.search(change['message'])
261 if match:
262 issue_ids.add(match.group(2))
263 else:
264 noissues.append(change)
265 if not self.NOISSUE_REGEX.search(change['message']):
266 msg = (
267 'warning: no issue reference in commit message: '
268 '"{message}" (commit {hg_hash} | {git_hash})\n'
269 ).format(**change)
270 logger.warn(msg)
271
272 return issue_ids, noissues
273
274 @property
275 def parsed_changes(self):
276 """Provide the list of changes, separated by issues and noissues.
277
278 Returns a dictionary, containing the following two key/value pairs:
279 'issue_ids': a list of issue IDs (as seen on
280 https://issues.adblockplus.org/)
281 'noissues': The remaining changes, with all original information (see
282 DepUpdate.changes) which could not be associated with any
283 issue.
284 """
285 if self._parsed_changes is None:
286 self._parsed_changes = {}
287 issue_ids, noissues = self._parse_changes(self.changes)
288 self._parsed_changes['issue_ids'] = issue_ids
289 self._parsed_changes['noissues'] = noissues
290 return self._parsed_changes
291
292 def _possible_sources(self):
293 root_conf = self._dep_config['_root']
294 config = self.dep_config[self.arguments.dependency]
295
296 # The fallback / main source paths for a repository are given in the
297 # dependencies file's _root section.
298 keys = ['hg', 'git']
299 possible_sources = {}
300 possible_sources.update({
301 key + '_root': root_conf[key]
302 for key in keys
303 })
304
305 # Any dependency may specify a custom source location.
306 possible_sources.update({
307 key: source for key, source in [
308 (key, config.get(key, (None, None))[0]) for key in keys
309 ] if source is not None
310 })
311
312 return possible_sources
313
314 def _mirror_location(self):
315 possible_sources = self._possible_sources()
316 mirror_ex = self._main_vcs._other_cls.EXECUTABLE
317
318 # If the user specified a local mirror, use it. Otherwise use the
319 # mirror, which was specified in the dependencies file.
320 if self.arguments.local_mirror:
321 mirror = self.arguments.local_mirror
322 else:
323 for key in [mirror_ex, mirror_ex + '_root']:
324 if key in possible_sources:
325 mirror = possible_sources[key]
326 break
327 return mirror
328
329 def _make_dependencies_string(self, hg_source=None, hg_rev=None,
330 git_source=None, git_rev=None,
331 remote_name=None):
332 dependency = '{} = {}'.format(self.arguments.dependency,
333 remote_name or self.arguments.dependency)
334
335 for prefix, rev, source in [(' hg:', hg_rev, hg_source),
336 (' git:', git_rev, git_source)]:
337 if rev is not None:
338 dependency += prefix
339 if source is not None:
340 dependency += source + '@'
341 dependency += rev
342
343 return dependency
344
345 def _update_dependencies_file(self):
346 config = self.dep_config[self.arguments.dependency]
347
348 remote_repository_name, none_hash_rev = config.get('*', (None, None))
349 hg_source, hg_rev = config.get('hg', (None, None))
350 git_source, git_rev = config.get('git', (None, None))
351
352 current_entry = self._make_dependencies_string(
353 hg_source, hg_rev, git_source, git_rev, remote_repository_name
354 )
355
356 new_entry = self._make_dependencies_string(
357 hg_source, self.changes[0]['hg_hash'], git_source,
358 self.changes[0]['git_hash'], remote_repository_name
359 )
360
361 dependency_path = os.path.join(self._cwd, 'dependencies')
362 with io.open(dependency_path, 'r', encoding='utf-8') as fp:
363 current_deps = fp.read()
364 with io.open(dependency_path, 'w', encoding='utf-8') as fp:
365 fp.write(current_deps.replace(current_entry, new_entry))
366
367 def _update_copied_code(self):
368 subprocess.check_output(
369 ['python2', 'ensure_dependencies.py'],
370 cwd=self._cwd
371 )
372
373 def build_diff(self):
374 """Generate a unified diff of all changes."""
375 return self._main_vcs.merged_diff(self.base_revision,
376 self.arguments.new_revision,
377 self.arguments.unified_lines)
378
379 def build_issue(self):
380 """Process all changes and render an issue."""
381 context = {}
382 context['repository'] = self.arguments.dependency
383 context['issue_ids'] = self.parsed_changes['issue_ids']
384 context['noissues'] = self.parsed_changes['noissues']
385 context['hg_hash'] = self.changes[0]['hg_hash']
386 context['git_hash'] = self.changes[0]['git_hash']
387 context['raw_changes'] = self.changes
388
389 path, filename = os.path.split(self.arguments.tmpl_path)
390
391 return jinja2.Environment(
392 loader=jinja2.FileSystemLoader(path or './')
393 ).get_template(filename).render(context)
394
395 def lookup_integration_notes(self):
396 """Search for any "Integration notes" mentions at the issue-tracker.
397
398 Cycle through the list of issue IDs and search for "Integration Notes"
399 in the associated issue on https://issues.adblockplus.org. If found,
400 write the corresponding url to STDERR.
401 """
402 # Let logger show INFO-level messages
403 logger.setLevel(logging.INFO)
404 integration_notes_regex = re.compile(r'Integration\s*notes', re.I)
405
406 def from_url(issue_url):
407 html = ''
408 content = urlopen(issue_url)
409
410 for line in content:
411 html += line.decode('utf-8')
412 return html
413
414 for issue_id in self.parsed_changes['issue_ids']:
415 issue_url = 'https://issues.adblockplus.org/ticket/' + issue_id
416 html = from_url(issue_url)
417 if not integration_notes_regex.search(html):
418 continue
419
420 logger.info('Integration notes found: ' + issue_url)
421
422 def build_changes(self):
423 """Write a descriptive list of the changes to STDOUT."""
424 return os.linesep.join(
425 [(
426 '( hg:{hg_hash} | git:{git_hash} ) : {message} (by {author})'
427 ).format(**change) for change in self.changes]
428 )
429
430 def commit_update(self):
431 """Commit the new dependency and potentially updated files."""
432 commit_msg = 'Issue {} - Update {} to {}'.format(
433 self.arguments.issue_number, self.arguments.dependency,
434 ' / '.join((self.changes[0]['hg_hash'],
435 self.changes[0]['git_hash'])))
436 try:
437 self._update_dependencies_file()
438 self._update_copied_code()
439 self.root_repo.commit_changes(commit_msg)
440
441 return commit_msg
442 except subprocess.CalledProcessError:
443 self._main_vcs.undo_changes()
444 logger.error('Could not safely commit the changes. Reverting.')
445
446 def __call__(self):
447 """Let this class's objects be callable, run all desired tasks."""
448 action_map = {
449 'diff': self.build_diff,
450 'changes': self.build_changes,
451 'issue': self.build_issue,
452 'commit': self.commit_update,
453 }
454
455 if len(self.changes) == 0:
456 print('NO CHANGES FOUND. You are either trying to update to a '
457 'revision, which the dependency already is at - or '
458 'something went wrong while executing the vcs.')
459 return
460
461 if self.arguments.lookup_inotes:
462 self.lookup_integration_notes()
463
464 output = action_map[self.arguments.action]()
465 if self.arguments.filename is not None:
466 with io.open(self.arguments.filename, 'w', encoding='utf-8') as fp:
467 fp.write(output)
468 print('Output writen to ' + self.arguments.filename)
469 else:
470 print(output)
LEFTRIGHT

Powered by Google App Engine
This is Rietveld