Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: eyeo-depup/src/vcs.py

Issue 29599579: OffTopic: DependencyUpdater
Patch Set: Created Nov. 27, 2017, 9:40 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 # This file is part of Adblock Plus <https://adblockplus.org/>,
2 # Copyright (C) 2006-present eyeo GmbH
3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation.
7 #
8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
12 #
13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15
16 """VCS related classes for eyeo-depup."""
17
18 from __future__ import print_function, unicode_literals
19
20 import io
21 import json
22 import logging
23 import os
24 import shutil
25 import subprocess
26 import sys
27 import tempfile
28
29 logging.basicConfig()
30 logger = logging.getLogger('vcs')
31
32
33 class Vcs(object):
34 """Baseclass for Git and Mercurial."""
35
36 JSON_DQUOTES = '__DQ__'
37
38 class VcsException(Exception):
39 """Raised when no distinct VCS for a given repository was found."""
40
41 def __init__(self, location, force_clone=False):
42 """Construct a Vcs object for a given location.
43
44 parameters:
45 location: The repository location, may be a local folder or a remote
46 location.
47
48 When the specified location does not exist locally, Vcs will attempt
49 to create a temporary repository, cloned from the given location.
50
51 """
52 self._source, self._repository = os.path.split(location)
53 if not os.path.exists(location) or force_clone:
54 self._make_temporary(location)
55 self._clean_up = True
56 else:
57 self._cwd = location
58 self._clean_up = False
59
60 def __enter__(self):
61 """Enter the object's context."""
62 return self
63
64 def __exit__(self, exc_type, exc_value, traceback):
65 """Exit the object'c context and delete any temporary data."""
66 if self._clean_up:
67 shutil.rmtree(self._cwd)
68
69 @classmethod
70 def is_vcs_for_repo(cls, path):
71 """Assert if cls is a suitable VCS for the given (repository-) path."""
72 try:
73 subprocess.check_output([cls.EXECUTABLE, 'status'], cwd=path,
74 stderr=subprocess.STDOUT)
75 return True
76 except subprocess.CalledProcessError:
77 return False
78
79 def run_cmd(self, *args, **kwargs):
80 """Run the vcs with the given commands."""
81 cmd = self.BASE_CMD + args
82 try:
83 return subprocess.check_output(
84 cmd,
85 cwd=os.path.join(self._cwd),
86 stderr=subprocess.STDOUT,
87 ).decode('utf-8')
88 except subprocess.CalledProcessError as e:
89 logger.error(e.output.decode('utf-8'))
90 sys.exit(1)
91
92 def _get_latest(self):
93 self.run_cmd(self.UPDATE_LOCAL_HISTORY)
94
95 def _escape_changes(self, changes):
96 return changes.replace('"', '\\"').replace(self.JSON_DQUOTES, '"')
97
98 def _changes_as_json(self, changes):
99 return json.loads(
100 '[{}]'.format(','.join(
101 self._escape_changes(changes).strip().splitlines()
102 )))
103
104 def merged_diff(self, rev_a, rev_b, n_unified=16):
105 """Invoke the VCS' functionality to create a unified diff.
106
107 Parameters:
108 rev_a: The revision representing the start.
109 rev_b: The revision representing the end. Defaults to
110 Cls.DEFAULT_NEW_REVISION
111 n_unified: The amount of context lines to add to the diff.
112
113 """
114 return self.run_cmd('diff', '--unified=' + str(n_unified),
115 *(self._rev_comb(
116 rev_a,
117 rev_b or self.DEFAULT_NEW_REVISION)))
118
119 def change_list(self, rev_a, rev_b):
120 """Return the repository's history from revisions a to b as JSON.
121
122 Parameters:
123 rev_a: The revision representing the start.
124 rev_b: The revision representing the end. Defaults to
125 Cls.DEFAULT_NEW_REVISION
126
127 """
128 self._get_latest()
129
130 log_format = self._log_format()
131 rev_cmd = self._rev_comb(rev_a, rev_b or self.DEFAULT_NEW_REVISION)
132
133 changes = self.run_cmd(*('log',) + log_format + rev_cmd)
134 return self._changes_as_json(changes)
135
136 def enhance_changes_information(self, changes, dependency_location, fake):
137 """Enhance the change list with matching revisions from a mirror.
138
139 Parameters:
140 changes: The list to enhance, containing dictionaries
141 with the keys "hash", "author", "date" and
142 "message".
143 dependency_location: The (remote or locale) location of the
144 repository, which is supposed to be the mirror
145 for the current repository.
146 fake {True, False}: Causes atual processing of a mirror repository
147 (False) or fakes values (True)
148
149 """
150 self_ex = self.EXECUTABLE
151 mirr_ex = self._other_cls.EXECUTABLE
152
153 if not fake:
154 with self._other_cls(dependency_location) as mirror:
155 mirror._get_latest()
156
157 mirrored_hashes = {
158 change['hash']: mirror.matching_hash(change['author'],
159 change['date'],
160 change['message'])
161 for change in changes
162 }
163 else:
164 mirrored_hashes = {}
165
166 for change in changes:
167 change[self_ex + '_url'] = self.REVISION_URL.format(
168 repository=self._repository, revision=change['hash'])
169 change[self_ex + '_hash'] = change['hash']
170
171 mirrored_hash = mirrored_hashes.get(change['hash'], 'NO MIRROR')
172 del change['hash']
173
174 change[mirr_ex + '_url'] = self._other_cls.REVISION_URL.format(
175 repository=self._repository, revision=mirrored_hash)
176 change[mirr_ex + '_hash'] = mirrored_hash
177
178 @staticmethod
179 def factory(location, force_clone=False):
180 """Get a suiting Vcs instance for the given repository path."""
181 obj = None
182 for cls in [Git, Mercurial]:
183 if cls.is_vcs_for_repo(location):
184 if obj is not None:
185 raise Vcs.VcsException(
186 "Found multiple possible VCS' for " + location)
187 obj = cls(location, force_clone)
188
189 if obj is None:
190 raise Vcs.VcsException('No valid VCS found for ' + location)
191 return obj
192
193
194 class Mercurial(Vcs):
195 """Mercurial specialization of VCS."""
196
197 EXECUTABLE = 'hg'
198 BASE_CMD = (EXECUTABLE, '--config', 'defaults.log=', '--config',
199 'defaults.pull=', '--config', 'defaults.diff=')
200 UPDATE_LOCAL_HISTORY = 'pull'
201 LOG_TEMLATE = ('\\{"hash":"{node|short}","author":"{author|person}",'
202 '"date":"{date|rfc822date}","message":"{desc|strip|'
203 'firstline}"}\n')
204 DEFAULT_NEW_REVISION = 'master'
205
206 REVISION_URL = 'https://hg.adblockplus.org/{repository}/rev/{revision}'
207
208 def __init__(self, *args):
209 """Construct a Mercurial object and specify Git as the mirror class."""
210 self._other_cls = Git
211 super(Mercurial, self).__init__(*args)
212
213 def _rev_comb(self, rev_a, rev_b):
214 # Only take into account those changesets, which are actually affecting
215 # the repository's content. See
216 # https://www.mercurial-scm.org/repo/hg/help/revsets
217 return ('-r', '{}::{}'.format(rev_a, rev_b))
218
219 def _log_format(self):
220 log_format = self.LOG_TEMLATE.replace('"', self.JSON_DQUOTES)
221 return ('--template', log_format)
222
223 def change_list(self, *args):
224 """Apply measures for hg log and call Vcs's change_list."""
225 # Mercurial's command for producing a log between revisions using the
226 # revision set produced by self._rev_comb returns the changesets in a
227 # reversed order. Additionally the current revision is returned.
228 return list(reversed(super(Mercurial, self).change_list(*args)[1:]))
229
230 def matching_hash(self, author, date, message):
231 """Get the responsible commit for the given information.
232
233 A commit must stafisy equailty for author, date and commit message, in
234 order to be recognized as the matching commit.
235
236 """
237 return self.run_cmd('log', '-u', author, '-d', date, '--keyword',
238 message, '--template', '{node|short}')
239
240 def _make_temporary(self, location):
241 self._cwd = tempfile.mkdtemp()
242 os.mkdir(os.path.join(self._cwd, '.hg'))
243
244 with io.open(os.path.join(self._cwd, '.hg', 'hgrc'), 'w') as fp:
245 fp.write('[paths]{}default = {}{}'.format(os.linesep, location,
246 os.linesep))
247
248 def commit_changes(self, msg):
249 """Add any local changes and commit the with <msg>."""
250 self.run_cmd('commit', '-m', msg)
251
252 def undo_changes(self):
253 """Undo all changes in local repsitory and leave no backup."""
254 self.run_cmd('revert', '--all', '--no-backup')
255
256 def repo_is_clean(self):
257 """Check whether the current repository is clean."""
258 buff = self.run_cmd('status')
259 return len(buff) == 0
260
261
262 class Git(Vcs):
263 """Git specialization of Vcs."""
264
265 EXECUTABLE = 'git'
266 BASE_CMD = (EXECUTABLE,)
267 UPDATE_LOCAL_HISTORY = 'fetch'
268 LOG_TEMLATE = '{"hash":"%h","author":"%an","date":"%aD","message":"%s"}'
269 DEFAULT_NEW_REVISION = 'origin/master'
270
271 REVISION_URL = ('https://www.github.com/adblockplus/{repository}/commit/'
272 '{revision}')
273
274 def __init__(self, *args):
275 """Construct a Git object and specify Mercurial as the mirror class."""
276 self._other_cls = Mercurial
277 super(Git, self).__init__(*args)
278
279 def _rev_comb(self, rev_a, rev_b):
280 return ('{}..{}'.format(rev_a, rev_b),)
281
282 def _log_format(self):
283 return ('--pretty=format:{}'.format(self.LOG_TEMLATE.replace(
284 '"', self.JSON_DQUOTES)),)
285
286 def matching_hash(self, author, date, message):
287 """Get the responsible commit for the given information.
288
289 A commit must stafisy equailty for author, date and commit message, in
290 order to be recognized as the matching commit.
291
292 """
293 # Git does not implement exact date matching directly. Additionally,
294 # git is only capable of filtering by COMMIT DATE instead of
295 # AUTHOR DATE (which is what we are actually looking for), see
296 # https://stackoverflow.com/q/37311494/
297 # Since naturally the COMMIT DATE allways is later then the AUTHOR
298 # DATE, we are at least able to limit the valid range to after our
299 # given date
300 result = self.run_cmd('log', '--author={}'.format(author),
301 '--grep={}'.format(message),
302 '--after={}'.format(date), '--pretty=format:%h')
303 if len(result.split()) > 1:
304 raise Vcs.VcsException('FATAL: Ambiguous commit filter!')
305 return result
306
307 def _make_temporary(self, location):
308 self._cwd = tempfile.mkdtemp()
309 self.run_cmd('clone', '--bare', location, self._cwd)
310
311 def commit_changes(self, msg):
312 """Add any local changes and commit the with <msg>."""
313 self.run_cmd('add', '.')
314 self.run_cmd('commit', '-m', msg)
315
316 def undo_changes(self):
317 """Undo all changes in local repsitory."""
318 self.run_cmd('checkout', '.')
319
320 def repo_is_clean(self):
321 """Check whether the current repository is clean."""
322 # unstaged changes
323 no_uncommited = len(self.run_cmd('diff-index', 'HEAD', '--')) == 0
324 # untracked changes
325 no_untracked = len(self.run_cmd('ls-files', '-o', '-d',
326 '--exclude-standard')) == 0
327 return no_uncommited and no_untracked
OLDNEW

Powered by Google App Engine
This is Rietveld