Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: ensure_dependencies.py

Issue 5652679430766592: Issue 2443 - Honour SKIP_DEPENDENCY_UPDATES environment variable (Closed)
Left Patch Set: Created May 1, 2015, 10:03 a.m.
Right Patch Set: Addressed further feedback from Sebastian Created May 7, 2015, 2:17 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « no previous file | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # coding: utf-8 2 # coding: utf-8
3 3
4 # This Source Code Form is subject to the terms of the Mozilla Public 4 # This Source Code Form is subject to the terms of the Mozilla Public
5 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # License, v. 2.0. If a copy of the MPL was not distributed with this
6 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 7
8 import sys 8 import sys
9 import os 9 import os
10 import posixpath 10 import posixpath
(...skipping 16 matching lines...) Expand all
27 # File to update this script from (optional) 27 # File to update this script from (optional)
28 _self = buildtools/ensure_dependencies.py 28 _self = buildtools/ensure_dependencies.py
29 # Check out elemhidehelper repository into extensions/elemhidehelper directory 29 # Check out elemhidehelper repository into extensions/elemhidehelper directory
30 # at tag "1.2". 30 # at tag "1.2".
31 extensions/elemhidehelper = elemhidehelper 1.2 31 extensions/elemhidehelper = elemhidehelper 1.2
32 # Check out buildtools repository into buildtools directory at VCS-specific 32 # Check out buildtools repository into buildtools directory at VCS-specific
33 # revision IDs. 33 # revision IDs.
34 buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5 34 buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
35 """ 35 """
36 36
37 SKIP_DEPENDENCY_UPDATES = os.environ.get(
38 "SKIP_DEPENDENCY_UPDATES", ""
39 ).lower() not in ("", "0", "false")
40
37 class Mercurial(): 41 class Mercurial():
38 def istype(self, repodir): 42 def istype(self, repodir):
39 return os.path.exists(os.path.join(repodir, ".hg")) 43 return os.path.exists(os.path.join(repodir, ".hg"))
40 44
41 def clone(self, source, target): 45 def clone(self, source, target):
42 if not source.endswith("/"): 46 if not source.endswith("/"):
43 source += "/" 47 source += "/"
44 subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, targe t]) 48 subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, targe t])
45 49
46 def get_revision_id(self, repo, rev=None): 50 def get_revision_id(self, repo, rev=None):
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
90 source = source.rstrip("/") 94 source = source.rstrip("/")
91 if not source.endswith(".git"): 95 if not source.endswith(".git"):
92 source += ".git" 96 source += ".git"
93 subprocess.check_call(["git", "clone", "--quiet", source, target]) 97 subprocess.check_call(["git", "clone", "--quiet", source, target])
94 98
95 def get_revision_id(self, repo, rev="HEAD"): 99 def get_revision_id(self, repo, rev="HEAD"):
96 command = ["git", "rev-parse", "--revs-only", rev + '^{commit}'] 100 command = ["git", "rev-parse", "--revs-only", rev + '^{commit}']
97 return subprocess.check_output(command, cwd=repo).strip() 101 return subprocess.check_output(command, cwd=repo).strip()
98 102
99 def pull(self, repo): 103 def pull(self, repo):
104 # Fetch tracked branches, new tags and the list of available remote branches
100 subprocess.check_call(["git", "fetch", "--quiet", "--all", "--tags"], cwd=re po) 105 subprocess.check_call(["git", "fetch", "--quiet", "--all", "--tags"], cwd=re po)
106 # Next we need to ensure all remote branches are tracked
107 newly_tracked = False
108 remotes = subprocess.check_output(["git", "branch", "--remotes"], cwd=repo)
109 for match in re.finditer(r"^\s*(origin/(\S+))$", remotes, re.M):
110 remote, local = match.groups()
111 with open(os.devnull, "wb") as devnull:
112 if subprocess.call(["git", "branch", "--track", local, remote],
113 cwd=repo, stdout=devnull, stderr=devnull) == 0:
114 newly_tracked = True
115 # Finally fetch any newly tracked remote branches
116 if newly_tracked:
117 subprocess.check_call(["git", "fetch", "--quiet", "origin"], cwd=repo)
101 118
102 def update(self, repo, rev): 119 def update(self, repo, rev):
103 subprocess.check_call(["git", "checkout", "--quiet", rev], cwd=repo) 120 subprocess.check_call(["git", "checkout", "--quiet", rev], cwd=repo)
104 121
105 def ignore(self, target, repo): 122 def ignore(self, target, repo):
106 module = os.path.relpath(target, repo) 123 module = os.path.relpath(target, repo)
107 exclude_file = os.path.join(repo, ".git", "info", "exclude") 124 exclude_file = os.path.join(repo, ".git", "info", "exclude")
108 _ensure_line_exists(exclude_file, module) 125 _ensure_line_exists(exclude_file, module)
109 126
110 def postprocess_url(self, url): 127 def postprocess_url(self, url):
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after
160 if spec: 177 if spec:
161 result[key] = spec 178 result[key] = spec
162 return result 179 return result
163 except IOError, e: 180 except IOError, e:
164 if e.errno != errno.ENOENT: 181 if e.errno != errno.ENOENT:
165 raise 182 raise
166 return None 183 return None
167 184
168 def safe_join(path, subpath): 185 def safe_join(path, subpath):
169 # This has been inspired by Flask's safe_join() function 186 # This has been inspired by Flask's safe_join() function
170 forbidden = set([os.sep, os.altsep]) - set([posixpath.sep, None]) 187 forbidden = {os.sep, os.altsep} - {posixpath.sep, None}
171 if any(sep in subpath for sep in forbidden): 188 if any(sep in subpath for sep in forbidden):
172 raise Exception("Illegal directory separator in dependency path %s" % subpat h) 189 raise Exception("Illegal directory separator in dependency path %s" % subpat h)
173 190
174 normpath = posixpath.normpath(subpath) 191 normpath = posixpath.normpath(subpath)
175 if posixpath.isabs(normpath): 192 if posixpath.isabs(normpath):
176 raise Exception("Dependency path %s cannot be absolute" % subpath) 193 raise Exception("Dependency path %s cannot be absolute" % subpath)
177 if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posi xpath.sep): 194 if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posi xpath.sep):
178 raise Exception("Dependency path %s has to be inside the repository" % subpa th) 195 raise Exception("Dependency path %s has to be inside the repository" % subpa th)
179 return os.path.join(path, *normpath.split(posixpath.sep)) 196 return os.path.join(path, *normpath.split(posixpath.sep))
180 197
181 def get_repo_type(repo): 198 def get_repo_type(repo):
182 for name, repotype in repo_types.iteritems(): 199 for name, repotype in repo_types.iteritems():
183 if repotype.istype(repo): 200 if repotype.istype(repo):
184 return name 201 return name
185 return None 202 return None
186 203
187 def ensure_repo(parentrepo, target, roots, sourcename): 204 def ensure_repo(parentrepo, target, roots, sourcename):
188 if os.path.exists(target): 205 if os.path.exists(target):
206 return
207
208 if SKIP_DEPENDENCY_UPDATES:
209 logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, "
210 "%s not cloned", target)
189 return 211 return
190 212
191 parenttype = get_repo_type(parentrepo) 213 parenttype = get_repo_type(parentrepo)
192 type = None 214 type = None
193 for key in roots: 215 for key in roots:
194 if key == parenttype or (key in repo_types and type is None): 216 if key == parenttype or (key in repo_types and type is None):
195 type = key 217 type = key
196 if type is None: 218 if type is None:
197 raise Exception("No valid source found to create %s" % target) 219 raise Exception("No valid source found to create %s" % target)
198 220
(...skipping 21 matching lines...) Expand all
220 242
221 if type in revisions: 243 if type in revisions:
222 revision = revisions[type] 244 revision = revisions[type]
223 elif "*" in revisions: 245 elif "*" in revisions:
224 revision = revisions["*"] 246 revision = revisions["*"]
225 else: 247 else:
226 logging.warning("No revision specified for repository %s (type %s), skipping update" % (target, type)) 248 logging.warning("No revision specified for repository %s (type %s), skipping update" % (target, type))
227 return 249 return
228 250
229 resolved_revision = repo_types[type].get_revision_id(target, revision) 251 resolved_revision = repo_types[type].get_revision_id(target, revision)
230 if not resolved_revision: 252 current_revision = repo_types[type].get_revision_id(target)
231 logging.info("Revision %s is unknown, downloading remote changes" % revision ) 253
232 repo_types[type].pull(target) 254 if resolved_revision != current_revision:
233 resolved_revision = repo_types[type].get_revision_id(target, revision) 255 if SKIP_DEPENDENCY_UPDATES:
256 logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, "
257 "%s not checked out to %s", target, revision)
258 return
259
234 if not resolved_revision: 260 if not resolved_revision:
235 raise Exception("Failed to resolve revision %s" % revision) 261 logging.info("Revision %s is unknown, downloading remote changes" % revisi on)
236 262 repo_types[type].pull(target)
237 current_revision = repo_types[type].get_revision_id(target) 263 resolved_revision = repo_types[type].get_revision_id(target, revision)
238 if resolved_revision != current_revision: 264 if not resolved_revision:
265 raise Exception("Failed to resolve revision %s" % revision)
266
239 logging.info("Updating repository %s to revision %s" % (target, resolved_rev ision)) 267 logging.info("Updating repository %s to revision %s" % (target, resolved_rev ision))
240 repo_types[type].update(target, resolved_revision) 268 repo_types[type].update(target, resolved_revision)
241 269
242 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdep endencies=set()): 270 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdep endencies=set()):
243 config = read_deps(repodir) 271 config = read_deps(repodir)
244 if config is None: 272 if config is None:
245 if level == 0: 273 if level == 0:
246 logging.warning("No dependencies file in directory %s, nothing to do...\n% s" % (repodir, USAGE)) 274 logging.warning("No dependencies file in directory %s, nothing to do...\n% s" % (repodir, USAGE))
247 return 275 return
248 if level >= 10: 276 if level >= 10:
249 logging.warning("Too much subrepository nesting, ignoring %s" % repo) 277 logging.warning("Too much subrepository nesting, ignoring %s" % repo)
278 return
250 279
251 if overrideroots is not None: 280 if overrideroots is not None:
252 config["_root"] = overrideroots 281 config["_root"] = overrideroots
253 282
254 for dir, revisions in config.iteritems(): 283 for dir, revisions in config.iteritems():
255 if dir.startswith("_") or revisions["_source"] in skipdependencies: 284 if dir.startswith("_") or revisions["_source"] in skipdependencies:
256 continue 285 continue
257 target = safe_join(repodir, dir) 286 target = safe_join(repodir, dir)
258 ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"]) 287 ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"])
259 update_repo(target, revisions) 288 update_repo(target, revisions)
(...skipping 30 matching lines...) Expand all
290 if not pattern in file_content: 319 if not pattern in file_content:
291 file_content.append(pattern) 320 file_content.append(pattern)
292 f.seek(0, os.SEEK_SET) 321 f.seek(0, os.SEEK_SET)
293 f.truncate() 322 f.truncate()
294 for l in file_content: 323 for l in file_content:
295 print >>f, l 324 print >>f, l
296 325
297 if __name__ == "__main__": 326 if __name__ == "__main__":
298 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) 327 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
299 328
300 if os.environ.get("SKIP_ENSURE_DEPENDENCIES"):
Sebastian Noack 2015/05/04 11:33:01 This will merely check whether the variable is set
kzar 2015/05/05 13:25:22 Well I realised that but sure, Done.
301 logging.warning("SKIP_ENSURE_DEPENDENCIES environment variable set, skipping .")
Sebastian Noack 2015/05/04 11:33:01 I think I'd prefer to only show a warning if there
kzar 2015/05/05 13:25:22 Disagree with this one, I think that people that s
Sebastian Noack 2015/05/05 14:33:24 There are two situations in which people might use
Wladimir Palant 2015/05/05 14:46:37 While I'm not sure I agree with Sebastian's argume
kzar 2015/05/06 16:23:53 Felix / Matze - As you two are the main users/prop
Felix Dahlke 2015/05/06 16:31:23 Regarding the warning: Let's definitely have that
Sebastian Noack 2015/05/06 16:59:23 Nobody argues about that. We already have consensu
Felix Dahlke 2015/05/06 17:14:57 It'd be clear if you call it SKIP_DEPENDENCY_HG_UP
Wladimir Palant 2015/05/06 17:40:47 Mostly because these people never manage to provid
302 sys.exit()
303
304 parser = argparse.ArgumentParser(description="Verify dependencies for a set of repositories, by default the repository of this script.") 329 parser = argparse.ArgumentParser(description="Verify dependencies for a set of repositories, by default the repository of this script.")
305 parser.add_argument("repos", metavar="repository", type=str, nargs="*", help=" Repository path") 330 parser.add_argument("repos", metavar="repository", type=str, nargs="*", help=" Repository path")
306 parser.add_argument("-q", "--quiet", action="store_true", help="Suppress infor mational output") 331 parser.add_argument("-q", "--quiet", action="store_true", help="Suppress infor mational output")
307 args = parser.parse_args() 332 args = parser.parse_args()
308 333
309 if args.quiet: 334 if args.quiet:
310 logging.disable(logging.INFO) 335 logging.disable(logging.INFO)
311 336
312 repos = args.repos 337 repos = args.repos
313 if not len(repos): 338 if not len(repos):
314 repos = [os.path.dirname(__file__)] 339 repos = [os.path.dirname(__file__)]
315 for repo in repos: 340 for repo in repos:
316 resolve_deps(repo) 341 resolve_deps(repo)
LEFTRIGHT
« no previous file | no next file » | Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Toggle Comments ('s')

Powered by Google App Engine
This is Rietveld