Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: ensure_dependencies.py

Issue 5168251361296384: Issue 170 - Replacing Mercurial subrepositories (Closed)
Left Patch Set: Do not reload when used as module Created Sept. 4, 2014, 5:16 p.m.
Right Patch Set: Addressed remaining comments Created Sept. 9, 2014, 3:46 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « no previous file | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # coding: utf-8 2 # coding: utf-8
3 3
4 # This file is part of the Adblock Plus build tools, 4 # This file is part of the Adblock Plus build tools,
5 # Copyright (C) 2006-2014 Eyeo GmbH 5 # Copyright (C) 2006-2014 Eyeo GmbH
6 # 6 #
7 # Adblock Plus is free software: you can redistribute it and/or modify 7 # Adblock Plus is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as 8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation. 9 # published by the Free Software Foundation.
10 # 10 #
11 # Adblock Plus is distributed in the hope that it will be useful, 11 # Adblock Plus is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details. 14 # GNU General Public License for more details.
15 # 15 #
16 # You should have received a copy of the GNU General Public License 16 # You should have received a copy of the GNU General Public License
17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
18 18
19 import sys 19 import sys
20 import os 20 import os
21 import posixpath
21 import re 22 import re
22 import io 23 import io
23 import errno 24 import errno
24 import logging 25 import logging
25 import subprocess 26 import subprocess
26 import urlparse 27 import urlparse
27 from collections import OrderedDict 28 from collections import OrderedDict
29
30 USAGE = """
31 A dependencies file should look like this:
32
33 # VCS-specific root URLs for the repositories
34 _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
35 # File to update this script from (optional)
36 _self = buildtools/ensure_dependencies.py
37 # Check out elemhidehelper repository into extensions/elemhidehelper directory
38 # at tag "1.2".
39 extensions/elemhidehelper = elemhidehelper 1.2
40 # Check out buildtools repository into buildtools directory at VCS-specific
41 # revision IDs.
42 buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
43 """
28 44
29 class Mercurial(): 45 class Mercurial():
30 def istype(self, repodir): 46 def istype(self, repodir):
31 return os.path.exists(os.path.join(repodir, ".hg")) 47 return os.path.exists(os.path.join(repodir, ".hg"))
32 48
33 def clone(self, source, target): 49 def clone(self, source, target):
34 if not source.endswith("/"): 50 if not source.endswith("/"):
35 source += "/" 51 source += "/"
36 subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, targe t]) 52 subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, targe t])
37 53
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after
117 key, spec = parse_spec(deps_path, line) 133 key, spec = parse_spec(deps_path, line)
118 if spec: 134 if spec:
119 result[key] = spec 135 result[key] = spec
120 return result 136 return result
121 except IOError, e: 137 except IOError, e:
122 if e.errno != errno.ENOENT: 138 if e.errno != errno.ENOENT:
123 raise 139 raise
124 return None 140 return None
125 141
126 def safe_join(path, subpath): 142 def safe_join(path, subpath):
127 return os.path.join(path, *[f for f in subpath.split("/") if f not in (os.curd ir, os.pardir)]) 143 # This has been inspired by Flask's safe_join() function
Felix Dahlke 2014/09/05 08:55:53 So we're ignoring . and .. in paths? Shouldn't we
Sebastian Noack 2014/09/05 09:29:31 It's common practice to just ignore '..' when sani
Felix Dahlke 2014/09/05 09:55:01 We're using this for local paths, don't we? Otherw
Wladimir Palant 2014/09/05 18:37:39 This implementation was indeed overly simplistic,
144 forbidden = set([os.sep, os.altsep]) - set([posixpath.sep, None])
145 if any(sep in subpath for sep in forbidden):
146 raise Exception("Illegal directory separator in dependency path %s" % subpat h)
147
148 normpath = posixpath.normpath(subpath)
149 if posixpath.isabs(normpath):
150 raise Exception("Dependency path %s cannot be absolute" % subpath)
151 if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posi xpath.sep):
152 raise Exception("Dependency path %s has to be inside the repository" % subpa th)
153 return os.path.join(path, *normpath.split(posixpath.sep))
128 154
129 def get_repo_type(repo): 155 def get_repo_type(repo):
130 for name, repotype in repo_types.iteritems(): 156 for name, repotype in repo_types.iteritems():
131 if repotype.istype(repo): 157 if repotype.istype(repo):
132 return name 158 return name
133 return None 159 return None
134 160
135 def ensure_repo(parentrepo, target, roots, sourcename): 161 def ensure_repo(parentrepo, target, roots, sourcename):
136 if os.path.exists(target): 162 if os.path.exists(target):
137 return 163 return
(...skipping 30 matching lines...) Expand all
168 repo_types[type].pull(target) 194 repo_types[type].pull(target)
169 resolved_revision = repo_types[type].get_revision_id(target, revision) 195 resolved_revision = repo_types[type].get_revision_id(target, revision)
170 if not resolved_revision: 196 if not resolved_revision:
171 raise Exception("Failed to resolve revision %s" % revision) 197 raise Exception("Failed to resolve revision %s" % revision)
172 198
173 current_revision = repo_types[type].get_revision_id(target) 199 current_revision = repo_types[type].get_revision_id(target)
174 if resolved_revision != current_revision: 200 if resolved_revision != current_revision:
175 logging.info("Updating repository %s to revision %s" % (target, resolved_rev ision)) 201 logging.info("Updating repository %s to revision %s" % (target, resolved_rev ision))
176 repo_types[type].update(target, resolved_revision) 202 repo_types[type].update(target, resolved_revision)
177 203
178 def resolve_deps(repodir, level=0): 204 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdep endencies=set()):
179 config = read_deps(repodir) 205 config = read_deps(repodir)
180 if config is None: 206 if config is None:
181 if level == 0: 207 if level == 0:
182 logging.warning("No dependencies file in directory %s, nothing to do..." % repodir) 208 logging.warning("No dependencies file in directory %s, nothing to do...\n% s" % (repodir, USAGE))
183 return 209 return
184 if level >= 10: 210 if level >= 10:
185 logging.warning("Too much subrepository nesting, ignoring %s" % repo) 211 logging.warning("Too much subrepository nesting, ignoring %s" % repo)
186 212
187 for dir in config: 213 if overrideroots is not None:
Felix Dahlke 2014/09/05 08:55:53 Why not do something like |for dir, dep in config|
Wladimir Palant 2014/09/05 18:37:39 Done.
188 if dir.startswith("_"): 214 config["_root"] = overrideroots
215
216 for dir, revisions in config.iteritems():
217 if dir.startswith("_") or revisions["_source"] in skipdependencies:
189 continue 218 continue
190 target = safe_join(repodir, dir) 219 target = safe_join(repodir, dir)
191 ensure_repo(repodir, target, config.get("_root", {}), config[dir]["_source"] ) 220 ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"])
192 update_repo(target, config[dir]) 221 update_repo(target, revisions)
193 resolve_deps(target, level + 1) 222 resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroo ts, skipdependencies=skipdependencies)
194 223
195 if level == 0 and "_self" in config and "*" in config["_self"]: 224 if self_update and "_self" in config and "*" in config["_self"]:
196 source = safe_join(repodir, config["_self"]["*"]) 225 source = safe_join(repodir, config["_self"]["*"])
197 try: 226 try:
198 with io.open(source, "rb") as handle: 227 with io.open(source, "rb") as handle:
199 sourcedata = handle.read() 228 sourcedata = handle.read()
200 except IOError, e: 229 except IOError, e:
201 if e.errno != errno.ENOENT: 230 if e.errno != errno.ENOENT:
202 raise 231 raise
203 logging.warning("File %s doesn't exist, skipping self-update" % source) 232 logging.warning("File %s doesn't exist, skipping self-update" % source)
204 return 233 return
205 234
(...skipping 11 matching lines...) Expand all
217 else: 246 else:
218 logging.warning("Cannot restart %s automatically, please rerun" % target ) 247 logging.warning("Cannot restart %s automatically, please rerun" % target )
219 248
220 if __name__ == "__main__": 249 if __name__ == "__main__":
221 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) 250 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
222 repos = sys.argv[1:] 251 repos = sys.argv[1:]
223 if not len(repos): 252 if not len(repos):
224 repos = [os.getcwd()] 253 repos = [os.getcwd()]
225 for repo in repos: 254 for repo in repos:
226 resolve_deps(repo) 255 resolve_deps(repo)
LEFTRIGHT
« no previous file | no next file » | Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Toggle Comments ('s')

Powered by Google App Engine
This is Rietveld