Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: ensure_dependencies.py

Issue 5924365505921024: Use ensure_dependencies.py instead of subrepositories (Closed)
Patch Set: Created March 13, 2015, 8:55 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« dependencies ('K') | « dependencies ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 # This Source Code Form is subject to the terms of the Mozilla Public
5 # License, v. 2.0. If a copy of the MPL was not distributed with this
6 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
7
8 import sys
9 import os
10 import posixpath
11 import re
12 import io
13 import errno
14 import logging
15 import subprocess
16 import urlparse
17 import argparse
18
19 from collections import OrderedDict
20 from ConfigParser import RawConfigParser
21
22 USAGE = """
23 A dependencies file should look like this:
24
25 # VCS-specific root URLs for the repositories
26 _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
27 # File to update this script from (optional)
28 _self = buildtools/ensure_dependencies.py
29 # Check out elemhidehelper repository into extensions/elemhidehelper directory
30 # at tag "1.2".
31 extensions/elemhidehelper = elemhidehelper 1.2
32 # Check out buildtools repository into buildtools directory at VCS-specific
33 # revision IDs.
34 buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
35 """
36
37 class Mercurial():
38 def istype(self, repodir):
39 return os.path.exists(os.path.join(repodir, ".hg"))
40
41 def clone(self, source, target):
42 if not source.endswith("/"):
43 source += "/"
44 subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, targe t])
45
46 def get_revision_id(self, repo, rev=None):
47 command = ["hg", "id", "--repository", repo, "--id"]
48 if rev:
49 command.extend(["--rev", rev])
50
51 # Ignore stderr output and return code here: if revision lookup failed we
52 # should simply return an empty string.
53 result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess .PIPE).communicate()[0]
54 return result.strip()
55
56 def pull(self, repo):
57 subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"])
58
59 def update(self, repo, rev):
60 subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--c heck", "--rev", rev])
61
62 def ignore(self, target, repo):
63
64 if not self.istype(target):
65
66 config_path = os.path.join(repo, ".hg", "hgrc")
67 ignore_path = os.path.abspath(os.path.join(repo, ".hg", "dependencies"))
68
69 config = RawConfigParser()
70 config.read(config_path)
71
72 if not config.has_section("ui"):
73 config.add_section("ui")
74
75 config.set("ui", "ignore.dependencies", ignore_path)
76 with open(config_path, "w") as stream:
77 config.write(stream)
78
79 module = os.path.relpath(target, repo)
80 _ensure_line_exists(ignore_path, module)
81
82 class Git():
83 def istype(self, repodir):
84 return os.path.exists(os.path.join(repodir, ".git"))
85
86 def clone(self, source, target):
87 source = source.rstrip("/")
88 if not source.endswith(".git"):
89 source += ".git"
90 subprocess.check_call(["git", "clone", "--quiet", source, target])
91
92 def get_revision_id(self, repo, rev="HEAD"):
93 command = ["git", "rev-parse", "--revs-only", rev + '^{commit}']
94 return subprocess.check_output(command, cwd=repo).strip()
95
96 def pull(self, repo):
97 subprocess.check_call(["git", "fetch", "--quiet", "--all", "--tags"], cwd=re po)
98
99 def update(self, repo, rev):
100 subprocess.check_call(["git", "checkout", "--quiet", rev], cwd=repo)
101
102 def ignore(self, target, repo):
103 module = os.path.relpath(target, repo)
104 exclude_file = os.path.join(repo, ".git", "info", "exclude")
105 _ensure_line_exists(exclude_file, module)
106
107 repo_types = OrderedDict((
108 ("hg", Mercurial()),
109 ("git", Git()),
110 ))
111
112 def parse_spec(path, line):
113 if "=" not in line:
114 logging.warning("Invalid line in file %s: %s" % (path, line))
115 return None, None
116
117 key, value = line.split("=", 1)
118 key = key.strip()
119 items = value.split()
120 if not len(items):
121 logging.warning("No value specified for key %s in file %s" % (key, path))
122 return key, None
123
124 result = OrderedDict()
125 if not key.startswith("_"):
126 result["_source"] = items.pop(0)
127
128 for item in items:
129 if ":" in item:
130 type, value = item.split(":", 1)
131 else:
132 type, value = ("*", item)
133 if type in result:
134 logging.warning("Ignoring duplicate value for type %s (key %s in file %s)" % (type, key, path))
135 else:
136 result[type] = value
137 return key, result
138
139 def read_deps(repodir):
140 result = {}
141 deps_path = os.path.join(repodir, "dependencies")
142 try:
143 with io.open(deps_path, "rt", encoding="utf-8") as handle:
144 for line in handle:
145 # Remove comments and whitespace
146 line = re.sub(r"#.*", "", line).strip()
147 if not line:
148 continue
149
150 key, spec = parse_spec(deps_path, line)
151 if spec:
152 result[key] = spec
153 return result
154 except IOError, e:
155 if e.errno != errno.ENOENT:
156 raise
157 return None
158
159 def safe_join(path, subpath):
160 # This has been inspired by Flask's safe_join() function
161 forbidden = set([os.sep, os.altsep]) - set([posixpath.sep, None])
162 if any(sep in subpath for sep in forbidden):
163 raise Exception("Illegal directory separator in dependency path %s" % subpat h)
164
165 normpath = posixpath.normpath(subpath)
166 if posixpath.isabs(normpath):
167 raise Exception("Dependency path %s cannot be absolute" % subpath)
168 if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posi xpath.sep):
169 raise Exception("Dependency path %s has to be inside the repository" % subpa th)
170 return os.path.join(path, *normpath.split(posixpath.sep))
171
172 def get_repo_type(repo):
173 for name, repotype in repo_types.iteritems():
174 if repotype.istype(repo):
175 return name
176 return None
177
178 def ensure_repo(parentrepo, target, roots, sourcename):
179 if os.path.exists(target):
180 return
181
182 parenttype = get_repo_type(parentrepo)
183 type = None
184 for key in roots:
185 if key == parenttype or (key in repo_types and type is None):
186 type = key
187 if type is None:
188 raise Exception("No valid source found to create %s" % target)
189
190 if os.path.exists(roots[type]):
191 url = os.path.join(roots[type], sourcename)
192 else:
193 url = urlparse.urljoin(roots[type], sourcename)
194
195 logging.info("Cloning repository %s into %s" % (url, target))
196 repo_types[type].clone(url, target)
197
198 for repo in repo_types.itervalues():
199 if repo.istype(parentrepo):
200 repo.ignore(target, parentrepo)
201
202 def update_repo(target, revisions):
203 type = get_repo_type(target)
204 if type is None:
205 logging.warning("Type of repository %s unknown, skipping update" % target)
206 return
207
208 if type in revisions:
209 revision = revisions[type]
210 elif "*" in revisions:
211 revision = revisions["*"]
212 else:
213 logging.warning("No revision specified for repository %s (type %s), skipping update" % (target, type))
214 return
215
216 resolved_revision = repo_types[type].get_revision_id(target, revision)
217 if not resolved_revision:
218 logging.info("Revision %s is unknown, downloading remote changes" % revision )
219 repo_types[type].pull(target)
220 resolved_revision = repo_types[type].get_revision_id(target, revision)
221 if not resolved_revision:
222 raise Exception("Failed to resolve revision %s" % revision)
223
224 current_revision = repo_types[type].get_revision_id(target)
225 if resolved_revision != current_revision:
226 logging.info("Updating repository %s to revision %s" % (target, resolved_rev ision))
227 repo_types[type].update(target, resolved_revision)
228
229 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdep endencies=set()):
230 config = read_deps(repodir)
231 if config is None:
232 if level == 0:
233 logging.warning("No dependencies file in directory %s, nothing to do...\n% s" % (repodir, USAGE))
234 return
235 if level >= 10:
236 logging.warning("Too much subrepository nesting, ignoring %s" % repo)
237
238 if overrideroots is not None:
239 config["_root"] = overrideroots
240
241 for dir, revisions in config.iteritems():
242 if dir.startswith("_") or revisions["_source"] in skipdependencies:
243 continue
244 target = safe_join(repodir, dir)
245 ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"])
246 update_repo(target, revisions)
247 resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroo ts, skipdependencies=skipdependencies)
248
249 if self_update and "_self" in config and "*" in config["_self"]:
250 source = safe_join(repodir, config["_self"]["*"])
251 try:
252 with io.open(source, "rb") as handle:
253 sourcedata = handle.read()
254 except IOError, e:
255 if e.errno != errno.ENOENT:
256 raise
257 logging.warning("File %s doesn't exist, skipping self-update" % source)
258 return
259
260 target = __file__
261 with io.open(target, "rb") as handle:
262 targetdata = handle.read()
263
264 if sourcedata != targetdata:
265 logging.info("Updating %s from %s, don't forget to commit" % (source, targ et))
266 with io.open(target, "wb") as handle:
267 handle.write(sourcedata)
268 if __name__ == "__main__":
269 logging.info("Restarting %s" % target)
270 os.execv(sys.executable, [sys.executable, target] + sys.argv[1:])
271 else:
272 logging.warning("Cannot restart %s automatically, please rerun" % target )
273
274 def _ensure_line_exists(path, pattern):
275 with open(path, 'a+') as f:
276 file_content = [l.strip() for l in f.readlines()]
277 if not pattern in file_content:
278 file_content.append(pattern)
279 f.seek(0, os.SEEK_SET)
280 f.truncate()
281 for l in file_content:
282 print >>f, l
283
284 if __name__ == "__main__":
285 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
286
287 parser = argparse.ArgumentParser(description="Verify dependencies for a set of repositories, by default the repository of this script.")
288 parser.add_argument("repos", metavar="repository", type=str, nargs="*", help=" Repository path")
289 parser.add_argument("-q", "--quiet", action="store_true", help="Suppress infor mational output")
290 args = parser.parse_args()
291
292 if args.quiet:
293 logging.disable(logging.INFO)
294
295 repos = args.repos
296 if not len(repos):
297 repos = [os.path.dirname(__file__)]
298 for repo in repos:
299 resolve_deps(repo)
OLDNEW
« dependencies ('K') | « dependencies ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld