Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: ensure_dependencies.py

Issue 5924365505921024: Use ensure_dependencies.py instead of subrepositories (Closed)
Patch Set: Use buildtools transitively Created March 17, 2015, 2:47 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « dependencies ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 # This Source Code Form is subject to the terms of the Mozilla Public
5 # License, v. 2.0. If a copy of the MPL was not distributed with this
6 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
7
8 import sys
9 import os
10 import posixpath
11 import re
12 import io
13 import errno
14 import logging
15 import subprocess
16 import urlparse
17
18 from collections import OrderedDict
19 from ConfigParser import RawConfigParser
20
21 USAGE = """
22 A dependencies file should look like this:
23
24 # VCS-specific root URLs for the repositories
25 _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
26 # File to update this script from (optional)
27 _self = buildtools/ensure_dependencies.py
28 # Check out elemhidehelper repository into extensions/elemhidehelper directory
29 # at tag "1.2".
30 extensions/elemhidehelper = elemhidehelper 1.2
31 # Check out buildtools repository into buildtools directory at VCS-specific
32 # revision IDs.
33 buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
34 """
35
36 class Mercurial():
37 def istype(self, repodir):
38 return os.path.exists(os.path.join(repodir, ".hg"))
39
40 def clone(self, source, target):
41 if not source.endswith("/"):
42 source += "/"
43 subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, targe t])
44
45 def get_revision_id(self, repo, rev=None):
46 command = ["hg", "id", "--repository", repo, "--id"]
47 if rev:
48 command.extend(["--rev", rev])
49
50 # Ignore stderr output and return code here: if revision lookup failed we
51 # should simply return an empty string.
52 result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess .PIPE).communicate()[0]
53 return result.strip()
54
55 def pull(self, repo):
56 subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"])
57
58 def update(self, repo, rev):
59 subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--c heck", "--rev", rev])
60
61 def ignore(self, target, repo):
62
63 if not self.istype(target):
64
65 config_path = os.path.join(repo, ".hg", "hgrc")
66 ignore_path = os.path.abspath(os.path.join(repo, ".hg", "dependencies"))
67
68 config = RawConfigParser()
69 config.read(config_path)
70
71 if not config.has_section("ui"):
72 config.add_section("ui")
73
74 config.set("ui", "ignore.dependencies", ignore_path)
75 with open(config_path, "w") as stream:
76 config.write(stream)
77
78 module = os.path.relpath(target, repo)
79 _ensure_line_exists(ignore_path, module)
80
81 class Git():
82 def istype(self, repodir):
83 return os.path.exists(os.path.join(repodir, ".git"))
84
85 def clone(self, source, target):
86 source = source.rstrip("/")
87 if not source.endswith(".git"):
88 source += ".git"
89 subprocess.check_call(["git", "clone", "--quiet", source, target])
90
91 def get_revision_id(self, repo, rev="HEAD"):
92 command = ["git", "rev-parse", "--revs-only", rev + '^{commit}']
93 return subprocess.check_output(command, cwd=repo).strip()
94
95 def pull(self, repo):
96 subprocess.check_call(["git", "fetch", "--quiet", "--all", "--tags"], cwd=re po)
97
98 def update(self, repo, rev):
99 subprocess.check_call(["git", "checkout", "--quiet", rev], cwd=repo)
100
101 def ignore(self, target, repo):
102 module = os.path.relpath(target, repo)
103 exclude_file = os.path.join(repo, ".git", "info", "exclude")
104 _ensure_line_exists(exclude_file, module)
105
106 repo_types = OrderedDict((
107 ("hg", Mercurial()),
108 ("git", Git()),
109 ))
110
111 def parse_spec(path, line):
112 if "=" not in line:
113 logging.warning("Invalid line in file %s: %s" % (path, line))
114 return None, None
115
116 key, value = line.split("=", 1)
117 key = key.strip()
118 items = value.split()
119 if not len(items):
120 logging.warning("No value specified for key %s in file %s" % (key, path))
121 return key, None
122
123 result = OrderedDict()
124 if not key.startswith("_"):
125 result["_source"] = items.pop(0)
126
127 for item in items:
128 if ":" in item:
129 type, value = item.split(":", 1)
130 else:
131 type, value = ("*", item)
132 if type in result:
133 logging.warning("Ignoring duplicate value for type %s (key %s in file %s)" % (type, key, path))
134 else:
135 result[type] = value
136 return key, result
137
138 def read_deps(repodir):
139 result = {}
140 deps_path = os.path.join(repodir, "dependencies")
141 try:
142 with io.open(deps_path, "rt", encoding="utf-8") as handle:
143 for line in handle:
144 # Remove comments and whitespace
145 line = re.sub(r"#.*", "", line).strip()
146 if not line:
147 continue
148
149 key, spec = parse_spec(deps_path, line)
150 if spec:
151 result[key] = spec
152 return result
153 except IOError, e:
154 if e.errno != errno.ENOENT:
155 raise
156 return None
157
158 def safe_join(path, subpath):
159 # This has been inspired by Flask's safe_join() function
160 forbidden = set([os.sep, os.altsep]) - set([posixpath.sep, None])
161 if any(sep in subpath for sep in forbidden):
162 raise Exception("Illegal directory separator in dependency path %s" % subpat h)
163
164 normpath = posixpath.normpath(subpath)
165 if posixpath.isabs(normpath):
166 raise Exception("Dependency path %s cannot be absolute" % subpath)
167 if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posi xpath.sep):
168 raise Exception("Dependency path %s has to be inside the repository" % subpa th)
169 return os.path.join(path, *normpath.split(posixpath.sep))
170
171 def get_repo_type(repo):
172 for name, repotype in repo_types.iteritems():
173 if repotype.istype(repo):
174 return name
175 return None
176
177 def ensure_repo(parentrepo, target, roots, sourcename):
178 if os.path.exists(target):
179 return
180
181 parenttype = get_repo_type(parentrepo)
182 type = None
183 for key in roots:
184 if key == parenttype or (key in repo_types and type is None):
185 type = key
186 if type is None:
187 raise Exception("No valid source found to create %s" % target)
188
189 url = urlparse.urljoin(roots[type], sourcename)
190 logging.info("Cloning repository %s into %s" % (url, target))
191 repo_types[type].clone(url, target)
192
193 for repo in repo_types.itervalues():
194 if repo.istype(parentrepo):
195 repo.ignore(target, parentrepo)
196
197 def update_repo(target, revisions):
198 type = get_repo_type(target)
199 if type is None:
200 logging.warning("Type of repository %s unknown, skipping update" % target)
201 return
202
203 if type in revisions:
204 revision = revisions[type]
205 elif "*" in revisions:
206 revision = revisions["*"]
207 else:
208 logging.warning("No revision specified for repository %s (type %s), skipping update" % (target, type))
209 return
210
211 resolved_revision = repo_types[type].get_revision_id(target, revision)
212 if not resolved_revision:
213 logging.info("Revision %s is unknown, downloading remote changes" % revision )
214 repo_types[type].pull(target)
215 resolved_revision = repo_types[type].get_revision_id(target, revision)
216 if not resolved_revision:
217 raise Exception("Failed to resolve revision %s" % revision)
218
219 current_revision = repo_types[type].get_revision_id(target)
220 if resolved_revision != current_revision:
221 logging.info("Updating repository %s to revision %s" % (target, resolved_rev ision))
222 repo_types[type].update(target, resolved_revision)
223
224 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdep endencies=set()):
225 config = read_deps(repodir)
226 if config is None:
227 if level == 0:
228 logging.warning("No dependencies file in directory %s, nothing to do...\n% s" % (repodir, USAGE))
229 return
230 if level >= 10:
231 logging.warning("Too much subrepository nesting, ignoring %s" % repo)
232
233 if overrideroots is not None:
234 config["_root"] = overrideroots
235
236 for dir, revisions in config.iteritems():
237 if dir.startswith("_") or revisions["_source"] in skipdependencies:
238 continue
239 target = safe_join(repodir, dir)
240 ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"])
241 update_repo(target, revisions)
242 resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroo ts, skipdependencies=skipdependencies)
243
244 if self_update and "_self" in config and "*" in config["_self"]:
245 source = safe_join(repodir, config["_self"]["*"])
246 try:
247 with io.open(source, "rb") as handle:
248 sourcedata = handle.read()
249 except IOError, e:
250 if e.errno != errno.ENOENT:
251 raise
252 logging.warning("File %s doesn't exist, skipping self-update" % source)
253 return
254
255 target = __file__
256 with io.open(target, "rb") as handle:
257 targetdata = handle.read()
258
259 if sourcedata != targetdata:
260 logging.info("Updating %s from %s, don't forget to commit" % (source, targ et))
261 with io.open(target, "wb") as handle:
262 handle.write(sourcedata)
263 if __name__ == "__main__":
264 logging.info("Restarting %s" % target)
265 os.execv(sys.executable, [sys.executable, target] + sys.argv[1:])
266 else:
267 logging.warning("Cannot restart %s automatically, please rerun" % target )
268
269 def _ensure_line_exists(path, pattern):
270 with open(path, 'a+') as f:
271 file_content = [l.strip() for l in f.readlines()]
272 if not pattern in file_content:
273 file_content.append(pattern)
274 f.seek(0, os.SEEK_SET)
275 f.truncate()
276 for l in file_content:
277 print >>f, l
278
279 if __name__ == "__main__":
280 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
281 repos = sys.argv[1:]
282 if not len(repos):
283 repos = [os.getcwd()]
284 for repo in repos:
285 resolve_deps(repo)
OLDNEW
« no previous file with comments | « dependencies ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld