Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: ensure_dependencies.py

Issue 4768985027444736: 1369 - Integrate ensure_dependencies.py (Closed)
Patch Set: Updates since last patch set: Created Sept. 18, 2014, 2:10 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « dependencies ('k') | kick.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 # This file is part of the Adblock Plus build tools,
5 # Copyright (C) 2006-2014 Eyeo GmbH
6 #
7 # Adblock Plus is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation.
10 #
11 # Adblock Plus is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
18
19 import sys
20 import os
21 import posixpath
22 import re
23 import io
24 import errno
25 import logging
26 import subprocess
27 import urlparse
28 from collections import OrderedDict
29
30 USAGE = """
31 A dependencies file should look like this:
32
33 # VCS-specific root URLs for the repositories
34 _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
35 # File to update this script from (optional)
36 _self = buildtools/ensure_dependencies.py
37 # Check out elemhidehelper repository into extensions/elemhidehelper directory
38 # at tag "1.2".
39 extensions/elemhidehelper = elemhidehelper 1.2
40 # Check out buildtools repository into buildtools directory at VCS-specific
41 # revision IDs.
42 buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
43 """
44
45 class Mercurial():
46 def istype(self, repodir):
47 return os.path.exists(os.path.join(repodir, ".hg"))
48
49 def clone(self, source, target):
50 if not source.endswith("/"):
51 source += "/"
52 subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, targe t])
53
54 def get_revision_id(self, repo, rev=None):
55 command = ["hg", "id", "--repository", repo, "--id"]
56 if rev:
57 command.extend(["--rev", rev])
58
59 # Ignore stderr output and return code here: if revision lookup failed we
60 # should simply return an empty string.
61 result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess .PIPE).communicate()[0]
62 return result.strip()
63
64 def pull(self, repo):
65 subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"])
66
67 def update(self, repo, rev):
68 subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--c heck", "--rev", rev])
69
70 class Git():
71 def istype(self, repodir):
72 return os.path.exists(os.path.join(repodir, ".git"))
73
74 def clone(self, source, target):
75 source = source.rstrip("/")
76 if not source.endswith(".git"):
77 source += ".git"
78 subprocess.check_call(["git", "clone", "--quiet", source, target])
79
80 def get_revision_id(self, repo, rev="HEAD"):
81 command = ["git", "-C", repo, "rev-parse", "--revs-only", rev]
82 return subprocess.check_output(command).strip()
83
84 def pull(self, repo):
85 subprocess.check_call(["git", "-C", repo, "fetch", "--quiet", "--all", "--ta gs"])
86
87 def update(self, repo, rev):
88 subprocess.check_call(["git", "-C", repo, "checkout", "--quiet", rev])
89
90 repo_types = {
91 "hg": Mercurial(),
92 "git": Git(),
93 }
94
95 def parse_spec(path, line):
96 if "=" not in line:
97 logging.warning("Invalid line in file %s: %s" % (path, line))
98 return None, None
99
100 key, value = line.split("=", 1)
101 key = key.strip()
102 items = value.split()
103 if not len(items):
104 logging.warning("No value specified for key %s in file %s" % (key, path))
105 return key, None
106
107 result = OrderedDict()
108 if not key.startswith("_"):
109 result["_source"] = items.pop(0)
110
111 for item in items:
112 if ":" in item:
113 type, value = item.split(":", 1)
114 else:
115 type, value = ("*", item)
116 if type in result:
117 logging.warning("Ignoring duplicate value for type %s (key %s in file %s)" % (type, key, path))
118 else:
119 result[type] = value
120 return key, result
121
122 def read_deps(repodir):
123 result = {}
124 deps_path = os.path.join(repodir, "dependencies")
125 try:
126 with io.open(deps_path, "rt", encoding="utf-8") as handle:
127 for line in handle:
128 # Remove comments and whitespace
129 line = re.sub(r"#.*", "", line).strip()
130 if not line:
131 continue
132
133 key, spec = parse_spec(deps_path, line)
134 if spec:
135 result[key] = spec
136 return result
137 except IOError, e:
138 if e.errno != errno.ENOENT:
139 raise
140 return None
141
142 def safe_join(path, subpath):
143 # This has been inspired by Flask's safe_join() function
144 forbidden = set([os.sep, os.altsep]) - set([posixpath.sep, None])
145 if any(sep in subpath for sep in forbidden):
146 raise Exception("Illegal directory separator in dependency path %s" % subpat h)
147
148 normpath = posixpath.normpath(subpath)
149 if posixpath.isabs(normpath):
150 raise Exception("Dependency path %s cannot be absolute" % subpath)
151 if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posi xpath.sep):
152 raise Exception("Dependency path %s has to be inside the repository" % subpa th)
153 return os.path.join(path, *normpath.split(posixpath.sep))
154
155 def get_repo_type(repo):
156 for name, repotype in repo_types.iteritems():
157 if repotype.istype(repo):
158 return name
159 return None
160
161 def ensure_repo(parentrepo, target, roots, sourcename):
162 if os.path.exists(target):
163 return
164
165 parenttype = get_repo_type(parentrepo)
166 type = None
167 for key in roots:
168 if key == parenttype or (key in repo_types and type is None):
169 type = key
170 if type is None:
171 raise Exception("No valid source found to create %s" % target)
172
173 url = urlparse.urljoin(roots[type], sourcename)
174 logging.info("Cloning repository %s into %s" % (url, target))
175 repo_types[type].clone(url, target)
176
177 def update_repo(target, revisions):
178 type = get_repo_type(target)
179 if type is None:
180 logging.warning("Type of repository %s unknown, skipping update" % target)
181 return
182
183 if type in revisions:
184 revision = revisions[type]
185 elif "*" in revisions:
186 revision = revisions["*"]
187 else:
188 logging.warning("No revision specified for repository %s (type %s), skipping update" % (target, type))
189 return
190
191 resolved_revision = repo_types[type].get_revision_id(target, revision)
192 if not resolved_revision:
193 logging.info("Revision %s is unknown, downloading remote changes" % revision )
194 repo_types[type].pull(target)
195 resolved_revision = repo_types[type].get_revision_id(target, revision)
196 if not resolved_revision:
197 raise Exception("Failed to resolve revision %s" % revision)
198
199 current_revision = repo_types[type].get_revision_id(target)
200 if resolved_revision != current_revision:
201 logging.info("Updating repository %s to revision %s" % (target, resolved_rev ision))
202 repo_types[type].update(target, resolved_revision)
203
204 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdep endencies=set()):
205 config = read_deps(repodir)
206 if config is None:
207 if level == 0:
208 logging.warning("No dependencies file in directory %s, nothing to do...\n% s" % (repodir, USAGE))
209 return
210 if level >= 10:
211 logging.warning("Too much subrepository nesting, ignoring %s" % repo)
212
213 if overrideroots is not None:
214 config["_root"] = overrideroots
215
216 for dir, revisions in config.iteritems():
217 if dir.startswith("_") or revisions["_source"] in skipdependencies:
218 continue
219 target = safe_join(repodir, dir)
220 ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"])
221 update_repo(target, revisions)
222 resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroo ts, skipdependencies=skipdependencies)
223
224 if self_update and "_self" in config and "*" in config["_self"]:
225 source = safe_join(repodir, config["_self"]["*"])
226 try:
227 with io.open(source, "rb") as handle:
228 sourcedata = handle.read()
229 except IOError, e:
230 if e.errno != errno.ENOENT:
231 raise
232 logging.warning("File %s doesn't exist, skipping self-update" % source)
233 return
234
235 target = __file__
236 with io.open(target, "rb") as handle:
237 targetdata = handle.read()
238
239 if sourcedata != targetdata:
240 logging.info("Updating %s from %s, don't forget to commit" % (source, targ et))
241 with io.open(target, "wb") as handle:
242 handle.write(sourcedata)
243 if __name__ == "__main__":
244 logging.info("Restarting %s" % target)
245 os.execv(sys.executable, [sys.executable, target] + sys.argv[1:])
246 else:
247 logging.warning("Cannot restart %s automatically, please rerun" % target )
248
249 if __name__ == "__main__":
250 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
251 repos = sys.argv[1:]
252 if not len(repos):
253 repos = [os.getcwd()]
254 for repo in repos:
255 resolve_deps(repo)
OLDNEW
« no previous file with comments | « dependencies ('k') | kick.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld