Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: ensure_dependencies.py

Issue 5168251361296384: Issue 170 - Replacing Mercurial subrepositories (Closed)
Patch Set: Removed useless exit() call Created Sept. 4, 2014, 2:33 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 # This file is part of the Adblock Plus build tools,
5 # Copyright (C) 2006-2014 Eyeo GmbH
6 #
7 # Adblock Plus is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation.
10 #
11 # Adblock Plus is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
18
19 import sys
20 import os
21 import re
22 import io
23 import errno
24 import logging
25 import subprocess
26 import urlparse
27 from collections import OrderedDict
28
29 class Mercurial():
30 def istype(self, repodir):
31 return os.path.exists(os.path.join(repodir, ".hg"))
32
33 def clone(self, source, target):
34 if not source.endswith("/"):
35 source += "/"
36 subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, targe t])
37
38 def get_revision_id(self, repo, rev=None):
39 command = ["hg", "id", "--repository", repo, "--id"]
40 if rev:
41 command.extend(["--rev", rev])
42
43 # Ignore stderr output and return code here: if revision lookup failed we
44 # should simply return an empty string.
45 result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess .PIPE).communicate()[0]
46 return result.strip()
47
48 def pull(self, repo):
49 subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"])
50
51 def update(self, repo, rev):
52 subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--c heck", "--rev", rev])
53
54 class Git():
55 def istype(self, repodir):
56 return os.path.exists(os.path.join(repodir, ".git"))
57
58 def clone(self, source, target):
59 source = source.rstrip("/")
60 if not source.endswith(".git"):
61 source += ".git"
62 subprocess.check_call(["git", "clone", "--quiet", "--no-checkout", source, t arget])
63
64 def get_revision_id(self, repo, rev="HEAD"):
65 command = ["git", "-C", repo, "rev-parse", "--revs-only", rev]
66 return subprocess.check_output(command).strip()
67
68 def pull(self, repo):
69 subprocess.check_call(["git", "-C", repo, "fetch", "--quiet", "--all", "--ta gs"])
70
71 def update(self, repo, rev):
72 subprocess.check_call(["git", "-C", repo, "checkout", "--quiet", rev])
73
74 repo_types = {
75 "hg": Mercurial(),
76 "git": Git(),
77 }
78
79 def parse_spec(path, line):
80 if "=" not in line:
81 logging.warning("Invalid line in file %s: %s" % (path, line))
82 return None, None
83
84 key, value = line.split("=", 1)
85 key = key.strip()
86 items = value.split()
87 if not len(items):
88 logging.warning("No value specified for key %s in file %s" % (key, path))
89 return key, None
90
91 result = OrderedDict()
92 if not key.startswith("_"):
93 result["_source"] = items.pop(0)
94
95 for item in items:
96 if ":" in item:
97 type, value = item.split(":", 1)
98 else:
99 type, value = ("*", item)
100 if type in result:
101 logging.warning("Ignoring duplicate value for type %s (key %s in file %s)" % (type, key, path))
102 else:
103 result[type] = value
104 return key, result
105
106 def read_deps(repodir):
107 result = {}
108 deps_path = os.path.join(repodir, "dependencies")
109 try:
110 with io.open(deps_path, "rt", encoding="utf-8") as handle:
111 for line in handle:
112 # Remove comments and whitespace
113 line = re.sub(r"#.*", "", line).strip()
114 if not line:
115 continue
116
117 key, spec = parse_spec(deps_path, line)
118 if spec:
119 result[key] = spec
120 return result
121 except IOError, e:
122 if e.errno != errno.ENOENT:
123 raise
124 return None
125
126 def safe_join(path, subpath):
127 return os.path.join(path, *[f for f in subpath.split("/") if f not in (os.curd ir, os.pardir)])
128
129 def get_repo_type(repo):
130 for name, repotype in repo_types.iteritems():
131 if repotype.istype(repo):
132 return name
133 return None
134
135 def ensure_repo(parentrepo, target, roots, sourcename):
136 if os.path.exists(target):
137 return
138
139 parenttype = get_repo_type(parentrepo)
140 type = None
141 for key in roots:
142 if key == parenttype or (key in repo_types and type is None):
143 type = key
144 if type is None:
145 raise Exception("No valid source found to create %s" % target)
146
147 url = urlparse.urljoin(roots[type], sourcename)
148 logging.info("Cloning repository %s into %s" % (url, target))
149 repo_types[type].clone(url, target)
150
151 def update_repo(target, revisions):
152 type = get_repo_type(target)
153 if type is None:
154 logging.warning("Type of repository %s unknown, skipping update" % target)
155 return
156
157 if type in revisions:
158 revision = revisions[type]
159 elif "*" in revisions:
160 revision = revisions["*"]
161 else:
162 logging.warning("No revision specified for repository %s (type %s), skipping update" % (target, type))
163 return
164
165 resolved_revision = repo_types[type].get_revision_id(target, revision)
166 if not resolved_revision:
167 logging.info("Revision %s is unknown, downloading remote changes" % revision )
168 repo_types[type].pull(target)
169 resolved_revision = repo_types[type].get_revision_id(target, revision)
170 if not resolved_revision:
171 raise Exception("Failed to resolve revision %s" % revision)
172
173 current_revision = repo_types[type].get_revision_id(target)
174 if resolved_revision != current_revision:
175 logging.info("Updating repository %s to revision %s" % (target, resolved_rev ision))
176 repo_types[type].update(target, resolved_revision)
177
178 def resolve_deps(repodir, level=0):
179 config = read_deps(repodir)
180 if config is None:
181 if level == 0:
182 logging.warning("No dependencies file in directory %s, nothing to do..." % repodir)
183 return
184 if level >= 10:
185 logging.warning("Too much subrepository nesting, ignoring %s" % repo)
186
187 for dir in config:
188 if dir.startswith("_"):
189 continue
190 target = safe_join(repodir, dir)
191 ensure_repo(repodir, target, config.get("_root", {}), config[dir]["_source"] )
192 update_repo(target, config[dir])
193 resolve_deps(target, level + 1)
194
195 if level == 0 and "_self" in config and "*" in config["_self"]:
196 source = safe_join(repodir, config["_self"]["*"])
197 try:
198 with io.open(source, "rb") as handle:
199 sourcedata = handle.read()
200 except IOError, e:
201 if e.errno != errno.ENOENT:
202 raise
203 logging.warning("File %s doesn't exist, skipping self-update" % source)
204 return
205
206 target = __file__
207 with io.open(target, "rb") as handle:
208 targetdata = handle.read()
209
210 if sourcedata != targetdata:
211 logging.info("Updating %s from %s, don't forget to commit" % (source, targ et))
212 with io.open(target, "wb") as handle:
213 handle.write(sourcedata)
214 logging.info("Restarting %s" % target)
215 if __name__ == "__main__":
216 os.execv(sys.executable, [sys.executable, target] + sys.argv[1:])
217 else:
218 import importlib
219 me = importlib.import_module(__name__)
220 reload(me)
221 me.resolve_deps(repodir, level)
222
223 if __name__ == "__main__":
224 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
225 repos = sys.argv[1:]
226 if not len(repos):
227 repos = [os.getcwd()]
228 for repo in repos:
229 resolve_deps(repo)
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld