Index: ensure_dependencies.py |
=================================================================== |
--- a/ensure_dependencies.py |
+++ b/ensure_dependencies.py |
@@ -18,7 +18,7 @@ |
from collections import OrderedDict |
from ConfigParser import RawConfigParser |
-USAGE = """ |
+USAGE = ''' |
A dependencies file should look like this: |
# VCS-specific root URLs for the repositories |
@@ -37,26 +37,26 @@ |
# Clone the adblockpluschrome repository into the adblockpluschrome directory, |
# from a specific Git repository, specifying the revision ID. |
adblockpluschrome = git:git@github.com:user/adblockpluschrome.git@1fad3a7 |
-""" |
+''' |
SKIP_DEPENDENCY_UPDATES = os.environ.get( |
- "SKIP_DEPENDENCY_UPDATES", "" |
-).lower() not in ("", "0", "false") |
+ 'SKIP_DEPENDENCY_UPDATES', '' |
+).lower() not in ('', '0', 'false') |
class Mercurial(): |
def istype(self, repodir): |
- return os.path.exists(os.path.join(repodir, ".hg")) |
+ return os.path.exists(os.path.join(repodir, '.hg')) |
def clone(self, source, target): |
- if not source.endswith("/"): |
- source += "/" |
- subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, target]) |
+ if not source.endswith('/'): |
+ source += '/' |
+ subprocess.check_call(['hg', 'clone', '--quiet', '--noupdate', source, target]) |
def get_revision_id(self, repo, rev=None): |
- command = ["hg", "id", "--repository", repo, "--id"] |
+ command = ['hg', 'id', '--repository', repo, '--id'] |
if rev: |
- command.extend(["--rev", rev]) |
+ command.extend(['--rev', rev]) |
# Ignore stderr output and return code here: if revision lookup failed we |
# should simply return an empty string. |
@@ -64,26 +64,26 @@ |
return result.strip() |
def pull(self, repo): |
- subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"]) |
+ subprocess.check_call(['hg', 'pull', '--repository', repo, '--quiet']) |
def update(self, repo, rev, revname): |
- subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--check", "--rev", rev]) |
+ subprocess.check_call(['hg', 'update', '--repository', repo, '--quiet', '--check', '--rev', rev]) |
def ignore(self, target, repo): |
if not self.istype(target): |
- config_path = os.path.join(repo, ".hg", "hgrc") |
- ignore_path = os.path.abspath(os.path.join(repo, ".hg", "dependencies")) |
+ config_path = os.path.join(repo, '.hg', 'hgrc') |
+ ignore_path = os.path.abspath(os.path.join(repo, '.hg', 'dependencies')) |
config = RawConfigParser() |
config.read(config_path) |
- if not config.has_section("ui"): |
- config.add_section("ui") |
+ if not config.has_section('ui'): |
+ config.add_section('ui') |
- config.set("ui", "ignore.dependencies", ignore_path) |
- with open(config_path, "w") as stream: |
+ config.set('ui', 'ignore.dependencies', ignore_path) |
+ with open(config_path, 'w') as stream: |
config.write(stream) |
module = os.path.relpath(target, repo) |
@@ -95,63 +95,63 @@ |
class Git(): |
def istype(self, repodir): |
- return os.path.exists(os.path.join(repodir, ".git")) |
+ return os.path.exists(os.path.join(repodir, '.git')) |
def clone(self, source, target): |
- source = source.rstrip("/") |
- if not source.endswith(".git"): |
- source += ".git" |
- subprocess.check_call(["git", "clone", "--quiet", source, target]) |
+ source = source.rstrip('/') |
+ if not source.endswith('.git'): |
+ source += '.git' |
+ subprocess.check_call(['git', 'clone', '--quiet', source, target]) |
- def get_revision_id(self, repo, rev="HEAD"): |
- command = ["git", "rev-parse", "--revs-only", rev + '^{commit}'] |
+ def get_revision_id(self, repo, rev='HEAD'): |
+ command = ['git', 'rev-parse', '--revs-only', rev + '^{commit}'] |
return subprocess.check_output(command, cwd=repo).strip() |
def pull(self, repo): |
# Fetch tracked branches, new tags and the list of available remote branches |
- subprocess.check_call(["git", "fetch", "--quiet", "--all", "--tags"], cwd=repo) |
+ subprocess.check_call(['git', 'fetch', '--quiet', '--all', '--tags'], cwd=repo) |
# Next we need to ensure all remote branches are tracked |
newly_tracked = False |
- remotes = subprocess.check_output(["git", "branch", "--remotes"], cwd=repo) |
- for match in re.finditer(r"^\s*(origin/(\S+))$", remotes, re.M): |
+ remotes = subprocess.check_output(['git', 'branch', '--remotes'], cwd=repo) |
+ for match in re.finditer(r'^\s*(origin/(\S+))$', remotes, re.M): |
remote, local = match.groups() |
- with open(os.devnull, "wb") as devnull: |
- if subprocess.call(["git", "branch", "--track", local, remote], |
+ with open(os.devnull, 'wb') as devnull: |
+ if subprocess.call(['git', 'branch', '--track', local, remote], |
cwd=repo, stdout=devnull, stderr=devnull) == 0: |
newly_tracked = True |
# Finally fetch any newly tracked remote branches |
if newly_tracked: |
- subprocess.check_call(["git", "fetch", "--quiet", "origin"], cwd=repo) |
+ subprocess.check_call(['git', 'fetch', '--quiet', 'origin'], cwd=repo) |
def update(self, repo, rev, revname): |
- subprocess.check_call(["git", "checkout", "--quiet", revname], cwd=repo) |
+ subprocess.check_call(['git', 'checkout', '--quiet', revname], cwd=repo) |
def ignore(self, target, repo): |
module = os.path.sep + os.path.relpath(target, repo) |
- exclude_file = os.path.join(repo, ".git", "info", "exclude") |
+ exclude_file = os.path.join(repo, '.git', 'info', 'exclude') |
_ensure_line_exists(exclude_file, module) |
def postprocess_url(self, url): |
# Handle alternative syntax of SSH URLS |
- if "@" in url and ":" in url and not urlparse.urlsplit(url).scheme: |
- return "ssh://" + url.replace(":", "/", 1) |
+ if '@' in url and ':' in url and not urlparse.urlsplit(url).scheme: |
+ return 'ssh://' + url.replace(':', '/', 1) |
return url |
repo_types = OrderedDict(( |
- ("hg", Mercurial()), |
- ("git", Git()), |
+ ('hg', Mercurial()), |
+ ('git', Git()), |
)) |
# [vcs:]value |
item_regexp = re.compile( |
- "^(?:(" + "|".join(map(re.escape, repo_types.keys())) + "):)?" |
- "(.+)$" |
+ '^(?:(' + '|'.join(map(re.escape, repo_types.keys())) + '):)?' |
+ '(.+)$' |
) |
# [url@]rev |
source_regexp = re.compile( |
- "^(?:(.*)@)?" |
- "(.+)$" |
+ '^(?:(.*)@)?' |
+ '(.+)$' |
) |
@@ -166,26 +166,26 @@ |
def parse_spec(path, line): |
- if "=" not in line: |
- logging.warning("Invalid line in file %s: %s" % (path, line)) |
+ if '=' not in line: |
+ logging.warning('Invalid line in file %s: %s' % (path, line)) |
return None, None |
- key, value = line.split("=", 1) |
+ key, value = line.split('=', 1) |
key = key.strip() |
items = value.split() |
if not len(items): |
- logging.warning("No value specified for key %s in file %s" % (key, path)) |
+ logging.warning('No value specified for key %s in file %s' % (key, path)) |
return key, None |
result = OrderedDict() |
- is_dependency_field = not key.startswith("_") |
+ is_dependency_field = not key.startswith('_') |
for i, item in enumerate(items): |
try: |
vcs, value = re.search(item_regexp, item).groups() |
- vcs = vcs or "*" |
+ vcs = vcs or '*' |
if is_dependency_field: |
- if i == 0 and vcs == "*": |
+ if i == 0 and vcs == '*': |
# In order to be backwards compatible we have to assume that the first |
# source contains only a URL/path for the repo if it does not contain |
# the VCS part |
@@ -195,24 +195,24 @@ |
result[vcs] = merge_seqs(result.get(vcs), url_rev) |
else: |
if vcs in result: |
- logging.warning("Ignoring duplicate value for type %r " |
- "(key %r in file %r)" % (vcs, key, path)) |
+ logging.warning('Ignoring duplicate value for type %r ' |
+ '(key %r in file %r)' % (vcs, key, path)) |
result[vcs] = value |
except AttributeError: |
- logging.warning("Ignoring invalid item %r for type %r " |
- "(key %r in file %r)" % (item, vcs, key, path)) |
+ logging.warning('Ignoring invalid item %r for type %r ' |
+ '(key %r in file %r)' % (item, vcs, key, path)) |
continue |
return key, result |
def read_deps(repodir): |
result = {} |
- deps_path = os.path.join(repodir, "dependencies") |
+ deps_path = os.path.join(repodir, 'dependencies') |
try: |
- with io.open(deps_path, "rt", encoding="utf-8") as handle: |
+ with io.open(deps_path, 'rt', encoding='utf-8') as handle: |
for line in handle: |
# Remove comments and whitespace |
- line = re.sub(r"#.*", "", line).strip() |
+ line = re.sub(r'#.*', '', line).strip() |
if not line: |
continue |
@@ -230,13 +230,13 @@ |
# This has been inspired by Flask's safe_join() function |
forbidden = {os.sep, os.altsep} - {posixpath.sep, None} |
if any(sep in subpath for sep in forbidden): |
- raise Exception("Illegal directory separator in dependency path %s" % subpath) |
+ raise Exception('Illegal directory separator in dependency path %s' % subpath) |
normpath = posixpath.normpath(subpath) |
if posixpath.isabs(normpath): |
- raise Exception("Dependency path %s cannot be absolute" % subpath) |
+ raise Exception('Dependency path %s cannot be absolute' % subpath) |
if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posixpath.sep): |
- raise Exception("Dependency path %s has to be inside the repository" % subpath) |
+ raise Exception('Dependency path %s has to be inside the repository' % subpath) |
return os.path.join(path, *normpath.split(posixpath.sep)) |
@@ -244,7 +244,7 @@ |
for name, repotype in repo_types.iteritems(): |
if repotype.istype(repo): |
return name |
- return "hg" |
+ return 'hg' |
def ensure_repo(parentrepo, parenttype, target, type, root, sourcename): |
@@ -252,8 +252,8 @@ |
return |
if SKIP_DEPENDENCY_UPDATES: |
- logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, " |
- "%s not cloned", target) |
+ logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, ' |
+ '%s not cloned', target) |
return |
postprocess_url = repo_types[type].postprocess_url |
@@ -265,7 +265,7 @@ |
else: |
url = urlparse.urljoin(root, sourcename) |
- logging.info("Cloning repository %s into %s" % (url, target)) |
+ logging.info('Cloning repository %s into %s' % (url, target)) |
repo_types[type].clone(url, target) |
repo_types[parenttype].ignore(target, parentrepo) |
@@ -276,18 +276,18 @@ |
if resolved_revision != current_revision: |
if SKIP_DEPENDENCY_UPDATES: |
- logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, " |
- "%s not checked out to %s", target, revision) |
+ logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, ' |
+ '%s not checked out to %s', target, revision) |
return |
if not resolved_revision: |
- logging.info("Revision %s is unknown, downloading remote changes" % revision) |
+ logging.info('Revision %s is unknown, downloading remote changes' % revision) |
repo_types[type].pull(target) |
resolved_revision = repo_types[type].get_revision_id(target, revision) |
if not resolved_revision: |
- raise Exception("Failed to resolve revision %s" % revision) |
+ raise Exception('Failed to resolve revision %s' % revision) |
- logging.info("Updating repository %s to revision %s" % (target, resolved_revision)) |
+ logging.info('Updating repository %s to revision %s' % (target, resolved_revision)) |
repo_types[type].update(target, resolved_revision, revision) |
@@ -295,42 +295,42 @@ |
config = read_deps(repodir) |
if config is None: |
if level == 0: |
- logging.warning("No dependencies file in directory %s, nothing to do...\n%s" % (repodir, USAGE)) |
+ logging.warning('No dependencies file in directory %s, nothing to do...\n%s' % (repodir, USAGE)) |
return |
if level >= 10: |
- logging.warning("Too much subrepository nesting, ignoring %s" % repo) |
+ logging.warning('Too much subrepository nesting, ignoring %s' % repo) |
return |
if overrideroots is not None: |
- config["_root"] = overrideroots |
+ config['_root'] = overrideroots |
for dir, sources in config.iteritems(): |
- if (dir.startswith("_") or |
+ if (dir.startswith('_') or |
skipdependencies.intersection([s[0] for s in sources if s[0]])): |
continue |
target = safe_join(repodir, dir) |
parenttype = get_repo_type(repodir) |
- _root = config.get("_root", {}) |
+ _root = config.get('_root', {}) |
for key in sources.keys() + _root.keys(): |
- if key == parenttype or key is None and vcs != "*": |
+ if key == parenttype or key is None and vcs != '*': |
vcs = key |
- source, rev = merge_seqs(sources.get("*"), sources.get(vcs)) |
+ source, rev = merge_seqs(sources.get('*'), sources.get(vcs)) |
if not (vcs and source and rev): |
- logging.warning("No valid source / revision found to create %s" % target) |
+ logging.warning('No valid source / revision found to create %s' % target) |
continue |
- ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ""), source) |
+ ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ''), source) |
update_repo(target, vcs, rev) |
resolve_deps(target, level + 1, self_update=False, |
overrideroots=overrideroots, skipdependencies=skipdependencies) |
- if self_update and "_self" in config and "*" in config["_self"]: |
- source = safe_join(repodir, config["_self"]["*"]) |
+ if self_update and '_self' in config and '*' in config['_self']: |
+ source = safe_join(repodir, config['_self']['*']) |
try: |
- with io.open(source, "rb") as handle: |
+ with io.open(source, 'rb') as handle: |
sourcedata = handle.read() |
except IOError, e: |
if e.errno != errno.ENOENT: |
@@ -339,18 +339,18 @@ |
return |
target = __file__ |
- with io.open(target, "rb") as handle: |
+ with io.open(target, 'rb') as handle: |
targetdata = handle.read() |
if sourcedata != targetdata: |
logging.info("Updating %s from %s, don't forget to commit" % (target, source)) |
- with io.open(target, "wb") as handle: |
+ with io.open(target, 'wb') as handle: |
handle.write(sourcedata) |
- if __name__ == "__main__": |
- logging.info("Restarting %s" % target) |
+ if __name__ == '__main__': |
+ logging.info('Restarting %s' % target) |
os.execv(sys.executable, [sys.executable, target] + sys.argv[1:]) |
else: |
- logging.warning("Cannot restart %s automatically, please rerun" % target) |
+ logging.warning('Cannot restart %s automatically, please rerun' % target) |
def _ensure_line_exists(path, pattern): |
@@ -363,12 +363,12 @@ |
for l in file_content: |
print >>f, l |
-if __name__ == "__main__": |
+if __name__ == '__main__': |
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) |
- parser = argparse.ArgumentParser(description="Verify dependencies for a set of repositories, by default the repository of this script.") |
- parser.add_argument("repos", metavar="repository", type=str, nargs="*", help="Repository path") |
- parser.add_argument("-q", "--quiet", action="store_true", help="Suppress informational output") |
+ parser = argparse.ArgumentParser(description='Verify dependencies for a set of repositories, by default the repository of this script.') |
+ parser.add_argument('repos', metavar='repository', type=str, nargs='*', help='Repository path') |
+ parser.add_argument('-q', '--quiet', action='store_true', help='Suppress informational output') |
args = parser.parse_args() |
if args.quiet: |