Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Unified Diff: ensure_dependencies.py

Issue 29802595: Issue 6664 - Update the buildtools dependency for adblockpluscore
Patch Set: Created June 8, 2018, 4:56 p.m.
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« dependencies ('K') | « dependencies ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: ensure_dependencies.py
===================================================================
--- a/ensure_dependencies.py
+++ b/ensure_dependencies.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python
-# coding: utf-8
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -15,11 +14,12 @@ import logging
import subprocess
import urlparse
import argparse
+import json
from collections import OrderedDict
from ConfigParser import RawConfigParser
-USAGE = """
+USAGE = '''
A dependencies file should look like this:
# VCS-specific root URLs for the repositories
@@ -38,26 +38,28 @@ A dependencies file should look like thi
# Clone the adblockpluschrome repository into the adblockpluschrome directory,
# from a specific Git repository, specifying the revision ID.
adblockpluschrome = git:git@github.com:user/adblockpluschrome.git@1fad3a7
-"""
+'''
SKIP_DEPENDENCY_UPDATES = os.environ.get(
- "SKIP_DEPENDENCY_UPDATES", ""
-).lower() not in ("", "0", "false")
+ 'SKIP_DEPENDENCY_UPDATES', ''
+).lower() not in ('', '0', 'false')
+
+NPM_LOCKFILE = '.npm_install_lock'
-class Mercurial():
+class Mercurial:
def istype(self, repodir):
- return os.path.exists(os.path.join(repodir, ".hg"))
+ return os.path.exists(os.path.join(repodir, '.hg'))
def clone(self, source, target):
- if not source.endswith("/"):
- source += "/"
- subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, target])
+ if not source.endswith('/'):
+ source += '/'
+ subprocess.check_call(['hg', 'clone', '--quiet', '--noupdate', source, target])
def get_revision_id(self, repo, rev=None):
- command = ["hg", "id", "--repository", repo, "--id"]
+ command = ['hg', 'id', '--repository', repo, '--id']
if rev:
- command.extend(["--rev", rev])
+ command.extend(['--rev', rev])
# Ignore stderr output and return code here: if revision lookup failed we
# should simply return an empty string.
@@ -65,94 +67,93 @@ class Mercurial():
return result.strip()
def pull(self, repo):
- subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"])
+ subprocess.check_call(['hg', 'pull', '--repository', repo, '--quiet'])
def update(self, repo, rev, revname):
- subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--check", "--rev", rev])
+ subprocess.check_call(['hg', 'update', '--repository', repo, '--quiet', '--check', '--rev', rev])
def ignore(self, target, repo):
-
- if not self.istype(target):
+ config_path = os.path.join(repo, '.hg', 'hgrc')
+ ignore_file = os.path.join('.hg', 'dependencies')
+ ignore_path = os.path.join(repo, ignore_file)
- config_path = os.path.join(repo, ".hg", "hgrc")
- ignore_path = os.path.abspath(os.path.join(repo, ".hg", "dependencies"))
-
- config = RawConfigParser()
- config.read(config_path)
+ config = RawConfigParser()
+ config.read(config_path)
- if not config.has_section("ui"):
- config.add_section("ui")
+ if not config.has_section('ui'):
+ config.add_section('ui')
- config.set("ui", "ignore.dependencies", ignore_path)
- with open(config_path, "w") as stream:
- config.write(stream)
+ config.set('ui', 'ignore.dependencies', ignore_file)
+ with open(config_path, 'w') as stream:
+ config.write(stream)
- module = os.path.relpath(target, repo)
- _ensure_line_exists(ignore_path, module)
+ module = os.path.relpath(target, repo)
+ _ensure_line_exists(ignore_path, module)
def postprocess_url(self, url):
return url
-class Git():
+class Git:
def istype(self, repodir):
- return os.path.exists(os.path.join(repodir, ".git"))
+ return os.path.exists(os.path.join(repodir, '.git'))
def clone(self, source, target):
- source = source.rstrip("/")
- if not source.endswith(".git"):
- source += ".git"
- subprocess.check_call(["git", "clone", "--quiet", source, target])
+ source = source.rstrip('/')
+ if not source.endswith('.git'):
+ source += '.git'
+ subprocess.check_call(['git', 'clone', '--quiet', source, target])
- def get_revision_id(self, repo, rev="HEAD"):
- command = ["git", "rev-parse", "--revs-only", rev + '^{commit}']
+ def get_revision_id(self, repo, rev='HEAD'):
+ command = ['git', 'rev-parse', '--revs-only', rev + '^{commit}']
return subprocess.check_output(command, cwd=repo).strip()
def pull(self, repo):
# Fetch tracked branches, new tags and the list of available remote branches
- subprocess.check_call(["git", "fetch", "--quiet", "--all", "--tags"], cwd=repo)
+ subprocess.check_call(['git', 'fetch', '--quiet', '--all', '--tags'], cwd=repo)
# Next we need to ensure all remote branches are tracked
newly_tracked = False
- remotes = subprocess.check_output(["git", "branch", "--remotes"], cwd=repo)
- for match in re.finditer(r"^\s*(origin/(\S+))$", remotes, re.M):
+ remotes = subprocess.check_output(['git', 'branch', '--remotes'], cwd=repo)
+ for match in re.finditer(r'^\s*(origin/(\S+))$', remotes, re.M):
remote, local = match.groups()
- with open(os.devnull, "wb") as devnull:
- if subprocess.call(["git", "branch", "--track", local, remote],
+ with open(os.devnull, 'wb') as devnull:
+ if subprocess.call(['git', 'branch', '--track', local, remote],
cwd=repo, stdout=devnull, stderr=devnull) == 0:
newly_tracked = True
# Finally fetch any newly tracked remote branches
if newly_tracked:
- subprocess.check_call(["git", "fetch", "--quiet", "origin"], cwd=repo)
+ subprocess.check_call(['git', 'fetch', '--quiet', 'origin'], cwd=repo)
def update(self, repo, rev, revname):
- subprocess.check_call(["git", "checkout", "--quiet", revname], cwd=repo)
+ subprocess.check_call(['git', 'checkout', '--quiet', revname], cwd=repo)
def ignore(self, target, repo):
module = os.path.sep + os.path.relpath(target, repo)
- exclude_file = os.path.join(repo, ".git", "info", "exclude")
+ exclude_file = os.path.join(repo, '.git', 'info', 'exclude')
_ensure_line_exists(exclude_file, module)
def postprocess_url(self, url):
# Handle alternative syntax of SSH URLS
- if "@" in url and ":" in url and not urlparse.urlsplit(url).scheme:
- return "ssh://" + url.replace(":", "/", 1)
+ if '@' in url and ':' in url and not urlparse.urlsplit(url).scheme:
+ return 'ssh://' + url.replace(':', '/', 1)
return url
+
repo_types = OrderedDict((
- ("hg", Mercurial()),
- ("git", Git()),
+ ('hg', Mercurial()),
+ ('git', Git()),
))
# [vcs:]value
item_regexp = re.compile(
- "^(?:(" + "|".join(map(re.escape, repo_types.keys())) + "):)?"
- "(.+)$"
+ '^(?:(' + '|'.join(map(re.escape, repo_types.keys())) + '):)?'
+ '(.+)$'
)
# [url@]rev
source_regexp = re.compile(
- "^(?:(.*)@)?"
- "(.+)$"
+ '^(?:(.*)@)?'
+ '(.+)$'
)
@@ -167,26 +168,26 @@ def merge_seqs(seq1, seq2):
def parse_spec(path, line):
- if "=" not in line:
- logging.warning("Invalid line in file %s: %s" % (path, line))
+ if '=' not in line:
+ logging.warning('Invalid line in file %s: %s' % (path, line))
return None, None
- key, value = line.split("=", 1)
+ key, value = line.split('=', 1)
key = key.strip()
items = value.split()
if not len(items):
- logging.warning("No value specified for key %s in file %s" % (key, path))
+ logging.warning('No value specified for key %s in file %s' % (key, path))
return key, None
result = OrderedDict()
- is_dependency_field = not key.startswith("_")
+ is_dependency_field = not key.startswith('_')
for i, item in enumerate(items):
try:
vcs, value = re.search(item_regexp, item).groups()
- vcs = vcs or "*"
+ vcs = vcs or '*'
if is_dependency_field:
- if i == 0 and vcs == "*":
+ if i == 0 and vcs == '*':
# In order to be backwards compatible we have to assume that the first
# source contains only a URL/path for the repo if it does not contain
# the VCS part
@@ -196,24 +197,24 @@ def parse_spec(path, line):
result[vcs] = merge_seqs(result.get(vcs), url_rev)
else:
if vcs in result:
- logging.warning("Ignoring duplicate value for type %r "
- "(key %r in file %r)" % (vcs, key, path))
+ logging.warning('Ignoring duplicate value for type %r '
+ '(key %r in file %r)' % (vcs, key, path))
result[vcs] = value
except AttributeError:
- logging.warning("Ignoring invalid item %r for type %r "
- "(key %r in file %r)" % (item, vcs, key, path))
+ logging.warning('Ignoring invalid item %r for type %r '
+ '(key %r in file %r)' % (item, vcs, key, path))
continue
return key, result
def read_deps(repodir):
result = {}
- deps_path = os.path.join(repodir, "dependencies")
+ deps_path = os.path.join(repodir, 'dependencies')
try:
- with io.open(deps_path, "rt", encoding="utf-8") as handle:
+ with io.open(deps_path, 'rt', encoding='utf-8') as handle:
for line in handle:
# Remove comments and whitespace
- line = re.sub(r"#.*", "", line).strip()
+ line = re.sub(r'#.*', '', line).strip()
if not line:
continue
@@ -221,7 +222,7 @@ def read_deps(repodir):
if spec:
result[key] = spec
return result
- except IOError, e:
+ except IOError as e:
if e.errno != errno.ENOENT:
raise
return None
@@ -231,13 +232,13 @@ def safe_join(path, subpath):
# This has been inspired by Flask's safe_join() function
forbidden = {os.sep, os.altsep} - {posixpath.sep, None}
if any(sep in subpath for sep in forbidden):
- raise Exception("Illegal directory separator in dependency path %s" % subpath)
+ raise Exception('Illegal directory separator in dependency path %s' % subpath)
normpath = posixpath.normpath(subpath)
if posixpath.isabs(normpath):
- raise Exception("Dependency path %s cannot be absolute" % subpath)
+ raise Exception('Dependency path %s cannot be absolute' % subpath)
if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posixpath.sep):
- raise Exception("Dependency path %s has to be inside the repository" % subpath)
+ raise Exception('Dependency path %s has to be inside the repository' % subpath)
return os.path.join(path, *normpath.split(posixpath.sep))
@@ -245,17 +246,72 @@ def get_repo_type(repo):
for name, repotype in repo_types.iteritems():
if repotype.istype(repo):
return name
- return "hg"
+ return 'hg'
+
+
+def resolve_npm_dependencies(target, vcs):
+ """Install Node.js production-only dependencies if necessary and desired.
+
+ When the target dependency has additional Node.js dependencies declared
+ run "npm install --only=production --loglevel=warn" to resolve the declared
+ dependencies.
+
+ Additionally, make sure that any VCS will ignore the installed files.
+
+ Requires Node.js to be installed locally.
+ """
+ try:
+ with open(os.path.join(target, 'package.json'), 'r') as fp:
+ package_data = json.load(fp)
+
+ # In case a package.json does not exist at all or if there are no
+ # production dependencies declared, we don't need to run npm and can
+ # bail out early.
+ if not package_data.get('dependencies', False):
+ return
+ except IOError:
+ return
+
+ try:
+ # Create an empty file, which gets deleted after successfully
+ # installing Node.js dependencies.
+ lockfile_path = os.path.join(target, NPM_LOCKFILE)
+ open(lockfile_path, 'a').close()
+
+ if os.name == 'nt':
+ # Windows' CreateProcess() (called by subprocess.Popen()) only
+ # resolves executables ending in .exe. The windows installation of
+ # Node.js only provides a npm.cmd, which is executable but won't
+ # be recognized as such by CreateProcess().
+ npm_exec = 'npm.cmd'
+ else:
+ npm_exec = 'npm'
+
+ cmd = [npm_exec, 'install', '--only=production', '--loglevel=warn',
+ '--no-package-lock', '--no-optional']
+ subprocess.check_output(cmd, cwd=target)
+
+ repo_types[vcs].ignore(os.path.join(target, NPM_LOCKFILE), target)
+ repo_types[vcs].ignore(os.path.join(target, 'node_modules'), target)
+
+ os.remove(lockfile_path)
+ except OSError as e:
+ import errno
+ if e.errno == errno.ENOENT:
+ logging.error('Failed to install Node.js dependencies for %s,'
+ ' please ensure Node.js is installed.', target)
+ else:
+ raise
def ensure_repo(parentrepo, parenttype, target, type, root, sourcename):
if os.path.exists(target):
- return
+ return False
if SKIP_DEPENDENCY_UPDATES:
- logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, "
- "%s not cloned", target)
- return
+ logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, '
+ '%s not cloned', target)
+ return False
postprocess_url = repo_types[type].postprocess_url
root = postprocess_url(root)
@@ -266,9 +322,10 @@ def ensure_repo(parentrepo, parenttype,
else:
url = urlparse.urljoin(root, sourcename)
- logging.info("Cloning repository %s into %s" % (url, target))
+ logging.info('Cloning repository %s into %s' % (url, target))
repo_types[type].clone(url, target)
repo_types[parenttype].ignore(target, parentrepo)
+ return True
def update_repo(target, type, revision):
@@ -277,85 +334,92 @@ def update_repo(target, type, revision):
if resolved_revision != current_revision:
if SKIP_DEPENDENCY_UPDATES:
- logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, "
- "%s not checked out to %s", target, revision)
- return
+ logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, '
+ '%s not checked out to %s', target, revision)
+ return False
if not resolved_revision:
- logging.info("Revision %s is unknown, downloading remote changes" % revision)
+ logging.info('Revision %s is unknown, downloading remote changes' % revision)
repo_types[type].pull(target)
resolved_revision = repo_types[type].get_revision_id(target, revision)
if not resolved_revision:
- raise Exception("Failed to resolve revision %s" % revision)
+ raise Exception('Failed to resolve revision %s' % revision)
- logging.info("Updating repository %s to revision %s" % (target, resolved_revision))
+ logging.info('Updating repository %s to revision %s' % (target, resolved_revision))
repo_types[type].update(target, resolved_revision, revision)
+ return True
+ return False
def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdependencies=set()):
config = read_deps(repodir)
if config is None:
if level == 0:
- logging.warning("No dependencies file in directory %s, nothing to do...\n%s" % (repodir, USAGE))
+ logging.warning('No dependencies file in directory %s, nothing to do...\n%s' % (repodir, USAGE))
return
if level >= 10:
- logging.warning("Too much subrepository nesting, ignoring %s" % repo)
+ logging.warning('Too much subrepository nesting, ignoring %s' % repo)
return
if overrideroots is not None:
- config["_root"] = overrideroots
+ config['_root'] = overrideroots
- for dir, sources in config.iteritems():
- if (dir.startswith("_") or
+ for dir, sources in sorted(config.iteritems()):
+ if (dir.startswith('_') or
skipdependencies.intersection([s[0] for s in sources if s[0]])):
continue
target = safe_join(repodir, dir)
parenttype = get_repo_type(repodir)
- _root = config.get("_root", {})
+ _root = config.get('_root', {})
for key in sources.keys() + _root.keys():
- if key == parenttype or key is None and vcs != "*":
+ if key == parenttype or key is None and vcs != '*':
vcs = key
- source, rev = merge_seqs(sources.get("*"), sources.get(vcs))
+ source, rev = merge_seqs(sources.get('*'), sources.get(vcs))
if not (vcs and source and rev):
- logging.warning("No valid source / revision found to create %s" % target)
+ logging.warning('No valid source / revision found to create %s' % target)
continue
- ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ""), source)
- update_repo(target, vcs, rev)
+ repo_cloned = ensure_repo(repodir, parenttype, target, vcs,
+ _root.get(vcs, ''), source)
+ repo_updated = update_repo(target, vcs, rev)
+ recent_npm_failed = os.path.exists(os.path.join(target, NPM_LOCKFILE))
+ if repo_cloned or repo_updated or recent_npm_failed:
+ resolve_npm_dependencies(target, vcs)
resolve_deps(target, level + 1, self_update=False,
overrideroots=overrideroots, skipdependencies=skipdependencies)
- if self_update and "_self" in config and "*" in config["_self"]:
- source = safe_join(repodir, config["_self"]["*"])
+ if self_update and '_self' in config and '*' in config['_self']:
+ source = safe_join(repodir, config['_self']['*'])
try:
- with io.open(source, "rb") as handle:
+ with io.open(source, 'rb') as handle:
sourcedata = handle.read()
- except IOError, e:
+ except IOError as e:
if e.errno != errno.ENOENT:
raise
logging.warning("File %s doesn't exist, skipping self-update" % source)
return
target = __file__
- with io.open(target, "rb") as handle:
+ with io.open(target, 'rb') as handle:
targetdata = handle.read()
if sourcedata != targetdata:
logging.info("Updating %s from %s, don't forget to commit" % (target, source))
- with io.open(target, "wb") as handle:
+ with io.open(target, 'wb') as handle:
handle.write(sourcedata)
- if __name__ == "__main__":
- logging.info("Restarting %s" % target)
+ if __name__ == '__main__':
+ logging.info('Restarting %s' % target)
os.execv(sys.executable, [sys.executable, target] + sys.argv[1:])
else:
- logging.warning("Cannot restart %s automatically, please rerun" % target)
+ logging.warning('Cannot restart %s automatically, please rerun' % target)
def _ensure_line_exists(path, pattern):
with open(path, 'a+') as f:
+ f.seek(0, os.SEEK_SET)
file_content = [l.strip() for l in f.readlines()]
if not pattern in file_content:
file_content.append(pattern)
@@ -364,12 +428,13 @@ def _ensure_line_exists(path, pattern):
for l in file_content:
print >>f, l
-if __name__ == "__main__":
+
+if __name__ == '__main__':
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
- parser = argparse.ArgumentParser(description="Verify dependencies for a set of repositories, by default the repository of this script.")
- parser.add_argument("repos", metavar="repository", type=str, nargs="*", help="Repository path")
- parser.add_argument("-q", "--quiet", action="store_true", help="Suppress informational output")
+ parser = argparse.ArgumentParser(description='Verify dependencies for a set of repositories, by default the repository of this script.')
+ parser.add_argument('repos', metavar='repository', type=str, nargs='*', help='Repository path')
+ parser.add_argument('-q', '--quiet', action='store_true', help='Suppress informational output')
args = parser.parse_args()
if args.quiet:
« dependencies ('K') | « dependencies ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld