Index: ensure_dependencies.py =================================================================== --- a/ensure_dependencies.py +++ b/ensure_dependencies.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -15,11 +14,12 @@ import logging import subprocess import urlparse import argparse +import json from collections import OrderedDict from ConfigParser import RawConfigParser -USAGE = """ +USAGE = ''' A dependencies file should look like this: # VCS-specific root URLs for the repositories @@ -38,26 +38,28 @@ A dependencies file should look like thi # Clone the adblockpluschrome repository into the adblockpluschrome directory, # from a specific Git repository, specifying the revision ID. adblockpluschrome = git:git@github.com:user/adblockpluschrome.git@1fad3a7 -""" +''' SKIP_DEPENDENCY_UPDATES = os.environ.get( - "SKIP_DEPENDENCY_UPDATES", "" -).lower() not in ("", "0", "false") + 'SKIP_DEPENDENCY_UPDATES', '' +).lower() not in ('', '0', 'false') + +NPM_LOCKFILE = '.npm_install_lock' -class Mercurial(): +class Mercurial: def istype(self, repodir): - return os.path.exists(os.path.join(repodir, ".hg")) + return os.path.exists(os.path.join(repodir, '.hg')) def clone(self, source, target): - if not source.endswith("/"): - source += "/" - subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, target]) + if not source.endswith('/'): + source += '/' + subprocess.check_call(['hg', 'clone', '--quiet', '--noupdate', source, target]) def get_revision_id(self, repo, rev=None): - command = ["hg", "id", "--repository", repo, "--id"] + command = ['hg', 'id', '--repository', repo, '--id'] if rev: - command.extend(["--rev", rev]) + command.extend(['--rev', rev]) # Ignore stderr output and return code here: if revision lookup failed we # should simply return an empty string. @@ -65,94 +67,93 @@ class Mercurial(): return result.strip() def pull(self, repo): - subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"]) + subprocess.check_call(['hg', 'pull', '--repository', repo, '--quiet']) def update(self, repo, rev, revname): - subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--check", "--rev", rev]) + subprocess.check_call(['hg', 'update', '--repository', repo, '--quiet', '--check', '--rev', rev]) def ignore(self, target, repo): - - if not self.istype(target): + config_path = os.path.join(repo, '.hg', 'hgrc') + ignore_file = os.path.join('.hg', 'dependencies') + ignore_path = os.path.join(repo, ignore_file) - config_path = os.path.join(repo, ".hg", "hgrc") - ignore_path = os.path.abspath(os.path.join(repo, ".hg", "dependencies")) - - config = RawConfigParser() - config.read(config_path) + config = RawConfigParser() + config.read(config_path) - if not config.has_section("ui"): - config.add_section("ui") + if not config.has_section('ui'): + config.add_section('ui') - config.set("ui", "ignore.dependencies", ignore_path) - with open(config_path, "w") as stream: - config.write(stream) + config.set('ui', 'ignore.dependencies', ignore_file) + with open(config_path, 'w') as stream: + config.write(stream) - module = os.path.relpath(target, repo) - _ensure_line_exists(ignore_path, module) + module = os.path.relpath(target, repo) + _ensure_line_exists(ignore_path, module) def postprocess_url(self, url): return url -class Git(): +class Git: def istype(self, repodir): - return os.path.exists(os.path.join(repodir, ".git")) + return os.path.exists(os.path.join(repodir, '.git')) def clone(self, source, target): - source = source.rstrip("/") - if not source.endswith(".git"): - source += ".git" - subprocess.check_call(["git", "clone", "--quiet", source, target]) + source = source.rstrip('/') + if not source.endswith('.git'): + source += '.git' + subprocess.check_call(['git', 'clone', '--quiet', source, target]) - def get_revision_id(self, repo, rev="HEAD"): - command = ["git", "rev-parse", "--revs-only", rev + '^{commit}'] + def get_revision_id(self, repo, rev='HEAD'): + command = ['git', 'rev-parse', '--revs-only', rev + '^{commit}'] return subprocess.check_output(command, cwd=repo).strip() def pull(self, repo): # Fetch tracked branches, new tags and the list of available remote branches - subprocess.check_call(["git", "fetch", "--quiet", "--all", "--tags"], cwd=repo) + subprocess.check_call(['git', 'fetch', '--quiet', '--all', '--tags'], cwd=repo) # Next we need to ensure all remote branches are tracked newly_tracked = False - remotes = subprocess.check_output(["git", "branch", "--remotes"], cwd=repo) - for match in re.finditer(r"^\s*(origin/(\S+))$", remotes, re.M): + remotes = subprocess.check_output(['git', 'branch', '--remotes'], cwd=repo) + for match in re.finditer(r'^\s*(origin/(\S+))$', remotes, re.M): remote, local = match.groups() - with open(os.devnull, "wb") as devnull: - if subprocess.call(["git", "branch", "--track", local, remote], + with open(os.devnull, 'wb') as devnull: + if subprocess.call(['git', 'branch', '--track', local, remote], cwd=repo, stdout=devnull, stderr=devnull) == 0: newly_tracked = True # Finally fetch any newly tracked remote branches if newly_tracked: - subprocess.check_call(["git", "fetch", "--quiet", "origin"], cwd=repo) + subprocess.check_call(['git', 'fetch', '--quiet', 'origin'], cwd=repo) def update(self, repo, rev, revname): - subprocess.check_call(["git", "checkout", "--quiet", revname], cwd=repo) + subprocess.check_call(['git', 'checkout', '--quiet', revname], cwd=repo) def ignore(self, target, repo): module = os.path.sep + os.path.relpath(target, repo) - exclude_file = os.path.join(repo, ".git", "info", "exclude") + exclude_file = os.path.join(repo, '.git', 'info', 'exclude') _ensure_line_exists(exclude_file, module) def postprocess_url(self, url): # Handle alternative syntax of SSH URLS - if "@" in url and ":" in url and not urlparse.urlsplit(url).scheme: - return "ssh://" + url.replace(":", "/", 1) + if '@' in url and ':' in url and not urlparse.urlsplit(url).scheme: + return 'ssh://' + url.replace(':', '/', 1) return url + repo_types = OrderedDict(( - ("hg", Mercurial()), - ("git", Git()), + ('hg', Mercurial()), + ('git', Git()), )) # [vcs:]value item_regexp = re.compile( - "^(?:(" + "|".join(map(re.escape, repo_types.keys())) + "):)?" - "(.+)$" + '^(?:(' + '|'.join(map(re.escape, repo_types.keys())) + '):)?' + '(.+)$' ) # [url@]rev source_regexp = re.compile( - "^(?:(.*)@)?" - "(.+)$" + '^(?:(.*)@)?' + '(.+)$' ) @@ -167,26 +168,26 @@ def merge_seqs(seq1, seq2): def parse_spec(path, line): - if "=" not in line: - logging.warning("Invalid line in file %s: %s" % (path, line)) + if '=' not in line: + logging.warning('Invalid line in file %s: %s' % (path, line)) return None, None - key, value = line.split("=", 1) + key, value = line.split('=', 1) key = key.strip() items = value.split() if not len(items): - logging.warning("No value specified for key %s in file %s" % (key, path)) + logging.warning('No value specified for key %s in file %s' % (key, path)) return key, None result = OrderedDict() - is_dependency_field = not key.startswith("_") + is_dependency_field = not key.startswith('_') for i, item in enumerate(items): try: vcs, value = re.search(item_regexp, item).groups() - vcs = vcs or "*" + vcs = vcs or '*' if is_dependency_field: - if i == 0 and vcs == "*": + if i == 0 and vcs == '*': # In order to be backwards compatible we have to assume that the first # source contains only a URL/path for the repo if it does not contain # the VCS part @@ -196,24 +197,24 @@ def parse_spec(path, line): result[vcs] = merge_seqs(result.get(vcs), url_rev) else: if vcs in result: - logging.warning("Ignoring duplicate value for type %r " - "(key %r in file %r)" % (vcs, key, path)) + logging.warning('Ignoring duplicate value for type %r ' + '(key %r in file %r)' % (vcs, key, path)) result[vcs] = value except AttributeError: - logging.warning("Ignoring invalid item %r for type %r " - "(key %r in file %r)" % (item, vcs, key, path)) + logging.warning('Ignoring invalid item %r for type %r ' + '(key %r in file %r)' % (item, vcs, key, path)) continue return key, result def read_deps(repodir): result = {} - deps_path = os.path.join(repodir, "dependencies") + deps_path = os.path.join(repodir, 'dependencies') try: - with io.open(deps_path, "rt", encoding="utf-8") as handle: + with io.open(deps_path, 'rt', encoding='utf-8') as handle: for line in handle: # Remove comments and whitespace - line = re.sub(r"#.*", "", line).strip() + line = re.sub(r'#.*', '', line).strip() if not line: continue @@ -221,7 +222,7 @@ def read_deps(repodir): if spec: result[key] = spec return result - except IOError, e: + except IOError as e: if e.errno != errno.ENOENT: raise return None @@ -231,13 +232,13 @@ def safe_join(path, subpath): # This has been inspired by Flask's safe_join() function forbidden = {os.sep, os.altsep} - {posixpath.sep, None} if any(sep in subpath for sep in forbidden): - raise Exception("Illegal directory separator in dependency path %s" % subpath) + raise Exception('Illegal directory separator in dependency path %s' % subpath) normpath = posixpath.normpath(subpath) if posixpath.isabs(normpath): - raise Exception("Dependency path %s cannot be absolute" % subpath) + raise Exception('Dependency path %s cannot be absolute' % subpath) if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posixpath.sep): - raise Exception("Dependency path %s has to be inside the repository" % subpath) + raise Exception('Dependency path %s has to be inside the repository' % subpath) return os.path.join(path, *normpath.split(posixpath.sep)) @@ -245,17 +246,72 @@ def get_repo_type(repo): for name, repotype in repo_types.iteritems(): if repotype.istype(repo): return name - return "hg" + return 'hg' + + +def resolve_npm_dependencies(target, vcs): + """Install Node.js production-only dependencies if necessary and desired. + + When the target dependency has additional Node.js dependencies declared + run "npm install --only=production --loglevel=warn" to resolve the declared + dependencies. + + Additionally, make sure that any VCS will ignore the installed files. + + Requires Node.js to be installed locally. + """ + try: + with open(os.path.join(target, 'package.json'), 'r') as fp: + package_data = json.load(fp) + + # In case a package.json does not exist at all or if there are no + # production dependencies declared, we don't need to run npm and can + # bail out early. + if not package_data.get('dependencies', False): + return + except IOError: + return + + try: + # Create an empty file, which gets deleted after successfully + # installing Node.js dependencies. + lockfile_path = os.path.join(target, NPM_LOCKFILE) + open(lockfile_path, 'a').close() + + if os.name == 'nt': + # Windows' CreateProcess() (called by subprocess.Popen()) only + # resolves executables ending in .exe. The windows installation of + # Node.js only provides a npm.cmd, which is executable but won't + # be recognized as such by CreateProcess(). + npm_exec = 'npm.cmd' + else: + npm_exec = 'npm' + + cmd = [npm_exec, 'install', '--only=production', '--loglevel=warn', + '--no-package-lock', '--no-optional'] + subprocess.check_output(cmd, cwd=target) + + repo_types[vcs].ignore(os.path.join(target, NPM_LOCKFILE), target) + repo_types[vcs].ignore(os.path.join(target, 'node_modules'), target) + + os.remove(lockfile_path) + except OSError as e: + import errno + if e.errno == errno.ENOENT: + logging.error('Failed to install Node.js dependencies for %s,' + ' please ensure Node.js is installed.', target) + else: + raise def ensure_repo(parentrepo, parenttype, target, type, root, sourcename): if os.path.exists(target): - return + return False if SKIP_DEPENDENCY_UPDATES: - logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, " - "%s not cloned", target) - return + logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, ' + '%s not cloned', target) + return False postprocess_url = repo_types[type].postprocess_url root = postprocess_url(root) @@ -266,9 +322,10 @@ def ensure_repo(parentrepo, parenttype, else: url = urlparse.urljoin(root, sourcename) - logging.info("Cloning repository %s into %s" % (url, target)) + logging.info('Cloning repository %s into %s' % (url, target)) repo_types[type].clone(url, target) repo_types[parenttype].ignore(target, parentrepo) + return True def update_repo(target, type, revision): @@ -277,85 +334,92 @@ def update_repo(target, type, revision): if resolved_revision != current_revision: if SKIP_DEPENDENCY_UPDATES: - logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, " - "%s not checked out to %s", target, revision) - return + logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, ' + '%s not checked out to %s', target, revision) + return False if not resolved_revision: - logging.info("Revision %s is unknown, downloading remote changes" % revision) + logging.info('Revision %s is unknown, downloading remote changes' % revision) repo_types[type].pull(target) resolved_revision = repo_types[type].get_revision_id(target, revision) if not resolved_revision: - raise Exception("Failed to resolve revision %s" % revision) + raise Exception('Failed to resolve revision %s' % revision) - logging.info("Updating repository %s to revision %s" % (target, resolved_revision)) + logging.info('Updating repository %s to revision %s' % (target, resolved_revision)) repo_types[type].update(target, resolved_revision, revision) + return True + return False def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdependencies=set()): config = read_deps(repodir) if config is None: if level == 0: - logging.warning("No dependencies file in directory %s, nothing to do...\n%s" % (repodir, USAGE)) + logging.warning('No dependencies file in directory %s, nothing to do...\n%s' % (repodir, USAGE)) return if level >= 10: - logging.warning("Too much subrepository nesting, ignoring %s" % repo) + logging.warning('Too much subrepository nesting, ignoring %s' % repo) return if overrideroots is not None: - config["_root"] = overrideroots + config['_root'] = overrideroots - for dir, sources in config.iteritems(): - if (dir.startswith("_") or + for dir, sources in sorted(config.iteritems()): + if (dir.startswith('_') or skipdependencies.intersection([s[0] for s in sources if s[0]])): continue target = safe_join(repodir, dir) parenttype = get_repo_type(repodir) - _root = config.get("_root", {}) + _root = config.get('_root', {}) for key in sources.keys() + _root.keys(): - if key == parenttype or key is None and vcs != "*": + if key == parenttype or key is None and vcs != '*': vcs = key - source, rev = merge_seqs(sources.get("*"), sources.get(vcs)) + source, rev = merge_seqs(sources.get('*'), sources.get(vcs)) if not (vcs and source and rev): - logging.warning("No valid source / revision found to create %s" % target) + logging.warning('No valid source / revision found to create %s' % target) continue - ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ""), source) - update_repo(target, vcs, rev) + repo_cloned = ensure_repo(repodir, parenttype, target, vcs, + _root.get(vcs, ''), source) + repo_updated = update_repo(target, vcs, rev) + recent_npm_failed = os.path.exists(os.path.join(target, NPM_LOCKFILE)) + if repo_cloned or repo_updated or recent_npm_failed: + resolve_npm_dependencies(target, vcs) resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroots, skipdependencies=skipdependencies) - if self_update and "_self" in config and "*" in config["_self"]: - source = safe_join(repodir, config["_self"]["*"]) + if self_update and '_self' in config and '*' in config['_self']: + source = safe_join(repodir, config['_self']['*']) try: - with io.open(source, "rb") as handle: + with io.open(source, 'rb') as handle: sourcedata = handle.read() - except IOError, e: + except IOError as e: if e.errno != errno.ENOENT: raise logging.warning("File %s doesn't exist, skipping self-update" % source) return target = __file__ - with io.open(target, "rb") as handle: + with io.open(target, 'rb') as handle: targetdata = handle.read() if sourcedata != targetdata: logging.info("Updating %s from %s, don't forget to commit" % (target, source)) - with io.open(target, "wb") as handle: + with io.open(target, 'wb') as handle: handle.write(sourcedata) - if __name__ == "__main__": - logging.info("Restarting %s" % target) + if __name__ == '__main__': + logging.info('Restarting %s' % target) os.execv(sys.executable, [sys.executable, target] + sys.argv[1:]) else: - logging.warning("Cannot restart %s automatically, please rerun" % target) + logging.warning('Cannot restart %s automatically, please rerun' % target) def _ensure_line_exists(path, pattern): with open(path, 'a+') as f: + f.seek(0, os.SEEK_SET) file_content = [l.strip() for l in f.readlines()] if not pattern in file_content: file_content.append(pattern) @@ -364,12 +428,13 @@ def _ensure_line_exists(path, pattern): for l in file_content: print >>f, l -if __name__ == "__main__": + +if __name__ == '__main__': logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - parser = argparse.ArgumentParser(description="Verify dependencies for a set of repositories, by default the repository of this script.") - parser.add_argument("repos", metavar="repository", type=str, nargs="*", help="Repository path") - parser.add_argument("-q", "--quiet", action="store_true", help="Suppress informational output") + parser = argparse.ArgumentParser(description='Verify dependencies for a set of repositories, by default the repository of this script.') + parser.add_argument('repos', metavar='repository', type=str, nargs='*', help='Repository path') + parser.add_argument('-q', '--quiet', action='store_true', help='Suppress informational output') args = parser.parse_args() if args.quiet: