Issue 5757 - Update buildtools dependency, (re)move legacy extensions

ensure_dependencies.py

 #!/usr/bin/env python

[Vasily Kuznetsov, 2017/11/30 17:34:00]

I have not properly reviewed this script, assuming that it's just copied from
buildtools.

[tlucas, 2017/11/30 18:15:26]

Yes, this is the result of ensure_dependencies.py's self update.

 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 import os
 import posixpath
 import re
 import io
 import errno
 import logging
 import subprocess
 import urlparse
 import argparse
+import json
 
 from collections import OrderedDict
 from ConfigParser import RawConfigParser
 
 USAGE = '''
 A dependencies file should look like this:
 
   # VCS-specific root URLs for the repositories
   _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
   # File to update this script from (optional)
   _self = buildtools/ensure_dependencies.py
   # Clone elemhidehelper repository into extensions/elemhidehelper directory at
   # tag "1.2".
   extensions/elemhidehelper = elemhidehelper 1.2
   # Clone buildtools repository into buildtools directory at VCS-specific
   # revision IDs.
   buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
   # Clone the adblockplus repository into adblockplus directory, overwriting the
   # usual source URL for Git repository and specifying VCS specific revision IDs.
   adblockplus = adblockplus hg:893426c6a6ab git:git@github.com:user/adblockplus.git@b2ffd52b
   # Clone the adblockpluschrome repository into the adblockpluschrome directory,
   # from a specific Git repository, specifying the revision ID.
   adblockpluschrome = git:git@github.com:user/adblockpluschrome.git@1fad3a7
 '''
 
 SKIP_DEPENDENCY_UPDATES = os.environ.get(
     'SKIP_DEPENDENCY_UPDATES', ''
 ).lower() not in ('', '0', 'false')
 
+NPM_LOCKFILE = '.npm_install_lock'
+
 
 class Mercurial():
     def istype(self, repodir):
         return os.path.exists(os.path.join(repodir, '.hg'))
 
     def clone(self, source, target):
         if not source.endswith('/'):
             source += '/'
         subprocess.check_call(['hg', 'clone', '--quiet', '--noupdate', source, target])
 
@@ skipping 72 matching lines @@
         module = os.path.sep + os.path.relpath(target, repo)
         exclude_file = os.path.join(repo, '.git', 'info', 'exclude')
         _ensure_line_exists(exclude_file, module)
 
     def postprocess_url(self, url):
         # Handle alternative syntax of SSH URLS
         if '@' in url and ':' in url and not urlparse.urlsplit(url).scheme:
             return 'ssh://' + url.replace(':', '/', 1)
         return url
 
+
 repo_types = OrderedDict((
     ('hg', Mercurial()),
     ('git', Git()),
 ))
 
 # [vcs:]value
 item_regexp = re.compile(
     '^(?:(' + '|'.join(map(re.escape, repo_types.keys())) + '):)?'
     '(.+)$'
 )
@@ skipping 90 matching lines @@
     return os.path.join(path, *normpath.split(posixpath.sep))
 
 
 def get_repo_type(repo):
     for name, repotype in repo_types.iteritems():
         if repotype.istype(repo):
             return name
     return 'hg'
 
 
+def resolve_npm_dependencies(target, vcs):
+    """Install Node.js production-only dependencies if necessary and desired.
+
+    When the target dependency has additional Node.js dependencies declared
+    run "npm install --only=production --loglevel=warn" to resolve the declared
+    dependencies.
+
+    Additionally, make sure that any VCS will ignore the installed files.
+
+    Requires Node.js to be installed locally.
+    """
+    try:
+        with open(os.path.join(target, 'package.json'), 'r') as fp:
+            package_data = json.load(fp)
+
+        # In case a package.json does not exist at all or if there are no
+        # production dependencies declared, we don't need to run npm and can
+        # bail out early.
+        if not package_data.get('dependencies', False):
+            return
+    except IOError:
+        return
+
+    try:
+        # Create an empty file, which gets deleted after successfully
+        # installing Node.js dependencies.
+        lockfile_path = os.path.join(target, NPM_LOCKFILE)
+        open(lockfile_path, 'a').close()
+
+        cmd = ['npm', 'install', '--only=production', '--loglevel=warn',
+               '--no-package-lock', '--no-optional']
+        subprocess.check_output(cmd, cwd=target)
+
+        repo_types[vcs].ignore(os.path.join(target, NPM_LOCKFILE), target)
+        repo_types[vcs].ignore(os.path.join(target, 'node_modules'), target)
+
+        os.remove(lockfile_path)
+    except OSError as e:
+        import errno
+        if e.errno == errno.ENOENT:
+            logging.error('Failed to install Node.js dependencies for %s,'
+                          ' please ensure Node.js is installed.', target)
+        else:
+            raise
+
+
 def ensure_repo(parentrepo, parenttype, target, type, root, sourcename):
     if os.path.exists(target):
-        return
+        return False
 
     if SKIP_DEPENDENCY_UPDATES:
         logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, '
                         '%s not cloned', target)
-        return
+        return False
 
     postprocess_url = repo_types[type].postprocess_url
     root = postprocess_url(root)
     sourcename = postprocess_url(sourcename)
 
     if os.path.exists(root):
         url = os.path.join(root, sourcename)
     else:
         url = urlparse.urljoin(root, sourcename)
 
     logging.info('Cloning repository %s into %s' % (url, target))
     repo_types[type].clone(url, target)
     repo_types[parenttype].ignore(target, parentrepo)
+    return True
 
 
 def update_repo(target, type, revision):
     resolved_revision = repo_types[type].get_revision_id(target, revision)
     current_revision = repo_types[type].get_revision_id(target)
 
     if resolved_revision != current_revision:
         if SKIP_DEPENDENCY_UPDATES:
             logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, '
                             '%s not checked out to %s', target, revision)
-            return
+            return False
 
         if not resolved_revision:
             logging.info('Revision %s is unknown, downloading remote changes' % revision)
             repo_types[type].pull(target)
             resolved_revision = repo_types[type].get_revision_id(target, revision)
             if not resolved_revision:
                 raise Exception('Failed to resolve revision %s' % revision)
 
         logging.info('Updating repository %s to revision %s' % (target, resolved_revision))
         repo_types[type].update(target, resolved_revision, revision)
+        return True
+    return False
 
 
 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdependencies=set()):
     config = read_deps(repodir)
     if config is None:
         if level == 0:
             logging.warning('No dependencies file in directory %s, nothing to do...\n%s' % (repodir, USAGE))
         return
     if level >= 10:
         logging.warning('Too much subrepository nesting, ignoring %s' % repo)
         return
 
     if overrideroots is not None:
         config['_root'] = overrideroots
 
-    for dir, sources in config.iteritems():
+    for dir, sources in sorted(config.iteritems()):
         if (dir.startswith('_') or
             skipdependencies.intersection([s[0] for s in sources if s[0]])):
             continue
 
         target = safe_join(repodir, dir)
         parenttype = get_repo_type(repodir)
         _root = config.get('_root', {})
 
         for key in sources.keys() + _root.keys():
             if key == parenttype or key is None and vcs != '*':
                 vcs = key
         source, rev = merge_seqs(sources.get('*'), sources.get(vcs))
 
         if not (vcs and source and rev):
             logging.warning('No valid source / revision found to create %s' % target)
             continue
 
-        ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ''), source)
-        update_repo(target, vcs, rev)
+        repo_cloned = ensure_repo(repodir, parenttype, target, vcs,
+                                  _root.get(vcs, ''), source)
+        repo_updated = update_repo(target, vcs, rev)
+        recent_npm_failed = os.path.exists(os.path.join(target, NPM_LOCKFILE))
+        if repo_cloned or repo_updated or recent_npm_failed:
+            resolve_npm_dependencies(target, vcs)
         resolve_deps(target, level + 1, self_update=False,
                      overrideroots=overrideroots, skipdependencies=skipdependencies)
 
     if self_update and '_self' in config and '*' in config['_self']:
         source = safe_join(repodir, config['_self']['*'])
         try:
             with io.open(source, 'rb') as handle:
                 sourcedata = handle.read()
         except IOError as e:
             if e.errno != errno.ENOENT:
@@ skipping 20 matching lines @@
     with open(path, 'a+') as f:
         f.seek(0, os.SEEK_SET)
         file_content = [l.strip() for l in f.readlines()]
         if not pattern in file_content:
             file_content.append(pattern)
             f.seek(0, os.SEEK_SET)
             f.truncate()
             for l in file_content:
                 print >>f, l
 
+
 if __name__ == '__main__':
     logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
 
     parser = argparse.ArgumentParser(description='Verify dependencies for a set of repositories, by default the repository of this script.')
     parser.add_argument('repos', metavar='repository', type=str, nargs='*', help='Repository path')
     parser.add_argument('-q', '--quiet', action='store_true', help='Suppress informational output')
     args = parser.parse_args()
 
     if args.quiet:
         logging.disable(logging.INFO)
 
     repos = args.repos
     if not len(repos):
         repos = [os.path.dirname(__file__)]
     for repo in repos:
         resolve_deps(repo)