Issue 2539 - Move VCS abstraction part of ensure_dependencies into a separate module

ensure_dependencies.py

Index: ensure_dependencies.py
===================================================================
--- a/ensure_dependencies.py
+++ b/ensure_dependencies.py
@@ -1,145 +1,51 @@
-#!/usr/bin/env python
 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 import os
 import posixpath
 import re
 import io
 import errno
 import logging
-import subprocess
+import traceback
 import urlparse
 import argparse
-
 from collections import OrderedDict
 from ConfigParser import RawConfigParser
 
+from buildtools.vcs import repo_types
+
 USAGE = """
 A dependencies file should look like this:
 
   # VCS-specific root URLs for the repositories
   _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
-  # File to update this script from (optional)
-  _self = buildtools/ensure_dependencies.py
+  # Enable self-updates
+  _self = true
+  # Directory to be added to module search path when locating
+  # buildtools.ensure_dependencies module (optional, for self-update)
+  _module_path = subdir
   # Check out elemhidehelper repository into extensions/elemhidehelper directory
   # at tag "1.2".
   extensions/elemhidehelper = elemhidehelper 1.2
   # Check out buildtools repository into buildtools directory at VCS-specific
   # revision IDs.
   buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
 """
 
 SKIP_DEPENDENCY_UPDATES = os.environ.get(
   "SKIP_DEPENDENCY_UPDATES", ""
 ).lower() not in ("", "0", "false")
 
-class Mercurial():
-  def istype(self, repodir):
-    return os.path.exists(os.path.join(repodir, ".hg"))
-
-  def clone(self, source, target):
-    if not source.endswith("/"):
-      source += "/"
-    subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, target])
-
-  def get_revision_id(self, repo, rev=None):
-    command = ["hg", "id", "--repository", repo, "--id"]
-    if rev:
-      command.extend(["--rev", rev])
-
-    # Ignore stderr output and return code here: if revision lookup failed we
-    # should simply return an empty string.
-    result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
-    return result.strip()
-
-  def pull(self, repo):
-    subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"])
-
-  def update(self, repo, rev):
-    subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--check", "--rev", rev])
-
-  def ignore(self, target, repo):
-
-    if not self.istype(target):
-
-      config_path = os.path.join(repo, ".hg", "hgrc")
-      ignore_path = os.path.abspath(os.path.join(repo, ".hg", "dependencies"))
-
-      config = RawConfigParser()
-      config.read(config_path)
-
-      if not config.has_section("ui"):
-        config.add_section("ui")
-
-      config.set("ui", "ignore.dependencies", ignore_path)
-      with open(config_path, "w") as stream:
-        config.write(stream)
-
-      module = os.path.relpath(target, repo)
-      _ensure_line_exists(ignore_path, module)
-
-  def postprocess_url(self, url):
-    return url
-
-class Git():
-  def istype(self, repodir):
-    return os.path.exists(os.path.join(repodir, ".git"))
-
-  def clone(self, source, target):
-    source = source.rstrip("/")
-    if not source.endswith(".git"):
-      source += ".git"
-    subprocess.check_call(["git", "clone", "--quiet", source, target])
-
-  def get_revision_id(self, repo, rev="HEAD"):
-    command = ["git", "rev-parse", "--revs-only", rev + '^{commit}']
-    return subprocess.check_output(command, cwd=repo).strip()
-
-  def pull(self, repo):
-    # Fetch tracked branches, new tags and the list of available remote branches
-    subprocess.check_call(["git", "fetch", "--quiet", "--all", "--tags"], cwd=repo)
-    # Next we need to ensure all remote branches are tracked
-    newly_tracked = False
-    remotes = subprocess.check_output(["git", "branch", "--remotes"], cwd=repo)
-    for match in re.finditer(r"^\s*(origin/(\S+))$", remotes, re.M):
-      remote, local = match.groups()
-      with open(os.devnull, "wb") as devnull:
-        if subprocess.call(["git", "branch", "--track", local, remote],
-                           cwd=repo, stdout=devnull, stderr=devnull) == 0:
-          newly_tracked = True
-    # Finally fetch any newly tracked remote branches
-    if newly_tracked:
-      subprocess.check_call(["git", "fetch", "--quiet", "origin"], cwd=repo)
-
-  def update(self, repo, rev):
-    subprocess.check_call(["git", "checkout", "--quiet", rev], cwd=repo)
-
-  def ignore(self, target, repo):
-    module = os.path.relpath(target, repo)
-    exclude_file = os.path.join(repo, ".git", "info", "exclude")
-    _ensure_line_exists(exclude_file, module)
-
-  def postprocess_url(self, url):
-    # Handle alternative syntax of SSH URLS
-    if "@" in url and ":" in url and not urlparse.urlsplit(url).scheme:
-      return "ssh://" + url.replace(":", "/", 1)
-    return url
-
-repo_types = OrderedDict((
-  ("hg", Mercurial()),
-  ("git", Git()),
-))
-
 def parse_spec(path, line):
   if "=" not in line:
     logging.warning("Invalid line in file %s: %s" % (path, line))
     return None, None
 
   key, value = line.split("=", 1)
   key = key.strip()
   items = value.split()
@@ -283,59 +189,64 @@ def resolve_deps(repodir, level=0, self_
   for dir, revisions in config.iteritems():
     if dir.startswith("_") or revisions["_source"] in skipdependencies:
       continue
     target = safe_join(repodir, dir)
     ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"])
     update_repo(target, revisions)
     resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroots, skipdependencies=skipdependencies)
 
-  if self_update and "_self" in config and "*" in config["_self"]:
-    source = safe_join(repodir, config["_self"]["*"])
+  if self_update and config.get("_self", {}).get("*", "").lower() not in ("", "0", "false"):

[Sebastian Noack, 2015/05/19 11:52:41]

The logic checking the "true"-ness of the value is redundant with the check for
SKIP_ENSURE_DEPENENDICES. Moving it to a function?

[Sebastian Noack, 2015/05/19 13:25:36]

Just realized that we have a ConfigParser here. So you should just use
getboolean() instead.

+    original_path = sys.path

[Sebastian Noack, 2015/05/19 11:52:41]

This is useless, as you merely backup the reference to the list, which you don't
change anyway. If you want to backup it's content you have to copy the list:

  orig_path = sys.path[:]

+    if "_module_path" in config and "*" in config["_module_path"]:
+      sys.path.insert(safe_join(repodir, config["_module_path"]["*"]), 0)
+
     try:
-      with io.open(source, "rb") as handle:
-        sourcedata = handle.read()
-    except IOError, e:
-      if e.errno != errno.ENOENT:
-        raise
-      logging.warning("File %s doesn't exist, skipping self-update" % source)
-      return
+      from buildtools.script_compiler import compile_script
+      sourcedata = "".join(compile_script("buildtools.ensure_dependencies", [
+        "buildtools",
+        "buildtools.script_compiler",
+        "buildtools.vcs",
+      ]))
 
-    target = __file__
-    with io.open(target, "rb") as handle:
-      targetdata = handle.read()
+      target = __file__
+      try:
+        with io.open(target, "rb") as handle:

[Sebastian Noack, 2015/05/19 11:52:41]

Nit: You can just use the open() built-in function here.

[Sebastian Noack, 2015/05/19 11:52:41]

Nit: "file" would be a more appropriate variable name, to match Python's
terminology. Feel free to change it in the vcs module as well while on it.

+          targetdata = handle.read()
+      except IOError, e:
+        if e.errno != errno.ENOENT:
+          raise
+        targetdata = None
 
-    if sourcedata != targetdata:
-      logging.info("Updating %s from %s, don't forget to commit" % (source, target))
-      with io.open(target, "wb") as handle:
-        handle.write(sourcedata)
-      if __name__ == "__main__":
-        logging.info("Restarting %s" % target)
-        os.execv(sys.executable, [sys.executable, target] + sys.argv[1:])
-      else:
-        logging.warning("Cannot restart %s automatically, please rerun" % target)
-
-def _ensure_line_exists(path, pattern):
-  with open(path, 'a+') as f:
-    file_content = [l.strip() for l in f.readlines()]
-    if not pattern in file_content:
-      file_content.append(pattern)
-      f.seek(0, os.SEEK_SET)
-      f.truncate()
-      for l in file_content:
-        print >>f, l
+      if sourcedata != targetdata:
+        logging.info("Updating %s, don't forget to commit" % target)
+        with io.open(target, "wb") as handle:
+          handle.write(sourcedata)
+        if __name__ == "__main__":
+          logging.info("Restarting %s" % target)
+          os.execv(sys.executable, [sys.executable, target] + sys.argv[1:])
+        else:
+          logging.warning("Cannot restart %s automatically, please rerun" % target)
+    except Exception, e:
+      logging.warning("Failed to update %s, skipping self-update" % __file__)

[Sebastian Noack, 2015/05/19 11:52:41]

Pro-Tip: logging.warning(.., exc_info=True)

+      traceback.print_exc()
+    finally:
+      sys.path = original_path
 
 if __name__ == "__main__":
   logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
 
   parser = argparse.ArgumentParser(description="Verify dependencies for a set of repositories, by default the repository of this script.")
   parser.add_argument("repos", metavar="repository", type=str, nargs="*", help="Repository path")
+  parser.add_argument("-s", "--self", metavar="path", type=str, help="Update ensure_dependencies.py at this location")

[Wladimir Palant, 2015/05/18 15:27:00]

This now allows two ways of setting up ensure_dependencies.py in a new
repository:

* Run |python -m buildtools.ensure_dependencies -s ensure_dependencies.py|
* Copy ensure_dependencies.py from buildtools repository and run |python
ensure_dependencies.py|

Both approaches will work only if self-update is enabled.

   parser.add_argument("-q", "--quiet", action="store_true", help="Suppress informational output")
   args = parser.parse_args()
 
   if args.quiet:
     logging.disable(logging.INFO)
+  if args.self:
+    __file__ = args.self
 
   repos = args.repos
   if not len(repos):
     repos = [os.path.dirname(__file__)]
   for repo in repos:
     resolve_deps(repo)