Issue 3194 - Allow multiple sources for a dependency

ensure_dependencies.py

 #!/usr/bin/env python
 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 import os
 import posixpath
 import re
 import io
 import errno
 import logging
 import subprocess
 import urlparse
 import argparse
 
 from collections import OrderedDict
 from ConfigParser import RawConfigParser
 
 USAGE = """
 A dependencies file should look like this:
 
   # VCS-specific root URLs for the repositories
   _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
   # File to update this script from (optional)
   _self = buildtools/ensure_dependencies.py
-  # Check out elemhidehelper repository into extensions/elemhidehelper directory
-  # at tag "1.2".
+  # Clone elemhidehelper repository into extensions/elemhidehelper directory at
+  # tag "1.2".
   extensions/elemhidehelper = elemhidehelper 1.2
-  # Check out buildtools repository into buildtools directory at VCS-specific
+  # Clone buildtools repository into buildtools directory at VCS-specific
   # revision IDs.
   buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
+  # Clone the adblockplus repository into adblockplus directory, overwriting the
+  # usual source URL for Git repository and specifying VCS specific revision IDs.
+  adblockplus = adblockplus hg:893426c6a6ab git:git@github.com:kzar/adblockplus.git@b2ffd52b
+  # Clone the adblockpluschrome repository into the adblockpluschrome directory,
+  # from a specific Git repository, specifying the revision ID.
+  adblockpluschrome = git:git@github.com:kzar/adblockpluschrome.git@1fad3a7

[Wladimir Palant, 2015/10/16 10:28:45]

Nit: I don't think your username belongs in the examples here - it should be
something generic like gituser.

[kzar, 2015/10/16 10:44:52]

Done.

 """
 
 SKIP_DEPENDENCY_UPDATES = os.environ.get(
   "SKIP_DEPENDENCY_UPDATES", ""
 ).lower() not in ("", "0", "false")
 
 class Mercurial():
   def istype(self, repodir):
     return os.path.exists(os.path.join(repodir, ".hg"))
 
@@ skipping 83 matching lines @@
     # Handle alternative syntax of SSH URLS
     if "@" in url and ":" in url and not urlparse.urlsplit(url).scheme:
       return "ssh://" + url.replace(":", "/", 1)
     return url
 
 repo_types = OrderedDict((
   ("hg", Mercurial()),
   ("git", Git()),
 ))
 
+# [vcs:]value
+item_regexp = re.compile(
+  "^(?:(" + "|".join(map(re.escape, repo_types.keys())) +"):)?"
+  "(.+)$"
+)
+
+# [url@]rev
+source_regexp = re.compile(
+  "^(?:(.*)@)?"
+  "(.+)$"
+)
+
+def merge_seqs(seq1, seq2):
+  """Return a tuple of any truthy values from the suplied sequences
+
+  (None, 2), (1,)      => (1, 2)
+  None, (1, 2)         => (1, 2)
+  (1, 2), (3, 4)       => (3, 4)
+  """
+  return tuple(map(lambda item1, item2: item2 or item1, seq1 or (), seq2 or ()))

[Sebastian Noack, 2015/10/16 10:24:49]

Nit: Is coercing to a tuple really necessary? Won't  list as returned by map do
as well?

[kzar, 2015/10/16 10:44:52]

You're right, a list will do. (I figured a tuple would be better as we're not
planning to mutate it, but on the other hand it's kind of pointless to coerce a
list that's already suitable for our needs.)
Done.

+
 def parse_spec(path, line):
   if "=" not in line:
     logging.warning("Invalid line in file %s: %s" % (path, line))
     return None, None
 
   key, value = line.split("=", 1)
   key = key.strip()
   items = value.split()
   if not len(items):
     logging.warning("No value specified for key %s in file %s" % (key, path))
     return key, None
 
   result = OrderedDict()
-  if not key.startswith("_"):
-    result["_source"] = items.pop(0)
+  is_dependency_field = not key.startswith("_")
 
-  for item in items:
-    if ":" in item:
-      type, value = item.split(":", 1)
-    else:
-      type, value = ("*", item)
-    if type in result:
-      logging.warning("Ignoring duplicate value for type %s (key %s in file %s)" % (type, key, path))
-    else:
-      result[type] = value
+  for i, item in enumerate(items):
+    try:
+      vcs, value = re.search(item_regexp, item).groups()
+      vcs = vcs or "*"
+      if is_dependency_field:
+        if i == 0 and vcs == "*":
+          # In order to be backwards compatible we have to assume that the first
+          # source contains only a URL/path for the repo if it does not contain
+          # the VCS part
+          url_rev = (value, None)
+        else:
+          url_rev = re.search(source_regexp, value).groups()
+        result[vcs] = merge_seqs(result.get(vcs), url_rev)
+      else:
+        if vcs in result:
+          logging.warning("Ignoring duplicate value for type %r "
+                          "(key %r in file %r)" % (vcs, key, path))
+        result[vcs] = value
+    except AttributeError:
+      logging.warning("Ignoring invalid item %r for type %r "
+                      "(key %r in file %r)" % (item, vcs, key, path))
+      continue
   return key, result
 
 def read_deps(repodir):
   result = {}
   deps_path = os.path.join(repodir, "dependencies")
   try:
     with io.open(deps_path, "rt", encoding="utf-8") as handle:
       for line in handle:
         # Remove comments and whitespace
         line = re.sub(r"#.*", "", line).strip()
@@ skipping 21 matching lines @@
   if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posixpath.sep):
     raise Exception("Dependency path %s has to be inside the repository" % subpath)
   return os.path.join(path, *normpath.split(posixpath.sep))
 
 def get_repo_type(repo):
   for name, repotype in repo_types.iteritems():
     if repotype.istype(repo):
       return name
   return None
 
-def ensure_repo(parentrepo, target, roots, sourcename):
+def ensure_repo(parentrepo, parenttype, target, type, root, sourcename):
   if os.path.exists(target):
     return
 
   if SKIP_DEPENDENCY_UPDATES:
     logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, "
                     "%s not cloned", target)
     return
 
-  parenttype = get_repo_type(parentrepo)
-  type = None
-  for key in roots:
-    if key == parenttype or (key in repo_types and type is None):
-      type = key
-  if type is None:
-    raise Exception("No valid source found to create %s" % target)
-
   postprocess_url = repo_types[type].postprocess_url
-  root = postprocess_url(roots[type])
+  root = postprocess_url(root)
   sourcename = postprocess_url(sourcename)
 
   if os.path.exists(root):
     url = os.path.join(root, sourcename)
   else:
     url = urlparse.urljoin(root, sourcename)
 
   logging.info("Cloning repository %s into %s" % (url, target))
   repo_types[type].clone(url, target)
+  repo_types[parenttype].ignore(target, parentrepo)
 
-  for repo in repo_types.itervalues():
-    if repo.istype(parentrepo):
-      repo.ignore(target, parentrepo)
-
-def update_repo(target, revisions):
-  type = get_repo_type(target)
-  if type is None:
-    logging.warning("Type of repository %s unknown, skipping update" % target)
-    return
-
-  if type in revisions:
-    revision = revisions[type]
-  elif "*" in revisions:
-    revision = revisions["*"]
-  else:
-    logging.warning("No revision specified for repository %s (type %s), skipping update" % (target, type))
-    return
-
+def update_repo(target, type, revision):
   resolved_revision = repo_types[type].get_revision_id(target, revision)
   current_revision = repo_types[type].get_revision_id(target)
 
   if resolved_revision != current_revision:
     if SKIP_DEPENDENCY_UPDATES:
       logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, "
                       "%s not checked out to %s", target, revision)
       return
 
     if not resolved_revision:
@@ skipping 12 matching lines @@
     if level == 0:
       logging.warning("No dependencies file in directory %s, nothing to do...\n%s" % (repodir, USAGE))
     return
   if level >= 10:
     logging.warning("Too much subrepository nesting, ignoring %s" % repo)
     return
 
   if overrideroots is not None:
     config["_root"] = overrideroots
 
-  for dir, revisions in config.iteritems():
-    if dir.startswith("_") or revisions["_source"] in skipdependencies:
+  for dir, sources in config.iteritems():
+    if (dir.startswith("_") or
+        skipdependencies.intersection([s[0] for s in sources if s[0]])):
       continue
+
     target = safe_join(repodir, dir)
-    ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"])
-    update_repo(target, revisions)
-    resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroots, skipdependencies=skipdependencies)
+    parenttype = get_repo_type(repodir)
+    _root = config.get("_root", {})
+
+    for key in sources.keys() + _root.keys():
+      if key == parenttype or key is None and vcs != "*":
+        vcs = key
+    source, rev = merge_seqs(sources.get("*"), sources.get(vcs))
+
+    if not (vcs and source and rev):
+      logging.warning("No valid source / revision found to create %s" % target)
+      continue
+
+    ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ""), source)
+    update_repo(target, vcs, rev)
+    resolve_deps(target, level + 1, self_update=False,
+                 overrideroots=overrideroots, skipdependencies=skipdependencies)
 
   if self_update and "_self" in config and "*" in config["_self"]:
     source = safe_join(repodir, config["_self"]["*"])
     try:
       with io.open(source, "rb") as handle:
         sourcedata = handle.read()
     except IOError, e:
       if e.errno != errno.ENOENT:
         raise
       logging.warning("File %s doesn't exist, skipping self-update" % source)
@@ skipping 32 matching lines @@
   args = parser.parse_args()
 
   if args.quiet:
     logging.disable(logging.INFO)
 
   repos = args.repos
   if not len(repos):
     repos = [os.path.dirname(__file__)]
   for repo in repos:
     resolve_deps(repo)