Issue 3194 - Allow multiple sources for a dependency

ensure_dependencies.py

 #!/usr/bin/env python
 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 import os
 import posixpath
@@ skipping 14 matching lines @@
   # VCS-specific root URLs for the repositories
   _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
   # File to update this script from (optional)
   _self = buildtools/ensure_dependencies.py
   # Check out elemhidehelper repository into extensions/elemhidehelper directory
   # at tag "1.2".
   extensions/elemhidehelper = elemhidehelper 1.2
   # Check out buildtools repository into buildtools directory at VCS-specific
   # revision IDs.
   buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
+  # Check out the adblockplus repository into adblockplus directory, overwriting

[Sebastian Noack, 2015/10/15 15:19:41]

Nit: "Check out" isn't hg/git terminology.

[kzar, 2015/10/15 16:12:25]

Done.

+  # the usual source URL for Git repository and specifying VCS specific revision
+  # IDs.
+  adblockplus = adblockplus hg:893426c6a6ab git:git@github.com:kzar/adblockplus.git@b2ffd52b
+  # Check out the adblockpluschrome repository into the adblockpluschrome
+  # directory, from a specific Git repository, specifying the revision ID.
+  adblockpluschrome = git:git@github.com:kzar/adblockpluschrome.git@1fad3a7
 """
 
 SKIP_DEPENDENCY_UPDATES = os.environ.get(
   "SKIP_DEPENDENCY_UPDATES", ""
 ).lower() not in ("", "0", "false")
 
 class Mercurial():
   def istype(self, repodir):
     return os.path.exists(os.path.join(repodir, ".hg"))
 
@@ skipping 83 matching lines @@
     # Handle alternative syntax of SSH URLS
     if "@" in url and ":" in url and not urlparse.urlsplit(url).scheme:
       return "ssh://" + url.replace(":", "/", 1)
     return url
 
 repo_types = OrderedDict((
   ("hg", Mercurial()),
   ("git", Git()),
 ))
 
+# [vcs:]value
+item_regexp = re.compile(
+  "^(?:(" + "|".join(repo_types.keys()) +"):)?"
+  "(.+)$"
+)
+
+# [url@]rev
+source_regexp = re.compile(
+  "^(?:(.*)@)?"
+  "(.+)$"
+)
+
+def merge_tuples(tuple_1, tuple_2):
+  """Return tuple containing any truthy values from the suplied tuples
+
+  (None, 2), (1,)      => (1, 2)
+  None, (1, 2)         => (1, 2)
+  (1, 2), (3, 4)       => (3, 4)
+  """
+  return tuple(i2 or i1 for i1, i2 in map(None, tuple_1 or (), tuple_2 or ()))
+
 def parse_spec(path, line):
   if "=" not in line:
     logging.warning("Invalid line in file %s: %s" % (path, line))
     return None, None
 
   key, value = line.split("=", 1)
   key = key.strip()
   items = value.split()
   if not len(items):
     logging.warning("No value specified for key %s in file %s" % (key, path))
     return key, None
 
   result = OrderedDict()
-  if not key.startswith("_"):
-    result["_source"] = items.pop(0)
+  is_dependency_field = not key.startswith("_")
 
-  for item in items:
-    if ":" in item:
-      type, value = item.split(":", 1)
-    else:
-      type, value = ("*", item)
-    if type in result:
-      logging.warning("Ignoring duplicate value for type %s (key %s in file %s)" % (type, key, path))
-    else:
-      result[type] = value
+  for i, item in enumerate(items):
+    try:
+      vcs, value = re.match(item_regexp, item).groups()
+      vcs = vcs or "*"
+      if is_dependency_field:
+        if i == 0 and vcs == "*":
+          # In order to be backwards compatible we have to assume that the first
+          # source contains only a URL/path for the repo if it does not contain
+          # the VCS part
+          url_rev = (value, None)
+        else:
+          url_rev = re.match(source_regexp, value).groups()

[Sebastian Noack, 2015/10/15 15:19:41]

We decided a while ago to not use .match() anymore, but only .search(). That is
because the behavior of .match(), implying end of string, but not not begin of
string, is apparently quite confusing to some developers.

[kzar, 2015/10/15 16:12:25]

Done.

+        result[vcs] = merge_tuples(result.get(vcs), url_rev)
+      else:
+        if vcs in result:
+          logging.warning("Ignoring duplicate value for type %s"
+                          "(key %s in file %s)" % (vcs, key, path))
+        result[vcs] = value
+    except AttributeError:
+      logging.warning("Ignoring invalid item '%s' for type %s"

[Sebastian Noack, 2015/10/15 15:19:41]

How about using %r instead '%s'?

[kzar, 2015/10/15 16:12:25]

(Some further testing showed it's clearer if all these values are quoted.)
Done.

+                      "(key %s in file %s)" % (item, vcs, key, path))
+      continue
   return key, result
 
 def read_deps(repodir):
   result = {}
   deps_path = os.path.join(repodir, "dependencies")
   try:
     with io.open(deps_path, "rt", encoding="utf-8") as handle:
       for line in handle:
         # Remove comments and whitespace
         line = re.sub(r"#.*", "", line).strip()
@@ skipping 21 matching lines @@
   if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posixpath.sep):
     raise Exception("Dependency path %s has to be inside the repository" % subpath)
   return os.path.join(path, *normpath.split(posixpath.sep))
 
 def get_repo_type(repo):
   for name, repotype in repo_types.iteritems():
     if repotype.istype(repo):
       return name
   return None
 
-def ensure_repo(parentrepo, target, roots, sourcename):
+def ensure_repo(parentrepo, parenttype, target, type, root, sourcename):
   if os.path.exists(target):
     return
 
   if SKIP_DEPENDENCY_UPDATES:
     logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, "
                     "%s not cloned", target)
     return
 
-  parenttype = get_repo_type(parentrepo)
-  type = None
-  for key in roots:
-    if key == parenttype or (key in repo_types and type is None):
-      type = key
-  if type is None:
-    raise Exception("No valid source found to create %s" % target)
-
   postprocess_url = repo_types[type].postprocess_url
-  root = postprocess_url(roots[type])
+  root = postprocess_url(root)
   sourcename = postprocess_url(sourcename)
 
   if os.path.exists(root):
     url = os.path.join(root, sourcename)
   else:
     url = urlparse.urljoin(root, sourcename)
 
   logging.info("Cloning repository %s into %s" % (url, target))
   repo_types[type].clone(url, target)
+  repo_types[parenttype].ignore(target, parentrepo)
 
-  for repo in repo_types.itervalues():
-    if repo.istype(parentrepo):
-      repo.ignore(target, parentrepo)
-
-def update_repo(target, revisions):
-  type = get_repo_type(target)
-  if type is None:
-    logging.warning("Type of repository %s unknown, skipping update" % target)
-    return
-
-  if type in revisions:
-    revision = revisions[type]
-  elif "*" in revisions:
-    revision = revisions["*"]
-  else:
-    logging.warning("No revision specified for repository %s (type %s), skipping update" % (target, type))
-    return
-
+def update_repo(target, type, revision):
   resolved_revision = repo_types[type].get_revision_id(target, revision)
   current_revision = repo_types[type].get_revision_id(target)
 
   if resolved_revision != current_revision:
     if SKIP_DEPENDENCY_UPDATES:
       logging.warning("SKIP_DEPENDENCY_UPDATES environment variable set, "
                       "%s not checked out to %s", target, revision)
       return
 
     if not resolved_revision:
@@ skipping 12 matching lines @@
     if level == 0:
       logging.warning("No dependencies file in directory %s, nothing to do...\n%s" % (repodir, USAGE))
     return
   if level >= 10:
     logging.warning("Too much subrepository nesting, ignoring %s" % repo)
     return
 
   if overrideroots is not None:
     config["_root"] = overrideroots
 
-  for dir, revisions in config.iteritems():
-    if dir.startswith("_") or revisions["_source"] in skipdependencies:
+  for dir, sources in config.iteritems():
+    if (dir.startswith("_") or
+        skipdependencies.intersection([s[0] for s in sources if s[0]])):
       continue
+
     target = safe_join(repodir, dir)
-    ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"])
-    update_repo(target, revisions)
-    resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroots, skipdependencies=skipdependencies)
+    parenttype = get_repo_type(repodir)
+    _root = config.get("_root", {})
+
+    for key in sources.keys() + _root.keys():
+      if key == parenttype or key is None and vcs != "*":
+        vcs = key
+    source, rev = merge_tuples(sources.get("*"), sources.get(vcs))
+
+    if not (vcs and source and rev):
+      logging.warning("No valid source / revision found to create %s" % target)
+      continue
+
+    ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ""), source)
+    update_repo(target, vcs, rev)
+    resolve_deps(target, level + 1, self_update=False,
+                 overrideroots=overrideroots, skipdependencies=skipdependencies)
 
   if self_update and "_self" in config and "*" in config["_self"]:
     source = safe_join(repodir, config["_self"]["*"])
     try:
       with io.open(source, "rb") as handle:
         sourcedata = handle.read()
     except IOError, e:
       if e.errno != errno.ENOENT:
         raise
       logging.warning("File %s doesn't exist, skipping self-update" % source)
@@ skipping 32 matching lines @@
   args = parser.parse_args()
 
   if args.quiet:
     logging.disable(logging.INFO)
 
   repos = args.repos
   if not len(repos):
     repos = [os.path.dirname(__file__)]
   for repo in repos:
     resolve_deps(repo)