Issue 3194 - Allow multiple sources for a dependency

ensure_dependencies.py

 #!/usr/bin/env python
 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 import os
 import posixpath
 import re
 import io
 import errno
 import logging
 import subprocess
 import urlparse
 import argparse
 
 from collections import OrderedDict
 from ConfigParser import RawConfigParser
 
 USAGE = """
 A dependencies file should look like this:
 
   # VCS-specific root URLs for the repositories
   _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
   # File to update this script from (optional)
   _self = buildtools/ensure_dependencies.py
-  # Check out elemhidehelper repository into extensions/elemhidehelper directory
-  # at tag "1.2".
+  # Clone elemhidehelper repository into extensions/elemhidehelper directory at
+  # tag "1.2".
   extensions/elemhidehelper = elemhidehelper 1.2
-  # Check out buildtools repository into buildtools directory at VCS-specific
+  # Clone buildtools repository into buildtools directory at VCS-specific
   # revision IDs.
   buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
-  # Check out the adblockplus repository into adblockplus directory, overwriting

[Sebastian Noack, 2015/10/15 15:19:41]

Nit: "Check out" isn't hg/git terminology.

[kzar, 2015/10/15 16:12:25]

Done.

-  # the usual source URL for Git repository and specifying VCS specific revision
-  # IDs.
-  adblockplus = adblockplus hg:893426c6a6ab git:git@github.com:kzar/adblockplus.git@b2ffd52b
-  # Check out the adblockpluschrome repository into the adblockpluschrome
-  # directory, from a specific Git repository, specifying the revision ID.
-  adblockpluschrome = git:git@github.com:kzar/adblockpluschrome.git@1fad3a7
+  # Clone the adblockplus repository into adblockplus directory, overwriting the
+  # usual source URL for Git repository and specifying VCS specific revision IDs.
+  adblockplus = adblockplus hg:893426c6a6ab git:git@github.com:user/adblockplus.git@b2ffd52b
+  # Clone the adblockpluschrome repository into the adblockpluschrome directory,
+  # from a specific Git repository, specifying the revision ID.
+  adblockpluschrome = git:git@github.com:user/adblockpluschrome.git@1fad3a7
 """
 
 SKIP_DEPENDENCY_UPDATES = os.environ.get(
   "SKIP_DEPENDENCY_UPDATES", ""
 ).lower() not in ("", "0", "false")
 
 class Mercurial():
   def istype(self, repodir):
     return os.path.exists(os.path.join(repodir, ".hg"))
 
@@ skipping 85 matching lines @@
       return "ssh://" + url.replace(":", "/", 1)
     return url
 
 repo_types = OrderedDict((
   ("hg", Mercurial()),
   ("git", Git()),
 ))
 
 # [vcs:]value
 item_regexp = re.compile(
-  "^(?:(" + "|".join(repo_types.keys()) +"):)?"
+  "^(?:(" + "|".join(map(re.escape, repo_types.keys())) +"):)?"
   "(.+)$"
 )
 
 # [url@]rev
 source_regexp = re.compile(
   "^(?:(.*)@)?"
   "(.+)$"
 )
 
-def merge_tuples(tuple_1, tuple_2):
-  """Return tuple containing any truthy values from the suplied tuples
-
-  (None, 2), (1,)      => (1, 2)
-  None, (1, 2)         => (1, 2)
-  (1, 2), (3, 4)       => (3, 4)
+def merge_seqs(seq1, seq2):
+  """Return a list of any truthy values from the suplied sequences
+
+  (None, 2), (1,)      => [1, 2]
+  None, (1, 2)         => [1, 2]
+  (1, 2), (3, 4)       => [3, 4]
   """
-  return tuple(i2 or i1 for i1, i2 in map(None, tuple_1 or (), tuple_2 or ()))
+  return map(lambda item1, item2: item2 or item1, seq1 or (), seq2 or ())
 
 def parse_spec(path, line):
   if "=" not in line:
     logging.warning("Invalid line in file %s: %s" % (path, line))
     return None, None
 
   key, value = line.split("=", 1)
   key = key.strip()
   items = value.split()
   if not len(items):
     logging.warning("No value specified for key %s in file %s" % (key, path))
     return key, None
 
   result = OrderedDict()
   is_dependency_field = not key.startswith("_")
 
   for i, item in enumerate(items):
     try:
-      vcs, value = re.match(item_regexp, item).groups()
+      vcs, value = re.search(item_regexp, item).groups()
       vcs = vcs or "*"
       if is_dependency_field:
         if i == 0 and vcs == "*":
           # In order to be backwards compatible we have to assume that the first
           # source contains only a URL/path for the repo if it does not contain
           # the VCS part
           url_rev = (value, None)
         else:
-          url_rev = re.match(source_regexp, value).groups()

[Sebastian Noack, 2015/10/15 15:19:41]

We decided a while ago to not use .match() anymore, but only .search(). That is
because the behavior of .match(), implying end of string, but not not begin of
string, is apparently quite confusing to some developers.

[kzar, 2015/10/15 16:12:25]

Done.

-        result[vcs] = merge_tuples(result.get(vcs), url_rev)
+          url_rev = re.search(source_regexp, value).groups()
+        result[vcs] = merge_seqs(result.get(vcs), url_rev)
       else:
         if vcs in result:
-          logging.warning("Ignoring duplicate value for type %s"
-                          "(key %s in file %s)" % (vcs, key, path))
+          logging.warning("Ignoring duplicate value for type %r "
+                          "(key %r in file %r)" % (vcs, key, path))
         result[vcs] = value
     except AttributeError:
-      logging.warning("Ignoring invalid item '%s' for type %s"

[Sebastian Noack, 2015/10/15 15:19:41]

How about using %r instead '%s'?

[kzar, 2015/10/15 16:12:25]

(Some further testing showed it's clearer if all these values are quoted.)
Done.

-                      "(key %s in file %s)" % (item, vcs, key, path))
+      logging.warning("Ignoring invalid item %r for type %r "
+                      "(key %r in file %r)" % (item, vcs, key, path))
       continue
   return key, result
 
 def read_deps(repodir):
   result = {}
   deps_path = os.path.join(repodir, "dependencies")
   try:
     with io.open(deps_path, "rt", encoding="utf-8") as handle:
       for line in handle:
         # Remove comments and whitespace
@@ skipping 89 matching lines @@
         skipdependencies.intersection([s[0] for s in sources if s[0]])):
       continue
 
     target = safe_join(repodir, dir)
     parenttype = get_repo_type(repodir)
     _root = config.get("_root", {})
 
     for key in sources.keys() + _root.keys():
       if key == parenttype or key is None and vcs != "*":
         vcs = key
-    source, rev = merge_tuples(sources.get("*"), sources.get(vcs))
+    source, rev = merge_seqs(sources.get("*"), sources.get(vcs))
 
     if not (vcs and source and rev):
       logging.warning("No valid source / revision found to create %s" % target)
       continue
 
     ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ""), source)
     update_repo(target, vcs, rev)
     resolve_deps(target, level + 1, self_update=False,
                  overrideroots=overrideroots, skipdependencies=skipdependencies)
 
@@ skipping 41 matching lines @@
   args = parser.parse_args()
 
   if args.quiet:
     logging.disable(logging.INFO)
 
   repos = args.repos
   if not len(repos):
     repos = [os.path.dirname(__file__)]
   for repo in repos:
     resolve_deps(repo)