Issue 3194 - Allow multiple sources for a dependency

ensure_dependencies.py

 #!/usr/bin/env python
 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 import os
 import posixpath
@@ skipping 16 matching lines @@
   # File to update this script from (optional)
   _self = buildtools/ensure_dependencies.py
   # Clone elemhidehelper repository into extensions/elemhidehelper directory at
   # tag "1.2".
   extensions/elemhidehelper = elemhidehelper 1.2
   # Clone buildtools repository into buildtools directory at VCS-specific
   # revision IDs.
   buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
   # Clone the adblockplus repository into adblockplus directory, overwriting the
   # usual source URL for Git repository and specifying VCS specific revision IDs.
-  adblockplus = adblockplus hg:893426c6a6ab git:git@github.com:kzar/adblockplus.git@b2ffd52b
+  adblockplus = adblockplus hg:893426c6a6ab git:git@github.com:user/adblockplus.git@b2ffd52b
   # Clone the adblockpluschrome repository into the adblockpluschrome directory,
   # from a specific Git repository, specifying the revision ID.
-  adblockpluschrome = git:git@github.com:kzar/adblockpluschrome.git@1fad3a7
+  adblockpluschrome = git:git@github.com:user/adblockpluschrome.git@1fad3a7
 """
 
 SKIP_DEPENDENCY_UPDATES = os.environ.get(
   "SKIP_DEPENDENCY_UPDATES", ""
 ).lower() not in ("", "0", "false")
 
 class Mercurial():
   def istype(self, repodir):
     return os.path.exists(os.path.join(repodir, ".hg"))
 
@@ skipping 85 matching lines @@
       return "ssh://" + url.replace(":", "/", 1)
     return url
 
 repo_types = OrderedDict((
   ("hg", Mercurial()),
   ("git", Git()),
 ))
 
 # [vcs:]value
 item_regexp = re.compile(
-  "^(?:(" + "|".join(repo_types.keys()) +"):)?"

[Sebastian Noack, 2015/10/15 17:03:07]

Please use map(re.escape, ...), just in case.

[kzar, 2015/10/16 10:20:05]

Done.

+  "^(?:(" + "|".join(map(re.escape, repo_types.keys())) +"):)?"
   "(.+)$"
 )
 
 # [url@]rev
 source_regexp = re.compile(
   "^(?:(.*)@)?"
   "(.+)$"
 )
 
-def merge_tuples(tuple_1, tuple_2):
-  """Return tuple containing any truthy values from the suplied tuples
-
-  (None, 2), (1,)      => (1, 2)
-  None, (1, 2)         => (1, 2)
-  (1, 2), (3, 4)       => (3, 4)
+def merge_seqs(seq1, seq2):
+  """Return a list of any truthy values from the suplied sequences
+
+  (None, 2), (1,)      => [1, 2]
+  None, (1, 2)         => [1, 2]
+  (1, 2), (3, 4)       => [3, 4]
   """
-  return tuple(i2 or i1 for i1, i2 in map(None, tuple_1 or (), tuple_2 or ()))

[Sebastian Noack, 2015/10/15 17:03:07]

While already using map, why not passing a function, but wrapping the result?

def merge_tuples(seq1, seq2):
  return map(lambda item1, item2: item2 or item1, seq1 or (), seq2 or ())

Also I'm not happy about the function names. The logic isn'T specifc to tuples,
neither does that name give any hint how these are merged. Not sure though
whether there is a reasonable name to address the latter.

[kzar, 2015/10/16 10:20:05]

Good points and I agree that the function's name doesn't explain it's
functionality too well. Only thing I could think to do was to add the
documentation string, hopefully that will be enough to make this clear to the
next person.
Done.

+  return map(lambda item1, item2: item2 or item1, seq1 or (), seq2 or ())
 
 def parse_spec(path, line):
   if "=" not in line:
     logging.warning("Invalid line in file %s: %s" % (path, line))
     return None, None
 
   key, value = line.split("=", 1)
   key = key.strip()
   items = value.split()
   if not len(items):
     logging.warning("No value specified for key %s in file %s" % (key, path))
     return key, None
 
   result = OrderedDict()
   is_dependency_field = not key.startswith("_")
 
   for i, item in enumerate(items):
     try:
       vcs, value = re.search(item_regexp, item).groups()
       vcs = vcs or "*"
       if is_dependency_field:
         if i == 0 and vcs == "*":
           # In order to be backwards compatible we have to assume that the first
           # source contains only a URL/path for the repo if it does not contain
           # the VCS part
           url_rev = (value, None)
         else:
           url_rev = re.search(source_regexp, value).groups()
-        result[vcs] = merge_tuples(result.get(vcs), url_rev)
+        result[vcs] = merge_seqs(result.get(vcs), url_rev)
       else:
         if vcs in result:
           logging.warning("Ignoring duplicate value for type %r "
                           "(key %r in file %r)" % (vcs, key, path))
         result[vcs] = value
     except AttributeError:
       logging.warning("Ignoring invalid item %r for type %r "
                       "(key %r in file %r)" % (item, vcs, key, path))
       continue
   return key, result
@@ skipping 97 matching lines @@
         skipdependencies.intersection([s[0] for s in sources if s[0]])):
       continue
 
     target = safe_join(repodir, dir)
     parenttype = get_repo_type(repodir)
     _root = config.get("_root", {})
 
     for key in sources.keys() + _root.keys():
       if key == parenttype or key is None and vcs != "*":
         vcs = key
-    source, rev = merge_tuples(sources.get("*"), sources.get(vcs))
+    source, rev = merge_seqs(sources.get("*"), sources.get(vcs))
 
     if not (vcs and source and rev):
       logging.warning("No valid source / revision found to create %s" % target)
       continue
 
     ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ""), source)
     update_repo(target, vcs, rev)
     resolve_deps(target, level + 1, self_update=False,
                  overrideroots=overrideroots, skipdependencies=skipdependencies)
 
@@ skipping 41 matching lines @@
   args = parser.parse_args()
 
   if args.quiet:
     logging.disable(logging.INFO)
 
   repos = args.repos
   if not len(repos):
     repos = [os.path.dirname(__file__)]
   for repo in repos:
     resolve_deps(repo)