Issue 1363 - Make sure source code archives created by release automation consider the new dependen…

releaseAutomation.py

 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os, re, codecs, subprocess, tarfile, json
-from StringIO import StringIO
 
 def get_dependencies(prefix, repos):
   from ensure_dependencies import read_deps, safe_join
   repo = repos[prefix]
   deps = read_deps(repo)
   if deps:
     for subpath in deps:
       if subpath.startswith('_'):
         continue
       depprefix = prefix + subpath + '/'
       deppath = safe_join(repo, subpath)
       repos[depprefix] = deppath
       get_dependencies(depprefix, repos)
 
 def create_sourcearchive(repo, output):
-  with open(output, 'wb') as handle:

[Sebastian Noack, 2014/09/18 16:20:06]

You can just pass the filename to tarfile.open() and save this line and a level
of indentation.

[Wladimir Palant, 2014/09/18 18:40:48]

Done, don't remember why the original code did it this way.

-    with tarfile.open(fileobj=handle, name=os.path.basename(output), mode='w:gz') as archive:
-      repos = {'': repo}
-      get_dependencies('', repos)
-      for prefix, path in repos.iteritems():
-        data = subprocess.check_output(['hg', 'archive', '-R', path, '-t', 'tar', '-S', '-'])
-        with tarfile.open(fileobj=StringIO(data), mode='r:') as repoarchive:

[Sebastian Noack, 2014/09/18 16:20:06]

I would prefer to directly pass stdout of the subprocess to the tarfile.open(),
instead loading the output into memory just to wrap it into a StringIO:

hg = subprocess.check_output(['hg', 'archive', ...], stdout=subprocess.PIPE)
try:
  with tarfile.open(fileobj=hg.stdout, mode='r:') as repoarchive:
    ...
finally:
  hg.stdout.close()
  hg.wait()

[Sebastian Noack, 2014/09/18 16:21:24]

hg = subprocess.Popen(['hg', 'archive', ...], stdout=subprocess.PIPE)

[Wladimir Palant, 2014/09/18 18:40:48]

Turns out that the ingredient I was missing here was mode='r|', without it
tarfile will try to seek the stream. Implemented without caching all the data
now.

I didn't bother waiting for the hg process however - what's the point if tarfile
will read until EOF anyway and an exception will terminate both processes?

[Sebastian Noack, 2014/09/18 18:58:12]

Killing zombies. ;)

Terminated processes stay around until their parent process terminates or
retrieved the exit code of the subprocess by calling wait.

So you should always wait for subprocesses, just like you always should close
file descriptors and release any other resource.

https://en.wikipedia.org/wiki/Zombie_process

[Wladimir Palant, 2014/09/18 19:23:57]

Done.

+  with tarfile.open(output, mode='w:gz') as archive:
+    repos = {'': repo}
+    get_dependencies('', repos)
+    for prefix, path in repos.iteritems():
+      process = subprocess.Popen(['hg', 'archive', '-R', path, '-t', 'tar', '-S', '-'], stdout=subprocess.PIPE)
+      try:
+        with tarfile.open(fileobj=process.stdout, mode='r|') as repoarchive:
           for fileinfo in repoarchive:
             if os.path.basename(fileinfo.name) in ('.hgtags', '.hgignore'):
               continue
             filedata = repoarchive.extractfile(fileinfo)
             fileinfo.name = re.sub(r'^[^/]+/', prefix, fileinfo.name)
             archive.addfile(fileinfo, filedata)
+      finally:
+        process.stdout.close()
+        process.wait()
 
 def run(baseDir, type, version, keyFiles, downloadsRepo):
   if type == "gecko":
     import buildtools.packagerGecko as packager
   elif type == "chrome":
     import buildtools.packagerChrome as packager
 
   # Replace version number in metadata file "manually", ConfigParser will mess
   # up the order of lines.
   metadata = packager.readMetadata(baseDir, type)
@@ skipping 58 matching lines @@
   create_sourcearchive(baseDir, archivePath)
   downloads.append(archivePath)
 
   # Now add the downloads and commit
   subprocess.check_call(['hg', 'add', '-R', downloadsRepo] + downloads)
   subprocess.check_call(['hg', 'commit', '-R', downloadsRepo, '-m', 'Releasing %s %s' % (extensionName, version)])
 
   # Push all changes
   subprocess.check_call(['hg', 'push', '-R', baseDir])
   subprocess.check_call(['hg', 'push', '-R', downloadsRepo])