Issue 1363 - Make sure source code archives created by release automation consider the new dependen…

releaseAutomation.py

 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os, re, codecs, subprocess, tarfile, json
 from StringIO import StringIO
 
+def get_dependencies(prefix, repos):
+  from ensure_dependencies import read_deps, safe_join
+  repo = repos[prefix]
+  deps = read_deps(repo)
+  if deps:
+    for subpath in deps:
+      if subpath.startswith('_'):
+        continue
+      depprefix = prefix + subpath + '/'
+      deppath = safe_join(repo, subpath)
+      repos[depprefix] = deppath
+      get_dependencies(depprefix, repos)
+
+def create_sourcearchive(repo, output):
+  with open(output, 'wb') as handle:

[Sebastian Noack, 2014/09/18 16:20:06]

You can just pass the filename to tarfile.open() and save this line and a level
of indentation.

[Wladimir Palant, 2014/09/18 18:40:48]

Done, don't remember why the original code did it this way.

+    with tarfile.open(fileobj=handle, name=os.path.basename(output), mode='w:gz') as archive:
+      repos = {'': repo}
+      get_dependencies('', repos)
+      for prefix, path in repos.iteritems():
+        data = subprocess.check_output(['hg', 'archive', '-R', path, '-t', 'tar', '-S', '-'])
+        with tarfile.open(fileobj=StringIO(data), mode='r:') as repoarchive:

[Sebastian Noack, 2014/09/18 16:20:06]

I would prefer to directly pass stdout of the subprocess to the tarfile.open(),
instead loading the output into memory just to wrap it into a StringIO:

hg = subprocess.check_output(['hg', 'archive', ...], stdout=subprocess.PIPE)
try:
  with tarfile.open(fileobj=hg.stdout, mode='r:') as repoarchive:
    ...
finally:
  hg.stdout.close()
  hg.wait()

[Sebastian Noack, 2014/09/18 16:21:24]

hg = subprocess.Popen(['hg', 'archive', ...], stdout=subprocess.PIPE)

[Wladimir Palant, 2014/09/18 18:40:48]

Turns out that the ingredient I was missing here was mode='r|', without it
tarfile will try to seek the stream. Implemented without caching all the data
now.

I didn't bother waiting for the hg process however - what's the point if tarfile
will read until EOF anyway and an exception will terminate both processes?

[Sebastian Noack, 2014/09/18 18:58:12]

Killing zombies. ;)

Terminated processes stay around until their parent process terminates or
retrieved the exit code of the subprocess by calling wait.

So you should always wait for subprocesses, just like you always should close
file descriptors and release any other resource.

https://en.wikipedia.org/wiki/Zombie_process

[Wladimir Palant, 2014/09/18 19:23:57]

Done.

+          for fileinfo in repoarchive:
+            if os.path.basename(fileinfo.name) in ('.hgtags', '.hgignore'):
+              continue
+            filedata = repoarchive.extractfile(fileinfo)
+            fileinfo.name = re.sub(r'^[^/]+/', prefix, fileinfo.name)
+            archive.addfile(fileinfo, filedata)
+
 def run(baseDir, type, version, keyFiles, downloadsRepo):
   if type == "gecko":
     import buildtools.packagerGecko as packager
   elif type == "chrome":
     import buildtools.packagerChrome as packager
 
   # Replace version number in metadata file "manually", ConfigParser will mess
   # up the order of lines.
   metadata = packager.readMetadata(baseDir, type)
   with open(metadata.option_source("general", "version"), 'r+b') as file:
@@ skipping 47 matching lines @@
     downloads.append(buildPathOpera)
 
     import buildtools.packagerSafari as packagerSafari
     metadataSafari = packagerSafari.readMetadata(baseDir, "safari")
     buildPathSafari = os.path.join(downloadsRepo, packagerSafari.getDefaultFileName(baseDir, metadataSafari, version, 'safariextz'))
     packagerSafari.createBuild(baseDir, type="safari", outFile=buildPathSafari, releaseBuild=True, keyFile=keyFiles[1])
     downloads.append(buildPathSafari)
 
   # Create source archive
   archivePath = os.path.splitext(buildPath)[0] + '-source.tgz'
-
-  archiveHandle = open(archivePath, 'wb')
-  archive = tarfile.open(fileobj=archiveHandle, name=os.path.basename(archivePath), mode='w:gz')
-  data = subprocess.check_output(['hg', 'archive', '-R', baseDir, '-t', 'tar', '-S', '-'])
-  repoArchive = tarfile.open(fileobj=StringIO(data), mode='r:')
-  for fileInfo in repoArchive:
-    if os.path.basename(fileInfo.name) in ('.hgtags', '.hgignore'):
-      continue
-    fileData = repoArchive.extractfile(fileInfo)
-    fileInfo.name = re.sub(r'^[^/]+/', '', fileInfo.name)
-    archive.addfile(fileInfo, fileData)
-  repoArchive.close()
-  archive.close()
-  archiveHandle.close()
+  create_sourcearchive(baseDir, archivePath)
   downloads.append(archivePath)
 
   # Now add the downloads and commit
   subprocess.check_call(['hg', 'add', '-R', downloadsRepo] + downloads)
   subprocess.check_call(['hg', 'commit', '-R', downloadsRepo, '-m', 'Releasing %s %s' % (extensionName, version)])
 
   # Push all changes
   subprocess.check_call(['hg', 'push', '-R', baseDir])
   subprocess.check_call(['hg', 'push', '-R', downloadsRepo])