Crawler backend

sitescripts/extensions/bin/createNightlies.py

 # coding: utf-8
 
 # This Source Code is subject to the terms of the Mozilla Public License
 # version 2.0 (the "License"). You can obtain a copy of the License at
 # http://mozilla.org/MPL/2.0/.
 
 """
 
 Nightly builds generation script
 ================================
 
   This script generates nightly builds of extensions, together
   with changelogs and documentation.
 
 """
 
 import sys, os, os.path, subprocess, ConfigParser, traceback, json, hashlib
-import tempfile, re, shutil, urlparse
+import tempfile, re, shutil, urlparse, pipes
 from datetime import datetime
+from xml.dom.minidom import parse as parseXml
 from sitescripts.utils import get_config, setupStderr, get_template
 from sitescripts.extensions.utils import compareVersions, Configuration
 import buildtools.packager as packager
 
 MAX_BUILDS = 50
 
 
 class NightlyBuild(object):
   """
     Performs the build process for an extension,
@@ skipping 80 matching lines @@
     metadata = packager.readMetadata(self.tempdir)
     self.extensionID = metadata.get("general", "id")
     self.version = '%s.%s' % (metadata.get("general", "version"), self.revision)
     self.basename = metadata.get("general", "basename")
     self.compat = []
     for key, value in packager.KNOWN_APPS.iteritems():
       if metadata.has_option('compat', key):
         minVersion, maxVersion = metadata.get('compat', key).split('/')
         self.compat.append({'id': value, 'minVersion': minVersion, 'maxVersion': maxVersion})
 
+  def readAndroidMetadata(self):
+    """
+      Read Android-specific metadata from AndroidManifest.xml file.
+    """
+    manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), 'r')
+    manifest = parseXml(manifestFile)
+    manifestFile.close()
+
+    root = manifest.documentElement
+    self.version = root.attributes["android:versionName"].value
+    while self.version.count('.') < 2:
+      self.version += '.0'
+    self.version = '%s.%s' % (self.version, self.revision)
+
+    usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
+    self.minSdkVersion = usesSdk.attributes["android:minSdkVersion"].value
+    self.basename = os.path.basename(self.config.repository)
+
   def readChromeMetadata(self):
     """
       Read Chrome-specific metadata from manifest.json file. It will also
       calculate extension ID from the private key.
     """
 
     # Calculate extension ID from public key
     # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-from-your-private-key-233)
     sys.path.append(self.tempdir)
     build = __import__('build')
@@ skipping 20 matching lines @@
     if 'minimum_chrome_version' in manifest:
       self.compat.append({'id': 'chrome', 'minVersion': manifest['minimum_chrome_version']})
 
   def writeUpdateManifest(self):
     """
       Writes update.rdf file for the current build
     """
     baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
     if not os.path.exists(baseDir):
       os.makedirs(baseDir)
-    if self.config.type != 'chrome':
+    if self.config.type == 'chrome':
+      manifestPath = os.path.join(baseDir, "updates.xml")
+      templateName = 'chromeUpdateManifest'
+    elif self.config.type == 'android':
+      manifestPath = os.path.join(baseDir, "updates.xml")
+      templateName = 'androidUpdateManifest'
+    else:
       manifestPath = os.path.join(baseDir, "update.rdf")
       templateName = 'geckoUpdateManifest'
-    else:
-      manifestPath = os.path.join(baseDir, "updates.xml")
-      templateName = 'chromeUpdateManifest'
 
     template = get_template(get_config().get('extensions', templateName))
     template.stream({'extensions': [self]}).dump(manifestPath)
 
   def build(self):
     """
       run the build command in the tempdir
     """
     baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
     if not os.path.exists(baseDir):
       os.makedirs(baseDir)
     outputFile = "%s-%s%s" % (self.basename, self.version, self.config.packageSuffix)
     outputPath = os.path.join(baseDir, outputFile)
     self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + outputFile + '?update')
 
-    if self.config.type != 'chrome':
-      packager.createBuild(self.tempdir, outFile=outputPath, buildNum=self.revision, keyFile=self.config.keyFile)
-    else:
+    if self.config.type == 'android':
+      apkFile = open(outputPath, 'wb')
+      try:
+        port = get_config().get('extensions', 'androidBuildPort')
+      except ConfigParser.NoOptionError:
+        port = '22'
+      buildCommand = ['ssh', '-p', port, get_config().get('extensions', 'androidBuildHost')]
+      buildCommand += map(pipes.quote, ['/home/android/bin/makedebugbuild.py', '--revision', self.revision, '--version', self.version, '--stdout'])
+      process = subprocess.Popen(buildCommand, stdout=apkFile, stderr=None)
+      status = process.wait()
+      apkFile.close()
+      if status:
+        # clear broken output if any
+        # exception will be raised later
+        if os.path.exists(outputPath):
+          os.remove(outputPath)
+    elif self.config.type == 'chrome':
       buildCommand = ['python', os.path.join(self.tempdir, 'build.py'), '-k', self.config.keyFile, '-b', self.revision, outputPath]
       if self.config.experimental:
         buildCommand[-1:0] = ['--experimental']
       subprocess.Popen(buildCommand, stdout=subprocess.PIPE).communicate()
+    else:
+      packager.createBuild(self.tempdir, outFile=outputPath, buildNum=self.revision, keyFile=self.config.keyFile)
 
     if not os.path.exists(outputPath):
       raise Exception("Build failed, output file hasn't been created")
 
     linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
     if hasattr(os, 'symlink'):
       if os.path.exists(linkPath):
         os.remove(linkPath)
       os.symlink(outputPath, linkPath)
     else:
@@ skipping 70 matching lines @@
     try:
       if self.config.type == 'kmeleon':
         # We cannot build K-Meleon builds, simply list the builds already in
         # the directory. Basename has to be deduced from the repository name.
         self.basename = os.path.basename(self.config.repository)
       else:
         # copy the repository into a temporary directory
         self.copyRepository()
 
         # get meta data from the repository
-        if self.config.type != 'chrome':
+        if self.config.type == 'android':
+          self.readAndroidMetadata()
+        elif self.config.type == 'chrome':
+          self.readChromeMetadata()
+        else:
           self.readMetadata()
-        else:
-          self.readChromeMetadata()
 
         # create development build
         self.build()
 
         # write out changelog
         self.writeChangelog(self.getChanges())
 
         # write update.rdf file
         self.writeUpdateManifest()
 
@@ skipping 37 matching lines @@
     except Exception, ex:
       print >>sys.stderr, "The build for %s failed:" % repo
       traceback.print_exc()
 
   file = open(nightlyConfigFile, 'wb')
   nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
   main()