Issue 356 - Update devbuild in the Chrome Web Store

sitescripts/extensions/bin/createNightlies.py

Index: sitescripts/extensions/bin/createNightlies.py
===================================================================
--- a/sitescripts/extensions/bin/createNightlies.py
+++ b/sitescripts/extensions/bin/createNightlies.py
@@ -26,8 +26,10 @@
 """
 
 import sys, os, os.path, codecs, subprocess, ConfigParser, traceback, json, hashlib
-import tempfile, re, shutil, urlparse, pipes
+import tempfile, re, shutil, urlparse, pipes, time, base64
 from datetime import datetime
+from urllib import urlencode
+from urllib2 import urlopen
 from xml.dom.minidom import parse as parseXml
 from sitescripts.utils import get_config, setupStderr, get_template
 from sitescripts.extensions.utils import compareVersions, Configuration
@@ -286,11 +288,11 @@
     if not os.path.exists(baseDir):
       os.makedirs(baseDir)
     outputFile = "%s-%s%s" % (self.basename, self.version, self.config.packageSuffix)
-    outputPath = os.path.join(baseDir, outputFile)
+    self.path = os.path.join(baseDir, outputFile)
     self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + outputFile + '?update')
 
     if self.config.type == 'android':
-      apkFile = open(outputPath, 'wb')
+      apkFile = open(self.path, 'wb')
 
       try:
         try:
@@ -302,29 +304,29 @@
         subprocess.check_call(buildCommand, stdout=apkFile, close_fds=True)
       except:
         # clear broken output if any
-        if os.path.exists(outputPath):
-          os.remove(outputPath)
+        if os.path.exists(self.path):
+          os.remove(self.path)
         raise
     elif self.config.type == 'chrome' or self.config.type == 'opera':
       import buildtools.packagerChrome as packager
-      packager.createBuild(self.tempdir, type=self.config.type, outFile=outputPath, buildNum=self.revision, keyFile=self.config.keyFile, experimentalAPI=self.config.experimental)
+      packager.createBuild(self.tempdir, type=self.config.type, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile, experimentalAPI=self.config.experimental)
     elif self.config.type == 'safari':
       import buildtools.packagerSafari as packager
-      packager.createBuild(self.tempdir, type=self.config.type, outFile=outputPath, buildNum=self.revision, keyFile=self.config.keyFile)
+      packager.createBuild(self.tempdir, type=self.config.type, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
     else:
       import buildtools.packagerGecko as packager
-      packager.createBuild(self.tempdir, outFile=outputPath, buildNum=self.revision, keyFile=self.config.keyFile)
+      packager.createBuild(self.tempdir, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
 
-    if not os.path.exists(outputPath):
+    if not os.path.exists(self.path):
       raise Exception("Build failed, output file hasn't been created")
 
     linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
     if hasattr(os, 'symlink'):
       if os.path.exists(linkPath):
         os.remove(linkPath)
-      os.symlink(os.path.basename(outputPath), linkPath)
+      os.symlink(os.path.basename(self.path), linkPath)
     else:
-      shutil.copyfile(outputPath, linkPath)
+      shutil.copyfile(self.path, linkPath)
 
   def retireBuilds(self):
     """
@@ -392,6 +394,57 @@
     finally:
       shutil.rmtree(docsdir, ignore_errors=True)
 
+  def uploadToChromeWebStore(self):
+    import M2Crypto
+
+    # log in with the Google Developer Service Account
+    # https://developers.google.com/accounts/docs/OAuth2ServiceAccount#creatingjwt
+
+    base64encode = lambda data: base64.urlsafe_b64encode(data).rstrip('=')
+    now = int(time.mktime(time.gmtime()))
+
+    jwt = '%s.%s' % (
+      base64encode(json.dumps({
+        'alg': 'RS256',
+        'typ': 'JWT',
+      })),
+      base64encode(json.dumps({
+         'iss': self.config.serviceAccountEmail,
+         'scope': 'https://www.googleapis.com/auth/chromewebstore',
+         'aud':'https://accounts.google.com/o/oauth2/token',
+         'exp': now + 3600, # fails if less than about an hour
+         'iat': now,
+      })),
+    )
+
+    jwt = '%s.%s' % (jwt, base64encode(
+      M2Crypto.RSA.load_key(self.config.serviceAccountKey).sign(
+        hashlib.sha256(jwt).digest(), 'sha256'
+      )
+    ))
+
+    response = json.load(urlopen(
+      'https://accounts.google.com/o/oauth2/token',
+
+      urlencode([
+        ('grant_type', 'urn:ietf:params:oauth:grant-type:jwt-bearer'),
+        ('assertion', jwt),
+      ])
+    ))

[Wladimir Palant, 2014/04/20 06:47:53]

Check for error responses?

[Sebastian Noack, 2014/04/20 16:03:08]

This API uses standard HTTP  error codes, which are automatically converted into
Python exceptions by urllib2.

+
+    # upload a new version with the Chrome Web Store API
+    # https://developer.chrome.com/webstore/using_webstore_api#uploadexisitng
+
+    subprocess.check_call([

[Sebastian Noack, 2014/04/19 16:15:34]

I use curl instead of urllib here, because urllib doesn't support file uploads.

[Wladimir Palant, 2014/04/20 06:47:53]

We do file uploads with urllib in buildtools/localeTools.py - not pretty but
better than calling an external application.

[Sebastian Noack, 2014/04/20 16:03:08]

It was actually easier to use urllib2 as I thought, since the API doesn't
actually require a multipart request.

+      'curl', 'https://www.googleapis.com/upload/chromewebstore/v1.1/items/' + self.config.galleryID
+
+      '--header', 'Authorization: %s %s' % (response['token_type'], response['access_token'])
+      '--header', 'x-goog-api-version: 2',
+      '--request',  'PUT',
+      '--upload-file', self.path,
+      '--silent',
+    ])

[Wladimir Palant, 2014/04/20 06:47:53]

Here as well - please check for error responses, if something goes wrong we
should know.

[Sebastian Noack, 2014/04/20 16:03:08]

Done.

+
   def run(self):
     """
       Run the nightly build process for one extension
@@ -438,6 +491,9 @@
 
       # update nightlies config
       self.config.latestRevision = self.revision
+
+      if self.config.type == 'chrome':

[Wladimir Palant, 2014/04/20 06:47:53]

This step needs to be optional, only if the necessary config options are present
- it should be possible to test things locally without triggering an upload
every time.

[Sebastian Noack, 2014/04/20 16:03:08]

Done.

+        self.uploadToChromeWebStore()
     finally:
       # clean up
       if self.tempdir: