Issue 356 - Update devbuild in the Chrome Web Store

sitescripts/extensions/bin/createNightlies.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2013 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 
 Nightly builds generation script
 ================================
 
   This script generates nightly builds of extensions, together
   with changelogs and documentation.
 
 """
 
 import sys, os, os.path, codecs, subprocess, ConfigParser, traceback, json, hashlib
-import tempfile, re, shutil, urlparse, pipes
+import tempfile, re, shutil, urlparse, pipes, time, urllib2, struct
 from datetime import datetime
+from urllib import urlencode
 from xml.dom.minidom import parse as parseXml
 from sitescripts.utils import get_config, setupStderr, get_template
 from sitescripts.extensions.utils import compareVersions, Configuration
 
 MAX_BUILDS = 50
 
 
 class NightlyBuild(object):
   """
     Performs the build process for an extension,
@@ skipping 238 matching lines @@
 
 
   def build(self):
     """
       run the build command in the tempdir
     """
     baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
     if not os.path.exists(baseDir):
       os.makedirs(baseDir)
     outputFile = "%s-%s%s" % (self.basename, self.version, self.config.packageSuffix)
-    outputPath = os.path.join(baseDir, outputFile)
+    self.path = os.path.join(baseDir, outputFile)
     self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + outputFile + '?update')
 
     if self.config.type == 'android':
-      apkFile = open(outputPath, 'wb')
+      apkFile = open(self.path, 'wb')
 
       try:
         try:
           port = get_config().get('extensions', 'androidBuildPort')
         except ConfigParser.NoOptionError:
           port = '22'
         buildCommand = ['ssh', '-p', port, get_config().get('extensions', 'androidBuildHost')]
         buildCommand.extend(map(pipes.quote, ['/home/android/bin/makedebugbuild.py', '--revision', self.revision, '--version', self.version, '--stdout']))
         subprocess.check_call(buildCommand, stdout=apkFile, close_fds=True)
       except:
         # clear broken output if any
-        if os.path.exists(outputPath):
-          os.remove(outputPath)
+        if os.path.exists(self.path):
+          os.remove(self.path)
         raise
     elif self.config.type == 'chrome' or self.config.type == 'opera':
       import buildtools.packagerChrome as packager
-      packager.createBuild(self.tempdir, type=self.config.type, outFile=outputPath, buildNum=self.revision, keyFile=self.config.keyFile, experimentalAPI=self.config.experimental)
+      packager.createBuild(self.tempdir, type=self.config.type, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile, experimentalAPI=self.config.experimental)
     elif self.config.type == 'safari':
       import buildtools.packagerSafari as packager
-      packager.createBuild(self.tempdir, type=self.config.type, outFile=outputPath, buildNum=self.revision, keyFile=self.config.keyFile)
+      packager.createBuild(self.tempdir, type=self.config.type, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
     else:
       import buildtools.packagerGecko as packager
-      packager.createBuild(self.tempdir, outFile=outputPath, buildNum=self.revision, keyFile=self.config.keyFile)
+      packager.createBuild(self.tempdir, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
 
-    if not os.path.exists(outputPath):
+    if not os.path.exists(self.path):
       raise Exception("Build failed, output file hasn't been created")
 
     linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
     if hasattr(os, 'symlink'):
       if os.path.exists(linkPath):
         os.remove(linkPath)
-      os.symlink(os.path.basename(outputPath), linkPath)
+      os.symlink(os.path.basename(self.path), linkPath)
     else:
-      shutil.copyfile(outputPath, linkPath)
+      shutil.copyfile(self.path, linkPath)
 
   def retireBuilds(self):
     """
       removes outdated builds, returns the sorted version numbers of remaining
       builds
     """
     baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
     versions = []
     prefix = self.basename + '-'
     suffix = self.config.packageSuffix
@@ skipping 47 matching lines @@
     command = ['hg', 'archive', '-R', get_config().get('extensions', 'jsdocRepository'), '-r', 'default', docsdir]
     subprocess.check_call(command)
 
     try:
       import buildtools.build as build
       outputPath = os.path.join(self.config.docsDirectory, self.basename)
       build.generateDocs(self.tempdir, None, [("-t", docsdir), ("-q", "")], [outputPath], self.config.type)
     finally:
       shutil.rmtree(docsdir, ignore_errors=True)
 
+  def uploadToChromeWebStore(self):
+    # use refresh token to obtain a valid access token
+    # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh
+
+    response = json.load(urllib2.urlopen(
+      'https://accounts.google.com/o/oauth2/token',
+
+      urlencode([
+        ('refresh_token', self.config.refreshToken),
+        ('client_id', self.config.clientID),
+        ('client_secret', self.config.clientSecret),
+        ('grant_type', 'refresh_token'),
+      ])
+    ))
+
+    # upload a new version with the Chrome Web Store API
+    # https://developer.chrome.com/webstore/using_webstore_api#uploadexisitng
+
+    request = urllib2.Request('https://www.googleapis.com/upload/chromewebstore/v1.1/items/' + self.config.galleryID)
+    request.get_method = lambda: 'PUT'
+    request.add_header('Authorization', '%s %s' % (response['token_type'], response['access_token']))
+    request.add_header('x-goog-api-version', '2')
+
+    with open(self.path, 'rb') as file:
+      # skip CRX magic and version
+      offset = 8
+      file.seek(offset)
+
+      # skip public key and signature
+      offset += 8 + sum(struct.unpack('<II', file.read(8)))

[Wladimir Palant, 2014/04/21 20:22:56]

Please don't skip magic number and version - read them to verify that they are
what you expect, otherwise you should raise an exception (particularly important
if CRX version changes).

[Sebastian Noack, 2014/04/21 20:46:36]

Done.

+      file.seek(offset)

[Wladimir Palant, 2014/04/21 20:22:56]

Please use os.SEEK_CUR as second parameter to file.seek(), then you won't need a
complicated offset calculation - just a sum of the two lengths. For
Content-Length below you can use file.tell().

[Sebastian Noack, 2014/04/21 20:46:36]

Done.

+
+      request.add_header('Content-Length', os.fstat(file.fileno()).st_size - offset)
+      request.add_data(file)
+
+      response = json.load(urllib2.urlopen(request))
+
+    if response['uploadState'] == 'FAILURE':
+      raise Exception(response['itemError'])
+
   def run(self):
     """
       Run the nightly build process for one extension
     """
     try:
       if self.config.type == 'ie':
         # We cannot build IE builds, simply list the builds already in
         # the directory. Basename has to be deduced from the repository name.
         self.basename = os.path.basename(self.config.repository)
       else:
@@ skipping 26 matching lines @@
       versions = self.retireBuilds()
 
       if self.config.type == 'ie':
         self.writeIEUpdateManifest(versions)
 
       # update index page
       self.updateIndex(versions)
 
       # update nightlies config
       self.config.latestRevision = self.revision
+
+      if self.config.clientID and self.config.clientSecret and self.config.refreshToken:

[Wladimir Palant, 2014/04/21 20:22:56]

I think it makes sense to keep the type == 'chrome' check - it makes no sense to
attempt CWS upload for non-Chrome extensions, even if the necessary parameters
are configured. Note that these parameters aren't explicitly CWS-related, we
might theoretically decide to use the same set of parameters with other
extension stores (unlikely of course).

[Sebastian Noack, 2014/04/21 20:46:36]

Done.

+        self.uploadToChromeWebStore()
     finally:
       # clean up
       if self.tempdir:
         shutil.rmtree(self.tempdir, ignore_errors=True)
 
 
 def main():
   """
     main function for createNightlies.py
   """
@@ skipping 16 matching lines @@
     except Exception, ex:
       print >>sys.stderr, "The build for %s failed:" % repo
       traceback.print_exc()
 
   file = open(nightlyConfigFile, 'wb')
   nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
   main()