Issue 2896 - Automate uploading of Firefox development builds to AMO

sitescripts/extensions/bin/createNightlies.py

Index: sitescripts/extensions/bin/createNightlies.py
===================================================================
--- a/sitescripts/extensions/bin/createNightlies.py
+++ b/sitescripts/extensions/bin/createNightlies.py
@@ -20,22 +20,25 @@
 Nightly builds generation script
 ================================
 
   This script generates nightly builds of extensions, together
   with changelogs and documentation.
 
 """
 
-import sys, os, os.path, subprocess, ConfigParser, traceback, json, hashlib
+import sys, os, os.path, subprocess, ConfigParser, json, hashlib

[Felix Dahlke, 2015/08/13 12:52:27]

Nit: Might be a good opportunity to clean the commits up (in regards to PEP-8),
but I wouldn't insist.

[Wladimir Palant, 2015/08/13 13:00:40]

As I said, I intentionally didn't clean up the imports - otherwise it would
obscure the actual changes. I did it in a follow-up commit now.

 import tempfile, shutil, urlparse, pipes, time, urllib2, struct
+import cookielib
+import HTMLParser
+import logging
 from datetime import datetime
 from urllib import urlencode
 from xml.dom.minidom import parse as parseXml
-from sitescripts.utils import get_config, setupStderr, get_template
+from sitescripts.utils import get_config, get_template
 from sitescripts.extensions.utils import (
   compareVersions, Configuration,
   writeAndroidUpdateManifest
 )
 
 MAX_BUILDS = 50
 
 
@@ -47,16 +50,18 @@ class NightlyBuild(object):
 
   def __init__(self, config):
     """
       Creates a NightlyBuild instance; we are simply
       recording the configuration settings here.
     """
     self.config = config
     self.revision = self.getCurrentRevision()
+    if self.config.type == 'gecko':
+      self.revision += '-beta'
     try:
       self.previousRevision = config.latestRevision
     except:
       self.previousRevision = '0'
     self.tempdir = None
     self.outputFilename = None
     self.changelogFilename = None
 
@@ -367,16 +372,119 @@ class NightlyBuild(object):
   def updateDocs(self):
     if self.config.type not in ('gecko', 'chrome'):
       return
 
     import buildtools.build as build
     outputPath = os.path.join(self.config.docsDirectory, self.basename)
     build.generateDocs(self.tempdir, None, [('--quiet', '')], [outputPath], self.config.type)
 
+  def uploadToMozillaAddons(self):
+    import urllib3
+
+    username = get_config().get('extensions', 'amo_username')
+    password = get_config().get('extensions', 'amo_password')
+
+    slug = self.config.galleryID
+    login_url= 'https://addons.mozilla.org/en-US/firefox/users/login'
+    upload_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/upload' % slug
+    add_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/versions/add' % slug
+
+    cookie_jar = cookielib.CookieJar()
+    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar))
+
+    def load_url(url, data=None):

[Sebastian Noack, 2015/08/12 13:29:26]

Any reason, why you don'T simply use urllib3.request, when relying on urllib3
anyway?

[Wladimir Palant, 2015/08/12 13:34:10]

Yes, urllib3 doesn't support cookie jars.

+      content_type = 'application/x-www-form-urlencoded'
+      if isinstance(data, dict):
+        if any(isinstance(v, tuple) for v in data.itervalues()):
+          data, content_type = urllib3.filepost.encode_multipart_formdata(data)
+        else:
+          data = urlencode(data.items())
+
+      request = urllib2.Request(url, data, headers={'Content-Type': content_type})
+      response = opener.open(request)
+      try:
+        return response.read()
+      finally:
+        response.close()
+
+    class CSRFParser(HTMLParser.HTMLParser):

[Sebastian Noack, 2015/08/12 13:29:26]

Don't we already use minidom somewhere else? That would be simpler to use than
implementing a parser.

[Wladimir Palant, 2015/08/12 13:34:10]

Does minidom parse HTML? Not that I know.

[Felix Dahlke, 2015/08/13 12:52:27]

I've used BeautifulSoup for this kind of stuff. But as far as I'm concerned, we
can leave it as it is, we're just trying to get the CSRF token after all.

[Sebastian Noack, 2015/08/18 09:04:13]

Alright, minidom is only for XML, though it also works with some HTML input. But
what you actually would need is lxml which has proper (and quite error-tolerant)
support of HTML. Note that BeautifulSoup is just an obscure wrapper around lxml
(and some other libraries). So if we add a dependency here, I'd rather use lxml
directly. However, I'm also fine with with sticking to HTMLParser, which comes
with the Python corelib, rather than adding a new dependency.

+      result = None
+      dummy_exception = Exception()
+
+      def __init__(self, data):
+        HTMLParser.HTMLParser.__init__(self)
+        try:
+          self.feed(data)
+          self.close()
+        except Exception, e:
+          if e != self.dummy_exception:
+            raise
+
+        if not self.result:
+          raise Exception('Failed to extract CSRF token')
+
+      def set_result(self, value):
+        self.result = value
+        raise self.dummy_exception
+
+      def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        if tag == 'meta' and attrs.get('name') == 'csrf':
+          self.set_result(attrs.get('content'))
+        if tag == 'input' and attrs.get('name') == 'csrfmiddlewaretoken':
+          self.set_result(attrs.get('value'))
+
+    # Extract anonymous CSRF token
+    login_page = load_url(login_url)
+    csrf_token = CSRFParser(login_page).result
+
+    # Log in and get session's CSRF token
+    main_page = load_url(
+      login_url,
+      {
+        'csrfmiddlewaretoken': csrf_token,
+        'username': username,
+        'password': password,
+      }
+    )
+    csrf_token = CSRFParser(main_page).result
+
+    # Upload build
+    with open(self.path, 'rb') as file:
+      upload_response = json.loads(load_url(
+        upload_url,
+        {
+          'csrfmiddlewaretoken': csrf_token,
+          'upload': (os.path.basename(self.path), file.read(), 'application/x-xpinstall'),
+        }
+      ))
+
+    # Wait for validation to finish
+    while not upload_response.get('validation'):
+      time.sleep(2)

[Sebastian Noack, 2015/08/12 13:29:26]

I suppose there is no way to get notified when it's finished? :(

Can/Should we at least check whether the validation is still pending and retry
in case it takes more than 2 seconds?

[Wladimir Palant, 2015/08/12 13:34:10]

No, it's exactly how the web interface does it - by polling.
That's exactly what we do - it's a while loop. The good news is, there is a
server-side timeout, so this won't be running forever.

+      upload_response = json.loads(load_url(
+        upload_url + '/' + upload_response.get('upload')
+      ))
+
+    if upload_response['validation'].get('errors', 0):

[Sebastian Noack, 2015/08/12 13:29:26]

Nit: Omit the default value? None evaluates to False as well.

[Wladimir Palant, 2015/08/12 13:34:10]

Yes, I considered it but the default value indicates that we expect a number
here.

+      raise Exception('Build failed AMO validation, see https://addons.mozilla.org%s' % upload_response.get('full_report_url'))
+
+    # Add version
+    add_response = json.loads(load_url(
+      add_url,
+      {
+        'csrfmiddlewaretoken': csrf_token,
+        'upload': upload_response.get('upload'),
+        'source': ('', '', 'application/octet-stream'),
+        'beta': 'on',
+        'supported_platforms': 1,       # PLATFORM_ANY.id
+      }
+    ))
+
   def uploadToChromeWebStore(self):
     # Google APIs use HTTP error codes with error message in body. So we add
     # the response body to the HTTPError to get more meaningful error messages.
 
     class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
       def http_error_default(self, req, fp, code, msg, hdrs):
         raise urllib2.HTTPError(req.get_full_url(), code, '%s\n%s' % (msg, fp.read()), hdrs, fp)
 
@@ -477,46 +585,46 @@ class NightlyBuild(object):
         self.writeIEUpdateManifest(versions)
 
       # update index page
       self.updateIndex(versions)
 
       # update nightlies config
       self.config.latestRevision = self.revision
 
-      if self.config.type == 'chrome' and self.config.clientID and self.config.clientSecret and self.config.refreshToken:
+      if self.config.type == 'gecko' and self.config.galleryID and get_config().get('extensions', 'amo_username'):
+        self.uploadToMozillaAddons()
+      elif self.config.type == 'chrome' and self.config.clientID and self.config.clientSecret and self.config.refreshToken:
         self.uploadToChromeWebStore()
     finally:
       # clean up
       if self.tempdir:
         shutil.rmtree(self.tempdir, ignore_errors=True)
 
 
 def main():
   """
     main function for createNightlies.py
   """
-  setupStderr()
-
   nightlyConfig = ConfigParser.SafeConfigParser()
   nightlyConfigFile = get_config().get('extensions', 'nightliesData')
   if os.path.exists(nightlyConfigFile):
     nightlyConfig.read(nightlyConfigFile)
 
   # build all extensions specified in the configuration file
   # and generate changelogs and documentations for each:
   data = None
   for repo in Configuration.getRepositoryConfigurations(nightlyConfig):
     build = None
     try:
       build = NightlyBuild(repo)
       if build.hasChanges():
         build.run()
     except Exception, ex:
-      print >>sys.stderr, "The build for %s failed:" % repo
-      traceback.print_exc()
+      logging.error("The build for %s failed:", repo)
+      logging.exception(ex)
 
   file = open(nightlyConfigFile, 'wb')
   nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
   main()