Issue 2896 - Automate uploading of Firefox development builds to AMO

sitescripts/extensions/bin/createNightlies.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 
 Nightly builds generation script
 ================================
 
   This script generates nightly builds of extensions, together
   with changelogs and documentation.
 
 """
 
-import sys, os, os.path, subprocess, ConfigParser, json, hashlib

[Felix Dahlke, 2015/08/13 12:52:27]

Nit: Might be a good opportunity to clean the commits up (in regards to PEP-8),
but I wouldn't insist.

[Wladimir Palant, 2015/08/13 13:00:40]

As I said, I intentionally didn't clean up the imports - otherwise it would
obscure the actual changes. I did it in a follow-up commit now.

-import tempfile, shutil, urlparse, pipes, time, urllib2, struct
+import ConfigParser
 import cookielib
+from datetime import datetime
+import hashlib
 import HTMLParser
+import json
 import logging
-from datetime import datetime
+import os
+import pipes
+import shutil
+import struct
+import subprocess
+import sys
+import tempfile
+import time
 from urllib import urlencode
+import urllib2
+import urlparse
 from xml.dom.minidom import parse as parseXml
-from sitescripts.utils import get_config, get_template
+
 from sitescripts.extensions.utils import (
   compareVersions, Configuration,
   writeAndroidUpdateManifest
 )
+from sitescripts.utils import get_config, get_template
 
 MAX_BUILDS = 50
 
 
 class NightlyBuild(object):
   """
     Performs the build process for an extension,
     generating changelogs and documentation.
   """
 
@@ skipping 333 matching lines @@
     password = get_config().get('extensions', 'amo_password')
 
     slug = self.config.galleryID
     login_url= 'https://addons.mozilla.org/en-US/firefox/users/login'
     upload_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/upload' % slug
     add_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/versions/add' % slug
 
     cookie_jar = cookielib.CookieJar()
     opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar))
 
     def load_url(url, data=None):

[Sebastian Noack, 2015/08/12 13:29:26]

Any reason, why you don'T simply use urllib3.request, when relying on urllib3
anyway?

[Wladimir Palant, 2015/08/12 13:34:10]

Yes, urllib3 doesn't support cookie jars.

       content_type = 'application/x-www-form-urlencoded'
       if isinstance(data, dict):
         if any(isinstance(v, tuple) for v in data.itervalues()):
           data, content_type = urllib3.filepost.encode_multipart_formdata(data)
         else:
           data = urlencode(data.items())
 
       request = urllib2.Request(url, data, headers={'Content-Type': content_type})
       response = opener.open(request)
       try:
         return response.read()
       finally:
         response.close()
 
     class CSRFParser(HTMLParser.HTMLParser):

[Sebastian Noack, 2015/08/12 13:29:26]

Don't we already use minidom somewhere else? That would be simpler to use than
implementing a parser.

[Wladimir Palant, 2015/08/12 13:34:10]

Does minidom parse HTML? Not that I know.

[Felix Dahlke, 2015/08/13 12:52:27]

I've used BeautifulSoup for this kind of stuff. But as far as I'm concerned, we
can leave it as it is, we're just trying to get the CSRF token after all.

[Sebastian Noack, 2015/08/18 09:04:13]

Alright, minidom is only for XML, though it also works with some HTML input. But
what you actually would need is lxml which has proper (and quite error-tolerant)
support of HTML. Note that BeautifulSoup is just an obscure wrapper around lxml
(and some other libraries). So if we add a dependency here, I'd rather use lxml
directly. However, I'm also fine with with sticking to HTMLParser, which comes
with the Python corelib, rather than adding a new dependency.

       result = None
       dummy_exception = Exception()
 
       def __init__(self, data):
         HTMLParser.HTMLParser.__init__(self)
         try:
           self.feed(data)
           self.close()
         except Exception, e:
           if e != self.dummy_exception:
@@ skipping 33 matching lines @@
       upload_response = json.loads(load_url(
         upload_url,
         {
           'csrfmiddlewaretoken': csrf_token,
           'upload': (os.path.basename(self.path), file.read(), 'application/x-xpinstall'),
         }
       ))
 
     # Wait for validation to finish
     while not upload_response.get('validation'):
       time.sleep(2)

[Sebastian Noack, 2015/08/12 13:29:26]

I suppose there is no way to get notified when it's finished? :(

Can/Should we at least check whether the validation is still pending and retry
in case it takes more than 2 seconds?

[Wladimir Palant, 2015/08/12 13:34:10]

No, it's exactly how the web interface does it - by polling.
That's exactly what we do - it's a while loop. The good news is, there is a
server-side timeout, so this won't be running forever.

       upload_response = json.loads(load_url(
         upload_url + '/' + upload_response.get('upload')
       ))
 
     if upload_response['validation'].get('errors', 0):

[Sebastian Noack, 2015/08/12 13:29:26]

Nit: Omit the default value? None evaluates to False as well.

[Wladimir Palant, 2015/08/12 13:34:10]

Yes, I considered it but the default value indicates that we expect a number
here.

       raise Exception('Build failed AMO validation, see https://addons.mozilla.org%s' % upload_response.get('full_report_url'))
 
     # Add version
     add_response = json.loads(load_url(
       add_url,
       {
         'csrfmiddlewaretoken': csrf_token,
         'upload': upload_response.get('upload'),
         'source': ('', '', 'application/octet-stream'),
         'beta': 'on',
@@ skipping 86 matching lines @@
           self.readSafariMetadata()
         else:
           self.readGeckoMetadata()
 
         # create development build
         self.build()
 
         # write out changelog
         self.writeChangelog(self.getChanges())
 
-        # write update.rdf file
-        self.writeUpdateManifest()
+        # write update manifest
+        if self.config.type != 'gecko':
+          self.writeUpdateManifest()
 
         # update documentation
         self.updateDocs()
 
       # retire old builds
       versions = self.retireBuilds()
 
       if self.config.type == 'ie':
         self.writeIEUpdateManifest(versions)
 
@@ skipping 34 matching lines @@
     except Exception, ex:
       logging.error("The build for %s failed:", repo)
       logging.exception(ex)
 
   file = open(nightlyConfigFile, 'wb')
   nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
   main()