Issue 4419 - Use proper API in order to upload devbuilds to AMO

sitescripts/extensions/bin/createNightlies.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2016 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 
 Nightly builds generation script
 ================================
 
   This script generates nightly builds of extensions, together
   with changelogs and documentation.
 
 """
 
 import ConfigParser
-import cookielib
+import base64
 from datetime import datetime
 import hashlib
-import HTMLParser
+import hmac
 import json
 import logging
 import os
 import pipes
+import random
 import shutil
 import struct
 import subprocess
 import sys
 import tempfile
 import time
 from urllib import urlencode
 import urllib2
 import urlparse
 from xml.dom.minidom import parse as parseXml
@@ skipping 339 matching lines @@
             }
             if os.path.exists(os.path.join(baseDir, changelogFile)):
                 link['changelog'] = changelogFile
             links.append(link)
         template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
         template.stream({'config': self.config, 'links': links}).dump(outputPath)
 
     def uploadToMozillaAddons(self):
         import urllib3
 
-        username = get_config().get('extensions', 'amo_username')
-        password = get_config().get('extensions', 'amo_password')
+        header = {
+            'alg': 'HS256',     # HMAC-SHA256
+            'typ': 'JWT',
+        }
 
-        slug = self.config.galleryID
-        login_url = 'https://addons.mozilla.org/en-US/firefox/users/login'
-        upload_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/upload' % slug
-        add_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/versions/add' % slug
+        issued = int(time.time())
+        payload = {
+            'iss': get_config().get('extensions', 'amo_key'),
+            'jti': random.random(),
+            'iat': issued,
+            'exp': issued + 60,
+        }
 
-        cookie_jar = cookielib.CookieJar()
-        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar))
+        input = '.'.join([

[Sebastian Noack, 2016/09/13 15:03:12]

'{}.{}'.format(...) seems more appropriate rather than using a list literal
(with fixed amount of elements) just for the matter of joining it. Same below.

[Wladimir Palant, 2016/09/13 15:16:54]

Done.

+            base64.b64encode(json.dumps(header)),
+            base64.b64encode(json.dumps(payload))
+        ])
 
-        def load_url(url, data=None):
-            content_type = 'application/x-www-form-urlencoded'
-            if isinstance(data, dict):
-                if any(isinstance(v, tuple) for v in data.itervalues()):
-                    data, content_type = urllib3.filepost.encode_multipart_formdata(data)
-                else:
-                    data = urlencode(data.items())
+        signature = hmac.new(get_config().get('extensions', 'amo_secret'),
+                             msg=input,
+                             digestmod=hashlib.sha256).digest()
+        token = '.'.join([input, base64.b64encode(signature)])
 
-            request = urllib2.Request(url, data, headers={'Content-Type': content_type})
-            response = opener.open(request)
-            try:
-                return response.read()
-            finally:
-                response.close()
+        upload_url = ('https://addons.mozilla.org/api/v3/addons/{0}/'

[Sebastian Noack, 2016/09/13 15:03:12]

Nit: Note that the indices in the placeholders are optional if you access them
in the order the values are given like you do here.

[Wladimir Palant, 2016/09/13 15:16:54]

Done.

+                      'versions/{1}/').format(self.extensionID, self.version)
 
-        class CSRFParser(HTMLParser.HTMLParser):
-            result = None
-            dummy_exception = Exception()
+        with open(self.path, 'rb') as file:
+            data, content_type = urllib3.filepost.encode_multipart_formdata({
+                'upload': (
+                    os.path.basename(self.path),
+                    file.read(),
+                    'application/x-xpinstall'
+                )
+            })
 
-            def __init__(self, data):
-                HTMLParser.HTMLParser.__init__(self)
-                try:
-                    self.feed(data)
-                    self.close()
-                except Exception, e:
-                    if e != self.dummy_exception:
-                        raise
+        request = urllib2.Request(upload_url, data=data)
+        request.add_header('Content-Type', content_type)
+        request.add_header('Authorization', 'JWT ' + token)
+        request.get_method = lambda: 'PUT'
 
-                if not self.result:
-                    raise Exception('Failed to extract CSRF token')
-
-            def set_result(self, value):
-                self.result = value
-                raise self.dummy_exception
-
-            def handle_starttag(self, tag, attrs):
-                attrs = dict(attrs)
-                if tag == 'meta' and attrs.get('name') == 'csrf':
-                    self.set_result(attrs.get('content'))
-                if tag == 'input' and attrs.get('name') == 'csrfmiddlewaretoken':
-                    self.set_result(attrs.get('value'))
-
-        # Extract anonymous CSRF token
-        login_page = load_url(login_url)
-        csrf_token = CSRFParser(login_page).result
-
-        # Log in and get session's CSRF token
-        main_page = load_url(
-            login_url,
-            {
-                'csrfmiddlewaretoken': csrf_token,
-                'username': username,
-                'password': password,
-            }
-        )
-        csrf_token = CSRFParser(main_page).result
-
-        # Upload build
-        with open(self.path, 'rb') as file:
-            upload_response = json.loads(load_url(
-                upload_url,
-                {
-                    'csrfmiddlewaretoken': csrf_token,
-                    'upload': (os.path.basename(self.path), file.read(), 'application/x-xpinstall'),
-                }
-            ))
-
-        # Wait for validation to finish
-        while not upload_response.get('validation'):
-            time.sleep(2)
-            upload_response = json.loads(load_url(
-                upload_url + '/' + upload_response.get('upload')
-            ))
-
-        if upload_response['validation'].get('errors', 0):
-            raise Exception('Build failed AMO validation, see https://addons.mozilla.org%s' % upload_response.get('full_report_url'))
-
-        # Add version
-        add_response = json.loads(load_url(
-            add_url,
-            {
-                'csrfmiddlewaretoken': csrf_token,
-                'upload': upload_response.get('upload'),
-                'source': ('', '', 'application/octet-stream'),
-                'beta': 'on',
-                'supported_platforms': 1,       # PLATFORM_ANY.id
-            }
-        ))
+        try:
+            urllib2.urlopen(request).close()
+        except urllib2.HTTPError as e:
+            logging.error(e.read())

[Sebastian Noack, 2016/09/13 15:03:12]

The error response should be closed as well.

[Wladimir Palant, 2016/09/13 15:16:54]

Done.

+            raise
 
     def uploadToChromeWebStore(self):
         # Google APIs use HTTP error codes with error message in body. So we add
         # the response body to the HTTPError to get more meaningful error messages.
 
         class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
             def http_error_default(self, req, fp, code, msg, hdrs):
                 raise urllib2.HTTPError(req.get_full_url(), code, '%s\n%s' % (msg, fp.read()), hdrs, fp)
 
         opener = urllib2.build_opener(HTTPErrorBodyHandler)
@@ skipping 90 matching lines @@
 
             if self.config.type == 'ie':
                 self.writeIEUpdateManifest(versions)
 
             # update index page
             self.updateIndex(versions)
 
             # update nightlies config
             self.config.latestRevision = self.revision
 
-            if self.config.type == 'gecko' and self.config.galleryID and get_config().get('extensions', 'amo_username'):
+            if self.config.type == 'gecko' and self.config.galleryID and get_config().has_option('extensions', 'amo_key'):
                 self.uploadToMozillaAddons()
             elif self.config.type == 'chrome' and self.config.clientID and self.config.clientSecret and self.config.refreshToken:
                 self.uploadToChromeWebStore()
         finally:
             # clean up
             if self.tempdir:
                 shutil.rmtree(self.tempdir, ignore_errors=True)
 
 
 def main():
@@ skipping 17 matching lines @@
         except Exception, ex:
             logging.error('The build for %s failed:', repo)
             logging.exception(ex)
 
     file = open(nightlyConfigFile, 'wb')
     nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
     main()