Issue 6291 - Use client certificate for Windows Store uploads

sitescripts/extensions/bin/createNightlies.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
@@ skipping 27 matching lines @@
 import subprocess
 import sys
 import tempfile
 import time
 import uuid
 from urllib import urlencode
 import urllib2
 import urlparse
 import zipfile
 import contextlib
+from xml.dom.minidom import parse as parseXml
 
 from Crypto.PublicKey import RSA
 from Crypto.Signature import PKCS1_v1_5
 import Crypto.Hash.SHA256
 
-from xml.dom.minidom import parse as parseXml

[Vasily Kuznetsov, 2018/04/17 17:43:54]

Nit: this should be above together with other stdlib imports.

[tlucas, 2018/04/18 12:58:30]

Done.

-
 from sitescripts.extensions.utils import (
     compareVersions, Configuration,
-    writeAndroidUpdateManifest
+    writeAndroidUpdateManifest,
 )
 from sitescripts.utils import get_config, get_template
 
 MAX_BUILDS = 50
 
 
 # Google and Microsoft APIs use HTTP error codes with error message in
 # body. So we add the response body to the HTTPError to get more
 # meaningful error messages.
 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
@@ skipping 28 matching lines @@
 
     def hasChanges(self):
         return self.revision != self.previousRevision
 
     def getCurrentRevision(self):
         """
             retrieves the current revision ID from the repository
         """
         command = [
             'hg', 'id', '-i', '-r', self.config.revision, '--config',
-            'defaults.id=', self.config.repository
+            'defaults.id=', self.config.repository,
         ]
         return subprocess.check_output(command).strip()
 
     def getCurrentBuild(self):
         """
             calculates the (typically numerical) build ID for the current build
         """
         command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
         build = subprocess.check_output(command).strip()
         return build
 
     def getChanges(self):
         """
           retrieve changes between the current and previous ("first") revision
         """
         command = [
             'hg', 'log', '-R', self.tempdir, '-r',
             'reverse(ancestors({}))'.format(self.config.revision), '-l', '50',
             '--encoding', 'utf-8', '--template',
             '{date|isodate}\\0{author|person}\\0{rev}\\0{desc}\\0\\0',
-            '--config', 'defaults.log='
+            '--config', 'defaults.log=',
         ]
         result = subprocess.check_output(command).decode('utf-8')
 
         for change in result.split('\x00\x00'):
             if change:
                 date, author, revision, description = change.split('\x00')
                 yield {'date': date, 'author': author, 'revision': revision, 'description': description}
 
     def copyRepository(self):
         """
@@ skipping 141 matching lines @@
             os.makedirs(baseDir)
 
         # ABP for Android used to have its own update manifest format. We need to
         # generate both that and the new one in the libadblockplus format as long
         # as a significant amount of users is on an old version.
         if self.config.type == 'android':
             newManifestPath = os.path.join(baseDir, 'update.json')
             writeAndroidUpdateManifest(newManifestPath, [{
                 'basename': self.basename,
                 'version': self.version,
-                'updateURL': self.updateURL
+                'updateURL': self.updateURL,
             }])
 
         template = get_template(get_config().get('extensions', templateName),
                                 autoescape=autoescape)
         template.stream({'extensions': [self]}).dump(manifestPath)
 
     def writeIEUpdateManifest(self, versions):
         """
           Writes update.json file for the latest IE build
         """
         if len(versions) == 0:
             return
 
         version = versions[0]
         packageName = self.basename + '-' + version + self.config.packageSuffix
         updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + packageName + '?update')
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         manifestPath = os.path.join(baseDir, 'update.json')
 
         from sitescripts.extensions.utils import writeIEUpdateManifest as doWrite
         doWrite(manifestPath, [{
             'basename': self.basename,
             'version': version,
-            'updateURL': updateURL
+            'updateURL': updateURL,
         }])
 
         for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']:
             linkPath = os.path.join(baseDir, '00latest%s' % suffix)
             outputPath = os.path.join(baseDir, self.basename + '-' + version + suffix)
             self.symlink_or_copy(outputPath, linkPath)
 
     def build(self):
         """
           run the build command in the tempdir
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if not os.path.exists(baseDir):
             os.makedirs(baseDir)
         outputFile = '%s-%s%s' % (self.basename, self.version, self.config.packageSuffix)
         self.path = os.path.join(baseDir, outputFile)
         self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + outputFile + '?update')
 
         if self.config.type == 'android':
             apkFile = open(self.path, 'wb')
 
             try:
                 try:
                     port = get_config().get('extensions', 'androidBuildPort')
                 except ConfigParser.NoOptionError:
                     port = '22'
                 command = ['ssh', '-p', port, get_config().get('extensions', 'androidBuildHost')]
                 command.extend(map(pipes.quote, [
                     '/home/android/bin/makedebugbuild.py', '--revision',
-                    self.buildNum, '--version', self.version, '--stdout'
+                    self.buildNum, '--version', self.version, '--stdout',
                 ]))
                 subprocess.check_call(command, stdout=apkFile, close_fds=True)
             except:
                 # clear broken output if any
                 if os.path.exists(self.path):
                     os.remove(self.path)
                 raise
         else:
             env = os.environ
             spiderMonkeyBinary = self.config.spiderMonkeyBinary
@@ skipping 53 matching lines @@
             packageFile = self.basename + '-' + version + self.config.packageSuffix
             changelogFile = self.basename + '-' + version + '.changelog.xhtml'
             if not os.path.exists(os.path.join(baseDir, packageFile)):
                 # Oops
                 continue
 
             link = {
                 'version': version,
                 'download': packageFile,
                 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)),
-                'size': os.path.getsize(os.path.join(baseDir, packageFile))
+                'size': os.path.getsize(os.path.join(baseDir, packageFile)),
             }
             if os.path.exists(os.path.join(baseDir, changelogFile)):
                 link['changelog'] = changelogFile
             links.append(link)
         template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
         template.stream({'config': self.config, 'links': links}).dump(outputPath)
 
     def read_downloads_lockfile(self):
         path = get_config().get('extensions', 'downloadLockFile')
         try:
@@ skipping 27 matching lines @@
                     del current[platform][i]
                 if len(current[platform]) == 0:
                     del current[platform]
         except KeyError:
             pass
         self.write_downloads_lockfile(current)
 
     def azure_jwt_signature_fnc(self):
         return (
             'RS256',
-            lambda s, m: PKCS1_v1_5.new(s).sign(Crypto.Hash.SHA256.new(m))

[Vasily Kuznetsov, 2018/04/17 17:43:54]

Nit: According to the recently landed coding style update, there should be a
comma at the end of this line. Ditto for line 471.

[tlucas, 2018/04/18 12:58:30]

Done.

+            lambda s, m: PKCS1_v1_5.new(s).sign(Crypto.Hash.SHA256.new(m)),
         )
 
     def mozilla_jwt_signature_fnc(self):
         return (
             'HS256',
-            lambda s, m: hmac.new(s, msg=m, digestmod=hashlib.sha256).digest()
+            lambda s, m: hmac.new(s, msg=m, digestmod=hashlib.sha256).digest(),
         )
 
     def sign_jwt(self, issuer, secret, url, signature_fnc, jwt_headers={}):
         alg, fnc = signature_fnc()
 
         header = {'typ': 'JWT'}
         header.update(jwt_headers)
         header.update({'alg': alg})
 
         issued = int(time.time())
@@ skipping 35 matching lines @@
         config = get_config()
 
         upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
                       'versions/{}/').format(self.extensionID, self.version)
 
         with open(self.path, 'rb') as file:
             data, content_type = urllib3.filepost.encode_multipart_formdata({
                 'upload': (
                     os.path.basename(self.path),
                     file.read(),
-                    'application/x-xpinstall'
-                )
+                    'application/x-xpinstall',
+                ),
             })
 
         request = self.generate_mozilla_jwt_request(
             config.get('extensions', 'amo_key'),
             config.get('extensions', 'amo_secret'),
             upload_url,
             'PUT',
             data,
             [('Content-Type', content_type)],
         )
 
         try:
             urllib2.urlopen(request).close()
         except urllib2.HTTPError as e:
             try:
                 logging.error(e.read())
             finally:
                 e.close()
             raise
 
         self.add_to_downloads_lockfile(
             self.config.type,
             {
                 'buildtype': 'devbuild',
                 'app_id': self.extensionID,
                 'version': self.version,
-            }
+            },
         )
         os.remove(self.path)
 
     def download_from_mozilla_addons(self, buildtype, version, app_id):
         config = get_config()
         iss = config.get('extensions', 'amo_key')
         secret = config.get('extensions', 'amo_secret')
 
         url = ('https://addons.mozilla.org/api/v3/addons/{}/'
                'versions/{}/').format(app_id, version)
 
         request = self.generate_mozilla_jwt_request(
             iss, secret, url, 'GET',
         )
         response = json.load(urllib2.urlopen(request))
 
         filename = '{}-{}.xpi'.format(self.basename, version)
         self.path = os.path.join(
             config.get('extensions', 'nightliesDirectory'),
             self.basename,
-            filename
+            filename,
         )
 
         necessary = ['passed_review', 'reviewed', 'processed', 'valid']
         if all(response[x] for x in necessary):
             download_url = response['files'][0]['download_url']
             checksum = response['files'][0]['hash']
 
             request = self.generate_mozilla_jwt_request(
                 iss, secret, download_url, 'GET',
             )
@@ skipping 10 matching lines @@
             if returned_checksum != checksum:
                 logging.error('Checksum could not be verified: {} vs {}'
                               ''.format(checksum, returned_checksum))
 
             with open(self.path, 'w') as fp:
                 fp.write(file_content)
 
             self.update_link = os.path.join(
                 config.get('extensions', 'nightliesURL'),
                 self.basename,
-                filename
+                filename,
             )
 
             self.remove_from_downloads_lockfile(self.config.type,
                                                 'version',
                                                 version)
         elif not response['passed_review'] or not response['valid']:
             # When the review failed for any reason, we want to know about it
             logging.error(json.dumps(response, indent=4))
             self.remove_from_downloads_lockfile(self.config.type,
                                                 'version',
                                                 version)
 
     def uploadToChromeWebStore(self):
 
         opener = urllib2.build_opener(HTTPErrorBodyHandler)
 
         # use refresh token to obtain a valid access token
         # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh
 
         response = json.load(opener.open(
             'https://accounts.google.com/o/oauth2/token',
 
             urlencode([
                 ('refresh_token', self.config.refreshToken),
                 ('client_id', self.config.clientID),
                 ('client_secret', self.config.clientSecret),
                 ('grant_type', 'refresh_token'),
-            ])
+            ]),
         ))
 
         auth_token = '%s %s' % (response['token_type'], response['access_token'])
 
         # upload a new version with the Chrome Web Store API
         # https://developer.chrome.com/webstore/using_webstore_api#uploadexisitng
 
         request = urllib2.Request('https://www.googleapis.com/upload/chromewebstore/v1.1/items/' + self.config.devbuildGalleryID)
         request.get_method = lambda: 'PUT'
         request.add_header('Authorization', auth_token)
@@ skipping 50 matching lines @@
             'client_assertion': signed,
         }
 
         request = urllib2.Request(url, urlencode(oauth_params))
         request.get_method = lambda: 'POST'
 
         return request
 
     def get_windows_store_access_token(self):
         # use client certificate to obtain a valid access token
-        url = 'https://login.microsoftonline.com/{}/oauth2/token'.format(

[Vasily Kuznetsov, 2018/04/17 17:43:54]

Preexisting version of this was kind of a hack to avoid splitting lines, so
probably good riddance, but this "one-liner" now takes 3 lines :/ Perhaps one of
those cases where strict line length rules are more of a pain than benefit.

What do you think about this version?

    url_template = 'https://login.microsoftonline.com/{}/oauth2/token'
    url = url_template.format(self.config.tenantID)

[tlucas, 2018/04/18 12:58:30]

Done.

-            self.config.tenantID
-        )
+        url_template = 'https://login.microsoftonline.com/{}/oauth2/token'
+        url = url_template.format(self.config.tenantID)
 
         with open(self.config.privateKey, 'r') as fp:
             private_key = fp.read()
 
         opener = urllib2.build_opener(HTTPErrorBodyHandler)
         request = self.generate_certificate_token_request(url, private_key)
 
         with contextlib.closing(opener.open(request)) as response:
             data = json.load(response)
             auth_token = '{0[token_type]} {0[access_token]}'.format(data)
@@ skipping 25 matching lines @@
     def upload_to_windows_store(self):
         opener = urllib2.build_opener(HTTPErrorBodyHandler)
 
         headers = {'Authorization': self.get_windows_store_access_token(),
                    'Content-type': 'application/json'}
 
         # Get application
         # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app
         api_path = '{}/v1.0/my/applications/{}'.format(
             'https://manage.devcenter.microsoft.com',
-            self.config.devbuildGalleryID
+            self.config.devbuildGalleryID,
         )
 
         request = urllib2.Request(api_path, None, headers)
         with contextlib.closing(opener.open(request)) as response:
             app_obj = json.load(response)
 
         # Delete existing in-progress submission
         # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-submission
         submissions_path = api_path + '/submissions'
         if 'pendingApplicationSubmission' in app_obj:
@@ skipping 126 matching lines @@
                         # write update manifest
                         self.writeUpdateManifest()
 
                         # retire old builds
                         versions = self.retireBuilds()
                         # update index page
                         self.updateIndex(versions)
 
                         # Update soft link to latest build
                         baseDir = os.path.join(
-                            self.config.nightliesDirectory, self.basename
+                            self.config.nightliesDirectory, self.basename,
                         )
                         linkPath = os.path.join(
-                            baseDir, '00latest' + self.config.packageSuffix
+                            baseDir, '00latest' + self.config.packageSuffix,
                         )
 
                         self.symlink_or_copy(self.path, linkPath)
             finally:
                 # clean up
                 if self.tempdir:
                     shutil.rmtree(self.tempdir, ignore_errors=True)
 
 
 def main(download=False):
@@ skipping 23 matching lines @@
 
     file = open(nightlyConfigFile, 'wb')
     nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('--download', action='store_true', default=False)
     args = parser.parse_args()
     main(args.download)