Issue 4549 - Implement the Windows Store API to upload development builds

sitescripts/extensions/bin/createNightlies.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2016 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 
 Nightly builds generation script
 ================================
 
   This script generates nightly builds of extensions, together
   with changelogs and documentation.
 
 """
 
 import ConfigParser
 import base64
-from datetime import datetime
 import hashlib
 import hmac
 import json
 import logging
 import os
 import pipes
 import random
 import shutil
 import struct
 import subprocess
 import sys
 import tempfile
 import time
 from urllib import urlencode
 import urllib2
 import urlparse
-import httplib
 import zipfile
+import contextlib
 
 from xml.dom.minidom import parse as parseXml
 
 from sitescripts.extensions.utils import (
     compareVersions, Configuration,
     writeAndroidUpdateManifest
 )
 from sitescripts.utils import get_config, get_template
 
 MAX_BUILDS = 50
+
+
+# Google and Microsoft APIs use HTTP error codes with error message in
+# body. So we add the response body to the HTTPError to get more
+# meaningful error messages.
+class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise urllib2.HTTPError(req.get_full_url(), code,
+                                '{}\n{}'.format(msg, fp.read()), hdrs, fp)
 
 
 class NightlyBuild(object):
     """
       Performs the build process for an extension,
       generating changelogs and documentation.
     """
 
     def __init__(self, config):
         """
@@ skipping 161 matching lines @@
         metadata = packager.readMetadata(self.tempdir, self.config.type)
         certs = xarfile.read_certificates_and_key(self.config.keyFile)[0]
 
         self.certificateID = packager.get_developer_identifier(certs)
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.shortVersion = metadata.get('general', 'version')
         self.basename = metadata.get('general', 'basename')
         self.updatedFromGallery = False
 
-    def readEdgeMetadata(self):
+    def read_edge_metadata(self):
         """
           Read Edge-specific metadata from metadata file.
         """
-        import buildtools.packagerEdge as packagerEdge
+        from buildtools import packager
         # Now read metadata file
-        metadata = packagerEdge.packager.readMetadata(self.tempdir,
-                                                      self.config.type)
-        self.version = packagerEdge.packager.getBuildVersion(self.tempdir,
-                                                             metadata, False,
-                                                             self.buildNum)
+        metadata = packager.readMetadata(self.tempdir, self.config.type)
+        self.version = packager.getBuildVersion(self.tempdir, metadata, False,
+                                                self.buildNum)
         self.basename = metadata.get('general', 'basename')
 
         self.compat = []
 
     def writeUpdateManifest(self):
         """
           Writes update manifest for the current build
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if self.config.type == 'safari':
@@ skipping 210 matching lines @@
         try:
             urllib2.urlopen(request).close()
         except urllib2.HTTPError as e:
             try:
                 logging.error(e.read())
             finally:
                 e.close()
             raise
 
     def uploadToChromeWebStore(self):
-        # Google APIs use HTTP error codes with error message in body. So we add
-        # the response body to the HTTPError to get more meaningful error messages.
-
-        class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
-            def http_error_default(self, req, fp, code, msg, hdrs):
-                raise urllib2.HTTPError(req.get_full_url(), code, '%s\n%s' % (msg, fp.read()), hdrs, fp)
 
         opener = urllib2.build_opener(HTTPErrorBodyHandler)
 
         # use refresh token to obtain a valid access token
         # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh
 
         response = json.load(opener.open(
             'https://accounts.google.com/o/oauth2/token',
 
             urlencode([
@@ skipping 36 matching lines @@
         request.get_method = lambda: 'POST'
         request.add_header('Authorization', auth_token)
         request.add_header('x-goog-api-version', '2')
         request.add_header('Content-Length', '0')
 
         response = json.load(opener.open(request))
 
         if any(status not in ('OK', 'ITEM_PENDING_REVIEW') for status in response['status']):
             raise Exception({'status': response['status'], 'statusDetail': response['statusDetail']})
 
-    def uploadToWindowsStore(self):
-
-        class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
-            def http_error_default(self, req, fp, code, msg, hdrs):
-                raise urllib2.HTTPError(req.get_full_url(), code,
-                                        '%s\n%s' % (msg, fp.read()), hdrs, fp)
-
-        opener = urllib2.build_opener(HTTPErrorBodyHandler)
-
+    def get_windows_store_access_token(self):
         # use refresh token to obtain a valid access token
         # https://docs.microsoft.com/en-us/azure/active-directory/active-directory-protocols-oauth-code#refreshing-the-access-tokens
-
-        response = json.load(opener.open(
-            'https://login.microsoftonline.com/{0}/oauth2/token'.format(
-                self.config.tenantID),
-            urlencode([
-                ('refresh_token', self.config.refreshToken),
-                ('client_id', self.config.clientID),
-                ('client_secret', self.config.clientSecret),
-                ('grant_type', 'refresh_token'),
-                ('resource', 'https://graph.windows.net')
-            ])
-        ))
-
-        auth_token = response['token_type'] + ' ' + response['access_token']
-
-        # Clone the previous submission for the new one. Largely based on code
-        # from https://msdn.microsoft.com/en-us/windows/uwp/monetize/python-code-examples-for-the-windows-store-submission-api#create-an-app-submission
-        headers = {'Authorization': auth_token,
-                   'Content-type': 'application/json',
-                   'User-Agent': 'Python'}
-
-        apiServer = 'manage.devcenter.microsoft.com'
-        ingestionConnection = httplib.HTTPSConnection(apiServer)
+        server = 'https://login.microsoftonline.com'
+        token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID)
+
+        opener = urllib2.build_opener(HTTPErrorBodyHandler)
+        post_data = urlencode([
+            ('refresh_token', self.config.refreshToken),
+            ('client_id', self.config.clientID),
+            ('client_secret', self.config.clientSecret),
+            ('grant_type', 'refresh_token'),
+            ('resource', 'https://graph.windows.net')
+        ])
+        request = urllib2.Request(token_path, post_data)
+        with contextlib.closing(opener.open(request)) as response:
+            data = json.load(response)
+            auth_token = '{0[token_type]} {0[access_token]}'.format(data)
+
+        return auth_token
+
+    def upload_appx_file_to_windows_store(self, file_upload_url):
+        # Add .appx file to a .zip file
+        zip_path = os.path.splitext(self.path)[0] + '.zip'
+        with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
+            zf.write(self.path, os.path.basename(self.path))
+
+        # Upload that .zip file
+        file_upload_url = file_upload_url.replace('+', '%2B')
+        request = urllib2.Request(file_upload_url)
+        request.get_method = lambda: 'PUT'
+        request.add_header('x-ms-blob-type', 'BlockBlob')
+
+        opener = urllib2.build_opener(HTTPErrorBodyHandler)
+
+        with open(zip_path, 'rb') as file:
+            request.add_header('Content-Length',
+                               os.fstat(file.fileno()).st_size - file.tell())
+            request.add_data(file)
+            opener.open(request).close()
+
+    # Clone the previous submission for the new one. Largely based on code
+    # from https://msdn.microsoft.com/en-us/windows/uwp/monetize/python-code-examples-for-the-windows-store-submission-api#create-an-app-submission
+    def upload_to_windows_store(self):
+        opener = urllib2.build_opener(HTTPErrorBodyHandler)
+
+        headers = {'Authorization': self.get_windows_store_access_token(),
+                   'Content-type': 'application/json'}
 
         # Get application
-        appPath = '/v1.0/my/applications/%s' % self.config.devbuildGalleryID
-
-        # https://msdn.microsoft.com/en-us/windows/uwp/monetize/get-an-add-on
-        ingestionConnection.request('GET', appPath, '', headers)
-        appResponse = ingestionConnection.getresponse()
+        # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app
+        api_path = '{}/v1.0/my/applications/{}'.format(
+            'https://manage.devcenter.microsoft.com',
+            self.config.devbuildGalleryID
+        )
+
+        request = urllib2.Request(api_path, None, headers)
+        with contextlib.closing(opener.open(request)) as response:
+            app_obj = json.load(response)
 
         # Delete existing in-progress submission
-        # https://msdn.microsoft.com/en-us/windows/uwp/monetize/get-an-add-on
-        appObj = json.loads(appResponse.read().decode())
-
-        submissionsPath = appPath + '/submissions'
-        if 'pendingApplicationSubmission' in appObj:
-            removeId = appObj['pendingApplicationSubmission']['id']
-            ingestionConnection.request('DELETE',
-                                        '%s/%s' % (submissionsPath, removeId),
-                                        '', headers)
-            deleteSubmissionResponse = ingestionConnection.getresponse()
-            deleteSubmissionResponse.read()
+        # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-submission
+        submissions_path = api_path + '/submissions'
+        if 'pendingApplicationSubmission' in app_obj:
+            remove_id = app_obj['pendingApplicationSubmission']['id']
+            remove_path = '{}/{}'.format(submissions_path, remove_id)
+            request = urllib2.Request(remove_path, '', headers)
+            request.get_method = lambda: 'DELETE'
+            opener.open(request).close()
 
         # Create submission
         # https://msdn.microsoft.com/en-us/windows/uwp/monetize/create-an-app-submission
-        ingestionConnection.request('POST', submissionsPath, '', headers)
-        createSubmissionResponse = ingestionConnection.getresponse()
-
-        submission = json.loads(
-                createSubmissionResponse.read().decode()
-            )
-
-        submissionId = submission['id']
-        fileUploadUrl = submission['fileUploadUrl']
-
-        # Update submission
-        oldSubmission = submission['applicationPackages'][0]
-        oldSubmission['fileStatus'] = 'PendingDelete'
-        submission['applicationPackages'].append(
-            {'fileStatus': 'PendingUpload'})
-        addedSubmission = submission['applicationPackages'][1]
-        addedSubmission['fileName'] = os.path.basename(self.path)
-        addedSubmission['minimumSystemRam'] = oldSubmission['minimumSystemRam']
-
-        oldDirectXVersion = oldSubmission['minimumDirectXVersion']
-        addedSubmission['minimumDirectXVersion'] = oldDirectXVersion
-
-        newSubmissionPath = '%s/%s' % (submissionsPath, submissionId)
-        ingestionConnection.request('PUT', newSubmissionPath,
-                                    json.dumps(submission), headers)
-        ingestionConnection.getresponse().read()
-
-        # Add .appx file to a .zip file
-        zipPath = os.path.splitext(self.path)[0] + '.zip'
-        with zipfile.ZipFile(zipPath, 'w', zipfile.ZIP_DEFLATED) as zf:
-            zf.write(self.path, os.path.basename(self.path))
-
-        # Upload that .zip file
-        request = urllib2.Request(fileUploadUrl.replace('+', '%2B'))
-        request.get_method = lambda: 'PUT'
-        request.add_header('x-ms-blob-type', 'BlockBlob')
-
-        with open(zipPath, 'rb') as file:
-            fileSize = os.fstat(file.fileno()).st_size - file.tell()
-            request.add_header('Content-Length', fileSize)
-            request.add_data(file)
-            opener.open(request)
+        request = urllib2.Request(submissions_path, '', headers)
+        request.get_method = lambda: 'POST'
+        with contextlib.closing(opener.open(request)) as response:
+            submission = json.load(response)
+
+        submission_id = submission['id']
+        file_upload_url = submission['fileUploadUrl']
+
+        new_submission_path = '{}/{}'.format(submissions_path,
+                                             submission_id)
+
+        request = urllib2.Request(new_submission_path, None, headers)
+        opener.open(request).close()
+
+        self.upload_appx_file_to_windows_store(file_upload_url)
 
         # Commit submission
         # https://msdn.microsoft.com/en-us/windows/uwp/monetize/commit-an-app-submission
-        ingestionConnection.request('POST',
-                                    newSubmissionPath + '/commit',
-                                    '', headers)
-        submission = json.loads(
-                ingestionConnection.getresponse().read().decode()
-            )
+        commit_path = '{}/commit'.format(new_submission_path)
+        request = urllib2.Request(commit_path, '', headers)
+        request.get_method = lambda: 'POST'
+        with contextlib.closing(opener.open(request)) as response:
+            submission = json.load(response)
 
         if submission['status'] != 'CommitStarted':
             raise Exception({'status': submission['status'],
-                            'statusDetails': submission['statusDetails']})
-        ingestionConnection.close()
+                             'statusDetails': submission['statusDetails']})
 
     def run(self):
         """
           Run the nightly build process for one extension
         """
         try:
             if self.config.type == 'ie':
                 # We cannot build IE builds, simply list the builds already in
                 # the directory. Basename has to be deduced from the repository name.
                 self.basename = os.path.basename(self.config.repository)
             else:
                 # copy the repository into a temporary directory
                 self.copyRepository()
                 self.buildNum = self.getCurrentBuild()
 
                 # get meta data from the repository
                 if self.config.type == 'android':
                     self.readAndroidMetadata()
                 elif self.config.type == 'chrome':
                     self.readChromeMetadata()
                 elif self.config.type == 'safari':
                     self.readSafariMetadata()
                 elif self.config.type in {'gecko', 'gecko-webext'}:
                     self.readGeckoMetadata()
                 elif self.config.type == 'edge':
-                    self.readEdgeMetadata()
+                    self.read_edge_metadata()
                 else:
                     raise Exception('Unknown build type {}' % self.config.type)
 
                 # create development build
                 self.build()
 
                 # write out changelog
                 self.writeChangelog(self.getChanges())
 
                 # write update manifest
@@ skipping 12 matching lines @@
             # update nightlies config
             self.config.latestRevision = self.revision
 
             if (self.config.type in {'gecko', 'gecko-webext'} and
                     self.config.galleryID and
                     get_config().has_option('extensions', 'amo_key')):
                 self.uploadToMozillaAddons()
             elif self.config.type == 'chrome' and self.config.clientID and self.config.clientSecret and self.config.refreshToken:
                 self.uploadToChromeWebStore()
             elif self.config.type == 'edge' and self.config.clientID and self.config.clientSecret and self.config.refreshToken and self.config.tenantID:
-                self.uploadToWindowsStore()
+                self.upload_to_windows_store()
 
         finally:
             # clean up
             if self.tempdir:
                 shutil.rmtree(self.tempdir, ignore_errors=True)
 
 
 def main():
     """
       main function for createNightlies.py
@@ skipping 15 matching lines @@
         except Exception as ex:
             logging.error('The build for %s failed:', repo)
             logging.exception(ex)
 
     file = open(nightlyConfigFile, 'wb')
     nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
     main()