Issue 4549 - Implement the Windows Store API to upload development builds

sitescripts/extensions/bin/createNightlies.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2016 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
@@ skipping 24 matching lines @@
 import random
 import shutil
 import struct
 import subprocess
 import sys
 import tempfile
 import time
 from urllib import urlencode
 import urllib2
 import urlparse
+import httplib
+import zipfile
+import contextlib
+
 from xml.dom.minidom import parse as parseXml
 
 from sitescripts.extensions.utils import (
     compareVersions, Configuration,
     writeAndroidUpdateManifest
 )
 from sitescripts.utils import get_config, get_template
 
 MAX_BUILDS = 50
 
@@ skipping 170 matching lines @@
         metadata = packager.readMetadata(self.tempdir, self.config.type)
         certs = xarfile.read_certificates_and_key(self.config.keyFile)[0]
 
         self.certificateID = packager.get_developer_identifier(certs)
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.shortVersion = metadata.get('general', 'version')
         self.basename = metadata.get('general', 'basename')
         self.updatedFromGallery = False
 
+    def read_edge_metadata(self):
+        """
+          Read Edge-specific metadata from metadata file.
+        """
+        from buildtools import packager
+        # Now read metadata file
+        metadata = packager.readMetadata(self.tempdir, self.config.type)
+        self.version = packager.getBuildVersion(self.tempdir, metadata, False,
+                                                self.buildNum)
+        self.basename = metadata.get('general', 'basename')
+
+        self.compat = []
+
     def writeUpdateManifest(self):
         """
           Writes update manifest for the current build
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if self.config.type == 'safari':
             manifestPath = os.path.join(baseDir, 'updates.plist')
             templateName = 'safariUpdateManifest'
             autoescape = True
         elif self.config.type == 'android':
@@ skipping 86 matching lines @@
                     os.remove(self.path)
                 raise
         else:
             env = os.environ
             spiderMonkeyBinary = self.config.spiderMonkeyBinary
             if spiderMonkeyBinary:
                 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
 
             command = [os.path.join(self.tempdir, 'build.py'),
                        '-t', self.config.type, 'build', '-b', self.buildNum]
-            if self.config.type not in {'gecko', 'gecko-webext'}:
+            if self.config.type not in {'gecko', 'gecko-webext', 'edge'}:
                 command.extend(['-k', self.config.keyFile])
             command.append(self.path)
             subprocess.check_call(command, env=env)
 
         if not os.path.exists(self.path):
             raise Exception("Build failed, output file hasn't been created")
 
         linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
         if hasattr(os, 'symlink'):
             if os.path.exists(linkPath):
@@ skipping 161 matching lines @@
         request.get_method = lambda: 'POST'
         request.add_header('Authorization', auth_token)
         request.add_header('x-goog-api-version', '2')
         request.add_header('Content-Length', '0')
 
         response = json.load(opener.open(request))
 
         if any(status not in ('OK', 'ITEM_PENDING_REVIEW') for status in response['status']):
             raise Exception({'status': response['status'], 'statusDetail': response['statusDetail']})
 
+    def get_response(self, connection, *args):
+        connection.request(*args)
+        response = connection.getresponse()
+        responseValue = response.read().decode()
+        if response.status >= 300 or response.status < 200:
+            raise Exception({'status': response.status,
+                             'statusDetail': response.reason})
+        return responseValue
+
+    def get_windows_store_access_token(self):
+
+        auth_token = ''
+        # use refresh token to obtain a valid access token
+        # https://docs.microsoft.com/en-us/azure/active-directory/active-directory-protocols-oauth-code#refreshing-the-access-tokens
+        token_server = 'login.microsoftonline.com'
+        with contextlib.closing(
+                httplib.HTTPSConnection(token_server)) as tokenConnection:
+
+            # Get access token
+            token_path = '/{}/oauth2/token'.format(self.config.tenantID)
+            response = json.loads(self.get_response(
+                tokenConnection, 'POST', token_path,
+                urlencode([
+                    ('refresh_token', self.config.refreshToken),
+                    ('client_id', self.config.clientID),
+                    ('client_secret', self.config.clientSecret),
+                    ('grant_type', 'refresh_token'),
+                    ('resource', 'https://graph.windows.net')
+                ])))
+            auth_token = '{0[token_type]} {0[access_token]}'.format(response)
+
+        return auth_token
+
+    def upload_appx_file_to_windows_store(self, file_upload_url):
+
+        # Add .appx file to a .zip file
+        zip_path = os.path.splitext(self.path)[0] + '.zip'
+        with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
+            zf.write(self.path, os.path.basename(self.path))
+
+        # Upload that .zip file
+        file_upload_url = file_upload_url.replace('+', '%2B')
+        parts = httplib.urlsplit(file_upload_url)
+        with contextlib.closing(
+                httplib.HTTPSConnection(parts.netloc)) as file_upload_con:
+            file_headers = {'x-ms-blob-type': 'BlockBlob'}
+            url = '{}?{}{}'.format(parts.path, parts.query, parts.fragment)
+            file_upload_con.request('PUT', url,
+                                    open(zip_path, 'rb'), file_headers)
+            file_upload_con.getresponse().read()
+
+    def upload_to_windows_store(self):
+
+        auth_token = self.get_windows_store_access_token()
+
+        # Clone the previous submission for the new one. Largely based on code
+        # from https://msdn.microsoft.com/en-us/windows/uwp/monetize/python-code-examples-for-the-windows-store-submission-api#create-an-app-submission
+        headers = {'Authorization': auth_token,
+                   'Content-type': 'application/json'}
+
+        api_server = 'manage.devcenter.microsoft.com'
+        with contextlib.closing(
+                httplib.HTTPSConnection(api_server)) as connection:
+
+            # Get application
+            # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app
+            api_path = '/v1.0/my/applications/{}'.format(
+                self.config.devbuildGalleryID)
+            app_obj = json.loads(self.get_response(connection, 'GET',
+                                                   api_path, '', headers))
+
+            # Delete existing in-progress submission
+            # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-submission
+            submissions_path = api_path + '/submissions'
+            if 'pendingApplicationSubmission' in app_obj:
+                remove_id = app_obj['pendingApplicationSubmission']['id']
+                self.get_response(connection, 'DELETE',
+                                  '%s/%s' % (submissions_path, remove_id),
+                                  '', headers)
+
+            # Create submission
+            # https://msdn.microsoft.com/en-us/windows/uwp/monetize/create-an-app-submission
+            submission = json.loads(self.get_response(
+                                    connection, 'POST',
+                                    submissions_path, '', headers))
+
+            submission_id = submission['id']
+            file_upload_url = submission['fileUploadUrl']
+
+            # Update submission
+            old_submission = submission['applicationPackages'][0]
+            old_submission['fileStatus'] = 'PendingDelete'
+            submission['applicationPackages'].append(
+                {'fileStatus': 'PendingUpload'})
+            added_submission = submission['applicationPackages'][1]
+            added_submission['fileName'] = os.path.basename(self.path)
+
+            old_min_sys_ram = old_submission['minimumSystemRam']
+            added_submission['minimumSystemRam'] = old_min_sys_ram
+
+            old_directx_version = old_submission['minimumDirectXVersion']
+            added_submission['minimumDirectXVersion'] = old_directx_version
+
+            new_submission_path = '{}/{}'.format(
+                submissions_path, submission_id)
+
+            self.get_response(connection, 'PUT', new_submission_path,
+                              json.dumps(submission), headers)
+
+            self.upload_appx_file_to_windows_store(file_upload_url)
+
+            # Commit submission
+            # https://msdn.microsoft.com/en-us/windows/uwp/monetize/commit-an-app-submission
+            submission = json.loads(self.get_response(connection, 'POST',
+                                    new_submission_path + '/commit',
+                                    '', headers))
+
+            if submission['status'] != 'CommitStarted':
+                raise Exception({'status': submission['status'],
+                                 'statusDetails': submission['statusDetails']})
+
     def run(self):
         """
           Run the nightly build process for one extension
         """
         try:
             if self.config.type == 'ie':
                 # We cannot build IE builds, simply list the builds already in
                 # the directory. Basename has to be deduced from the repository name.
                 self.basename = os.path.basename(self.config.repository)
             else:
                 # copy the repository into a temporary directory
                 self.copyRepository()
                 self.buildNum = self.getCurrentBuild()
 
                 # get meta data from the repository
                 if self.config.type == 'android':
                     self.readAndroidMetadata()
                 elif self.config.type == 'chrome':
                     self.readChromeMetadata()
                 elif self.config.type == 'safari':
                     self.readSafariMetadata()
                 elif self.config.type in {'gecko', 'gecko-webext'}:
                     self.readGeckoMetadata()
+                elif self.config.type == 'edge':
+                    self.read_edge_metadata()
                 else:
                     raise Exception('Unknown build type {}' % self.config.type)
 
                 # create development build
                 self.build()
 
                 # write out changelog
                 self.writeChangelog(self.getChanges())
 
                 # write update manifest
@@ skipping 11 matching lines @@
 
             # update nightlies config
             self.config.latestRevision = self.revision
 
             if (self.config.type in {'gecko', 'gecko-webext'} and
                     self.config.galleryID and
                     get_config().has_option('extensions', 'amo_key')):
                 self.uploadToMozillaAddons()
             elif self.config.type == 'chrome' and self.config.clientID and self.config.clientSecret and self.config.refreshToken:
                 self.uploadToChromeWebStore()
+            elif self.config.type == 'edge' and self.config.clientID and self.config.clientSecret and self.config.refreshToken and self.config.tenantID:
+                self.upload_to_windows_store()
+
         finally:
             # clean up
             if self.tempdir:
                 shutil.rmtree(self.tempdir, ignore_errors=True)
 
 
 def main():
     """
       main function for createNightlies.py
     """
@@ skipping 14 matching lines @@
         except Exception as ex:
             logging.error('The build for %s failed:', repo)
             logging.exception(ex)
 
     file = open(nightlyConfigFile, 'wb')
     nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
     main()