Issue 4549 - Implement the Windows Store API to upload development builds

sitescripts/extensions/bin/createNightlies.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2016 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
@@ skipping 24 matching lines @@
 import random
 import shutil
 import struct
 import subprocess
 import sys
 import tempfile
 import time
 from urllib import urlencode
 import urllib2
 import urlparse
+import httplib
+import zipfile
+
 from xml.dom.minidom import parse as parseXml
 
 from sitescripts.extensions.utils import (
     compareVersions, Configuration,
     writeAndroidUpdateManifest
 )
 from sitescripts.utils import get_config, get_template
 
 MAX_BUILDS = 50
 
@@ skipping 170 matching lines @@
         metadata = packager.readMetadata(self.tempdir, self.config.type)
         certs = xarfile.read_certificates_and_key(self.config.keyFile)[0]
 
         self.certificateID = packager.get_developer_identifier(certs)
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.shortVersion = metadata.get('general', 'version')
         self.basename = metadata.get('general', 'basename')
         self.updatedFromGallery = False
 
+    def readEdgeMetadata(self):
+        """
+          Read Edge-specific metadata from metadata file.
+        """
+        import buildtools.packagerEdge as packagerEdge
+        # Now read metadata file
+        metadata = packagerEdge.packager.readMetadata(self.tempdir,
+                                                      self.config.type)
+        self.version = packagerEdge.packager.getBuildVersion(self.tempdir,
+                                                             metadata, False,
+                                                             self.buildNum)
+        self.basename = metadata.get('general', 'basename')
+
+        self.compat = []
+
     def writeUpdateManifest(self):
         """
           Writes update manifest for the current build
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if self.config.type == 'safari':
             manifestPath = os.path.join(baseDir, 'updates.plist')
             templateName = 'safariUpdateManifest'
             autoescape = True
         elif self.config.type == 'android':
@@ skipping 86 matching lines @@
                     os.remove(self.path)
                 raise
         else:
             env = os.environ
             spiderMonkeyBinary = self.config.spiderMonkeyBinary
             if spiderMonkeyBinary:
                 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
 
             command = [os.path.join(self.tempdir, 'build.py'),
                        '-t', self.config.type, 'build', '-b', self.buildNum]
-            if self.config.type not in {'gecko', 'gecko-webext'}:
+            if self.config.type not in {'gecko', 'gecko-webext', 'edge'}:
                 command.extend(['-k', self.config.keyFile])
             command.append(self.path)
             subprocess.check_call(command, env=env)
 
         if not os.path.exists(self.path):
             raise Exception("Build failed, output file hasn't been created")
 
         linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
         if hasattr(os, 'symlink'):
             if os.path.exists(linkPath):
@@ skipping 161 matching lines @@
         request.get_method = lambda: 'POST'
         request.add_header('Authorization', auth_token)
         request.add_header('x-goog-api-version', '2')
         request.add_header('Content-Length', '0')
 
         response = json.load(opener.open(request))
 
         if any(status not in ('OK', 'ITEM_PENDING_REVIEW') for status in response['status']):
             raise Exception({'status': response['status'], 'statusDetail': response['statusDetail']})
 
+    def uploadToWindowsStore(self):
+
+        class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
+            def http_error_default(self, req, fp, code, msg, hdrs):
+                raise urllib2.HTTPError(req.get_full_url(), code,
+                                        '%s\n%s' % (msg, fp.read()), hdrs, fp)
+
+        opener = urllib2.build_opener(HTTPErrorBodyHandler)
+
+        # use refresh token to obtain a valid access token
+        # https://docs.microsoft.com/en-us/azure/active-directory/active-directory-protocols-oauth-code#refreshing-the-access-tokens
+
+        response = json.load(opener.open(
+            'https://login.microsoftonline.com/{0}/oauth2/token'.format(
+                self.config.tenantID),
+            urlencode([
+                ('refresh_token', self.config.refreshToken),
+                ('client_id', self.config.clientID),
+                ('client_secret', self.config.clientSecret),
+                ('grant_type', 'refresh_token'),
+                ('resource', 'https://graph.windows.net')
+            ])
+        ))
+
+        auth_token = response['token_type'] + ' ' + response['access_token']
+
+        # Clone the previous submission for the new one. Largely based on code
+        # from https://msdn.microsoft.com/en-us/windows/uwp/monetize/python-code-examples-for-the-windows-store-submission-api#create-an-app-submission
+        headers = {'Authorization': auth_token,
+                   'Content-type': 'application/json',
+                   'User-Agent': 'Python'}
+
+        apiServer = 'manage.devcenter.microsoft.com'
+        ingestionConnection = httplib.HTTPSConnection(apiServer)
+
+        # Get application
+        appPath = '/v1.0/my/applications/%s' % self.config.devbuildGalleryID
+
+        # https://msdn.microsoft.com/en-us/windows/uwp/monetize/get-an-add-on
+        ingestionConnection.request('GET', appPath, '', headers)
+        appResponse = ingestionConnection.getresponse()
+
+        # Delete existing in-progress submission
+        # https://msdn.microsoft.com/en-us/windows/uwp/monetize/get-an-add-on
+        appObj = json.loads(appResponse.read().decode())
+
+        submissionsPath = appPath + '/submissions'
+        if 'pendingApplicationSubmission' in appObj:
+            removeId = appObj['pendingApplicationSubmission']['id']
+            ingestionConnection.request('DELETE',
+                                        '%s/%s' % (submissionsPath, removeId),
+                                        '', headers)
+            deleteSubmissionResponse = ingestionConnection.getresponse()
+            deleteSubmissionResponse.read()
+
+        # Create submission
+        # https://msdn.microsoft.com/en-us/windows/uwp/monetize/create-an-app-submission
+        ingestionConnection.request('POST', submissionsPath, '', headers)
+        createSubmissionResponse = ingestionConnection.getresponse()
+
+        submission = json.loads(
+                createSubmissionResponse.read().decode()
+            )
+
+        submissionId = submission['id']
+        fileUploadUrl = submission['fileUploadUrl']
+
+        # Update submission
+        oldSubmission = submission['applicationPackages'][0]
+        oldSubmission['fileStatus'] = 'PendingDelete'
+        submission['applicationPackages'].append(
+            {'fileStatus': 'PendingUpload'})
+        addedSubmission = submission['applicationPackages'][1]
+        addedSubmission['fileName'] = os.path.basename(self.path)
+        addedSubmission['minimumSystemRam'] = oldSubmission['minimumSystemRam']
+
+        oldDirectXVersion = oldSubmission['minimumDirectXVersion']
+        addedSubmission['minimumDirectXVersion'] = oldDirectXVersion
+
+        newSubmissionPath = '%s/%s' % (submissionsPath, submissionId)
+        ingestionConnection.request('PUT', newSubmissionPath,
+                                    json.dumps(submission), headers)
+        ingestionConnection.getresponse().read()
+
+        # Add .appx file to a .zip file
+        zipPath = os.path.splitext(self.path)[0] + '.zip'
+        with zipfile.ZipFile(zipPath, 'w', zipfile.ZIP_DEFLATED) as zf:
+            zf.write(self.path, os.path.basename(self.path))
+
+        # Upload that .zip file
+        request = urllib2.Request(fileUploadUrl.replace('+', '%2B'))
+        request.get_method = lambda: 'PUT'
+        request.add_header('x-ms-blob-type', 'BlockBlob')
+
+        with open(zipPath, 'rb') as file:
+            fileSize = os.fstat(file.fileno()).st_size - file.tell()
+            request.add_header('Content-Length', fileSize)
+            request.add_data(file)
+            opener.open(request)
+
+        # Commit submission
+        # https://msdn.microsoft.com/en-us/windows/uwp/monetize/commit-an-app-submission
+        ingestionConnection.request('POST',
+                                    newSubmissionPath + '/commit',
+                                    '', headers)
+        submission = json.loads(
+                ingestionConnection.getresponse().read().decode()
+            )
+
+        if submission['status'] != 'CommitStarted':
+            raise Exception({'status': submission['status'],
+                            'statusDetails': submission['statusDetails']})
+        ingestionConnection.close()
+
     def run(self):
         """
           Run the nightly build process for one extension
         """
         try:
             if self.config.type == 'ie':
                 # We cannot build IE builds, simply list the builds already in
                 # the directory. Basename has to be deduced from the repository name.
                 self.basename = os.path.basename(self.config.repository)
             else:
                 # copy the repository into a temporary directory
                 self.copyRepository()
                 self.buildNum = self.getCurrentBuild()
 
                 # get meta data from the repository
                 if self.config.type == 'android':
                     self.readAndroidMetadata()
                 elif self.config.type == 'chrome':
                     self.readChromeMetadata()
                 elif self.config.type == 'safari':
                     self.readSafariMetadata()
                 elif self.config.type in {'gecko', 'gecko-webext'}:
                     self.readGeckoMetadata()
+                elif self.config.type == 'edge':
+                    self.readEdgeMetadata()
                 else:
                     raise Exception('Unknown build type {}' % self.config.type)
 
                 # create development build
                 self.build()
 
                 # write out changelog
                 self.writeChangelog(self.getChanges())
 
                 # write update manifest
@@ skipping 11 matching lines @@
 
             # update nightlies config
             self.config.latestRevision = self.revision
 
             if (self.config.type in {'gecko', 'gecko-webext'} and
                     self.config.galleryID and
                     get_config().has_option('extensions', 'amo_key')):
                 self.uploadToMozillaAddons()
             elif self.config.type == 'chrome' and self.config.clientID and self.config.clientSecret and self.config.refreshToken:
                 self.uploadToChromeWebStore()
+            elif self.config.type == 'edge' and self.config.clientID and self.config.clientSecret and self.config.refreshToken and self.config.tenantID:
+                self.uploadToWindowsStore()
+
         finally:
             # clean up
             if self.tempdir:
                 shutil.rmtree(self.tempdir, ignore_errors=True)
 
 
 def main():
     """
       main function for createNightlies.py
     """
@@ skipping 14 matching lines @@
         except Exception as ex:
             logging.error('The build for %s failed:', repo)
             logging.exception(ex)
 
     file = open(nightlyConfigFile, 'wb')
     nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
     main()