Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: sitescripts/extensions/bin/createNightlies.py

Issue 29374637: Issue 4549 - Implement the Windows Store API to upload development builds (Closed)
Left Patch Set: Move to underscore notation. Simplify the networking code. Address comments. Created Feb. 9, 2017, 10:47 a.m.
Right Patch Set: Rebase Created Feb. 16, 2017, 1:14 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « ensure_dependencies.py ('k') | sitescripts/extensions/test/sitescripts.ini.template » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-2016 Eyeo GmbH 2 # Copyright (C) 2006-2016 Eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 """ 16 """
17 17
18 Nightly builds generation script 18 Nightly builds generation script
19 ================================ 19 ================================
20 20
21 This script generates nightly builds of extensions, together 21 This script generates nightly builds of extensions, together
22 with changelogs and documentation. 22 with changelogs and documentation.
23 23
24 """ 24 """
25 25
26 import ConfigParser 26 import ConfigParser
27 import base64 27 import base64
28 from datetime import datetime
29 import hashlib 28 import hashlib
30 import hmac 29 import hmac
31 import json 30 import json
32 import logging 31 import logging
33 import os 32 import os
34 import pipes 33 import pipes
35 import random 34 import random
36 import shutil 35 import shutil
37 import struct 36 import struct
38 import subprocess 37 import subprocess
39 import sys 38 import sys
40 import tempfile 39 import tempfile
41 import time 40 import time
42 from urllib import urlencode 41 from urllib import urlencode
43 import urllib2 42 import urllib2
44 import urlparse 43 import urlparse
45 import httplib
46 import zipfile 44 import zipfile
47 import contextlib 45 import contextlib
48 46
49 from xml.dom.minidom import parse as parseXml 47 from xml.dom.minidom import parse as parseXml
50 48
51 from sitescripts.extensions.utils import ( 49 from sitescripts.extensions.utils import (
52 compareVersions, Configuration, 50 compareVersions, Configuration,
53 writeAndroidUpdateManifest 51 writeAndroidUpdateManifest
54 ) 52 )
55 from sitescripts.utils import get_config, get_template 53 from sitescripts.utils import get_config, get_template
56 54
57 MAX_BUILDS = 50 55 MAX_BUILDS = 50
56
57
58 # Google and Microsoft APIs use HTTP error codes with error message in
59 # body. So we add the response body to the HTTPError to get more
60 # meaningful error messages.
61 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
62 def http_error_default(self, req, fp, code, msg, hdrs):
63 raise urllib2.HTTPError(req.get_full_url(), code,
64 '{}\n{}'.format(msg, fp.read()), hdrs, fp)
58 65
59 66
60 class NightlyBuild(object): 67 class NightlyBuild(object):
61 """ 68 """
62 Performs the build process for an extension, 69 Performs the build process for an extension,
63 generating changelogs and documentation. 70 generating changelogs and documentation.
64 """ 71 """
65 72
66 def __init__(self, config): 73 def __init__(self, config):
67 """ 74 """
(...skipping 400 matching lines...) Expand 10 before | Expand all | Expand 10 after
468 try: 475 try:
469 urllib2.urlopen(request).close() 476 urllib2.urlopen(request).close()
470 except urllib2.HTTPError as e: 477 except urllib2.HTTPError as e:
471 try: 478 try:
472 logging.error(e.read()) 479 logging.error(e.read())
473 finally: 480 finally:
474 e.close() 481 e.close()
475 raise 482 raise
476 483
477 def uploadToChromeWebStore(self): 484 def uploadToChromeWebStore(self):
478 # Google APIs use HTTP error codes with error message in body. So we add
479 # the response body to the HTTPError to get more meaningful error messag es.
480
481 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
482 def http_error_default(self, req, fp, code, msg, hdrs):
483 raise urllib2.HTTPError(req.get_full_url(), code, '%s\n%s' % (ms g, fp.read()), hdrs, fp)
484 485
485 opener = urllib2.build_opener(HTTPErrorBodyHandler) 486 opener = urllib2.build_opener(HTTPErrorBodyHandler)
486 487
487 # use refresh token to obtain a valid access token 488 # use refresh token to obtain a valid access token
488 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh 489 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh
489 490
490 response = json.load(opener.open( 491 response = json.load(opener.open(
491 'https://accounts.google.com/o/oauth2/token', 492 'https://accounts.google.com/o/oauth2/token',
492 493
493 urlencode([ 494 urlencode([
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after
530 request.get_method = lambda: 'POST' 531 request.get_method = lambda: 'POST'
531 request.add_header('Authorization', auth_token) 532 request.add_header('Authorization', auth_token)
532 request.add_header('x-goog-api-version', '2') 533 request.add_header('x-goog-api-version', '2')
533 request.add_header('Content-Length', '0') 534 request.add_header('Content-Length', '0')
534 535
535 response = json.load(opener.open(request)) 536 response = json.load(opener.open(request))
536 537
537 if any(status not in ('OK', 'ITEM_PENDING_REVIEW') for status in respons e['status']): 538 if any(status not in ('OK', 'ITEM_PENDING_REVIEW') for status in respons e['status']):
538 raise Exception({'status': response['status'], 'statusDetail': respo nse['statusDetail']}) 539 raise Exception({'status': response['status'], 'statusDetail': respo nse['statusDetail']})
539 540
540 def get_response(self, connection,
541 method, url, body=None, headers={}):
Sebastian Noack 2017/02/09 12:51:57 Using mutable types as default arguments is rather
Oleksandr 2017/02/13 02:57:36 Done.
542 connection.request(method, url, body, headers)
543 response = connection.getresponse()
544 if (response.status >= 300):
Sebastian Noack 2017/02/09 12:51:57 The parentheses here are redundant, flake8-abp wou
Sebastian Noack 2017/02/09 12:51:57 Shouldn't we rather check for non-2xx? In Python t
Vasily Kuznetsov 2017/02/09 16:37:58 Your condition seems to check for 2xx instead of n
Sebastian Noack 2017/02/09 20:50:23 Would it be a problem if a new connection is used
Oleksandr 2017/02/13 02:57:36 Original thinking is that we do not handle HTTP 3x
Vasily Kuznetsov 2017/02/13 10:44:13 I think I would prefer to just use urllib2 too bec
Sebastian Noack 2017/02/14 12:36:00 Well, the HTTP status codes 100, 101 and 102 are d
545 raise Exception({'status': response.status,
546 'statusDetail': response.reason})
547 return response.read().decode()
Sebastian Noack 2017/02/09 12:51:57 I looked at the code of httplib, and indeed respon
Sebastian Noack 2017/02/09 12:51:57 It seems wherever get_repsonse() is called, and th
Vasily Kuznetsov 2017/02/09 16:37:58 You're right, it does have a close() method indeed
Sebastian Noack 2017/02/09 20:50:23 The code is rather complex with a lot of special c
Vasily Kuznetsov 2017/02/09 21:01:55 Yeah, I was assuming always parsing, I also think
Oleksandr 2017/02/13 02:57:36 There is no json response for deleting a submissio
Oleksandr 2017/02/13 02:57:36 It is not possible to reuse the connection object
Sebastian Noack 2017/02/14 12:36:00 I see. So if json.load() would fail for delete/upd
Sebastian Noack 2017/02/14 12:36:00 Good point. So it seems, in fact, there isn't any
548
549 def get_windows_store_access_token(self): 541 def get_windows_store_access_token(self):
550
551 auth_token = ''
552 # use refresh token to obtain a valid access token 542 # use refresh token to obtain a valid access token
553 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo ry-protocols-oauth-code#refreshing-the-access-tokens 543 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo ry-protocols-oauth-code#refreshing-the-access-tokens
554 token_server = 'login.microsoftonline.com' 544 server = 'https://login.microsoftonline.com'
555 with contextlib.closing( 545 token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID)
556 httplib.HTTPSConnection(token_server)) as tokenConnection: 546
557 547 opener = urllib2.build_opener(HTTPErrorBodyHandler)
558 # Get access token 548 post_data = urlencode([
559 token_path = '/{}/oauth2/token'.format(self.config.tenantID) 549 ('refresh_token', self.config.refreshToken),
560 token_response = json.loads(self.get_response( 550 ('client_id', self.config.clientID),
561 tokenConnection, 'POST', token_path, 551 ('client_secret', self.config.clientSecret),
562 urlencode([ 552 ('grant_type', 'refresh_token'),
563 ('refresh_token', self.config.refreshToken), 553 ('resource', 'https://graph.windows.net')
564 ('client_id', self.config.clientID), 554 ])
565 ('client_secret', self.config.clientSecret), 555 request = urllib2.Request(token_path, post_data)
566 ('grant_type', 'refresh_token'), 556 with contextlib.closing(opener.open(request)) as response:
567 ('resource', 'https://graph.windows.net') 557 data = json.load(response)
568 ]))) 558 auth_token = '{0[token_type]} {0[access_token]}'.format(data)
569 auth_token = '{0[token_type]} {0[access_token]}'.format(
570 token_response)
Sebastian Noack 2017/02/09 12:51:57 Nit: If you just call the variable "response", you
Oleksandr 2017/02/13 02:57:36 Done.
571 559
572 return auth_token 560 return auth_token
573 561
574 def upload_appx_file_to_windows_store(self, file_upload_url): 562 def upload_appx_file_to_windows_store(self, file_upload_url):
575
576 # Add .appx file to a .zip file 563 # Add .appx file to a .zip file
577 zip_path = os.path.splitext(self.path)[0] + '.zip' 564 zip_path = os.path.splitext(self.path)[0] + '.zip'
578 with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf: 565 with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
579 zf.write(self.path, os.path.basename(self.path)) 566 zf.write(self.path, os.path.basename(self.path))
580 567
581 # Upload that .zip file 568 # Upload that .zip file
582 file_upload_url = file_upload_url.replace('+', '%2B') 569 file_upload_url = file_upload_url.replace('+', '%2B')
583 parts = httplib.urlsplit(file_upload_url) 570 request = urllib2.Request(file_upload_url)
584 with contextlib.closing( 571 request.get_method = lambda: 'PUT'
585 httplib.HTTPSConnection(parts.netloc)) as file_upload_con: 572 request.add_header('x-ms-blob-type', 'BlockBlob')
586 file_headers = {'x-ms-blob-type': 'BlockBlob'} 573
587 file_upload_con.request('PUT', '{}?{}{}'.format( 574 opener = urllib2.build_opener(HTTPErrorBodyHandler)
Vasily Kuznetsov 2017/02/09 16:37:58 Maybe combining the url fragments on a separate li
Oleksandr 2017/02/13 02:57:36 Done.
588 parts.path, 575
589 parts.query, 576 with open(zip_path, 'rb') as file:
590 parts.fragment), 577 request.add_header('Content-Length',
591 open(zip_path, 'rb'), file_headers) 578 os.fstat(file.fileno()).st_size - file.tell())
592 file_upload_con.getresponse().read() 579 request.add_data(file)
593 580 opener.open(request).close()
581
582 # Clone the previous submission for the new one. Largely based on code
583 # from https://msdn.microsoft.com/en-us/windows/uwp/monetize/python-code-exa mples-for-the-windows-store-submission-api#create-an-app-submission
594 def upload_to_windows_store(self): 584 def upload_to_windows_store(self):
595 585 opener = urllib2.build_opener(HTTPErrorBodyHandler)
596 auth_token = self.get_windows_store_access_token() 586
597 587 headers = {'Authorization': self.get_windows_store_access_token(),
598 # Clone the previous submission for the new one. Largely based on code
599 # from https://msdn.microsoft.com/en-us/windows/uwp/monetize/python-code -examples-for-the-windows-store-submission-api#create-an-app-submission
600 headers = {'Authorization': auth_token,
601 'Content-type': 'application/json'} 588 'Content-type': 'application/json'}
602 589
603 api_server = 'manage.devcenter.microsoft.com' 590 # Get application
604 with contextlib.closing( 591 # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app
605 httplib.HTTPSConnection(api_server)) as connection: 592 api_path = '{}/v1.0/my/applications/{}'.format(
606 593 'https://manage.devcenter.microsoft.com',
607 # Get application 594 self.config.devbuildGalleryID
608 # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app 595 )
609 api_path = '/v1.0/my/applications/{}'.format( 596
610 self.config.devbuildGalleryID) 597 request = urllib2.Request(api_path, None, headers)
611 app_obj = json.loads(self.get_response(connection, 'GET', 598 with contextlib.closing(opener.open(request)) as response:
612 api_path, '', headers)) 599 app_obj = json.load(response)
613 600
614 # Delete existing in-progress submission 601 # Delete existing in-progress submission
615 # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-ap p-submission 602 # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-su bmission
616 submissions_path = api_path + '/submissions' 603 submissions_path = api_path + '/submissions'
617 if 'pendingApplicationSubmission' in app_obj: 604 if 'pendingApplicationSubmission' in app_obj:
618 remove_id = app_obj['pendingApplicationSubmission']['id'] 605 remove_id = app_obj['pendingApplicationSubmission']['id']
619 self.get_response(connection, 'DELETE', 606 remove_path = '{}/{}'.format(submissions_path, remove_id)
620 '%s/%s' % (submissions_path, remove_id), 607 request = urllib2.Request(remove_path, '', headers)
621 '', headers) 608 request.get_method = lambda: 'DELETE'
622 609 opener.open(request).close()
623 # Create submission 610
624 # https://msdn.microsoft.com/en-us/windows/uwp/monetize/create-an-ap p-submission 611 # Create submission
625 submission = json.loads(self.get_response( 612 # https://msdn.microsoft.com/en-us/windows/uwp/monetize/create-an-app-su bmission
626 connection, 'POST', 613 request = urllib2.Request(submissions_path, '', headers)
627 submissions_path, '', headers)) 614 request.get_method = lambda: 'POST'
628 615 with contextlib.closing(opener.open(request)) as response:
629 submission_id = submission['id'] 616 submission = json.load(response)
630 file_upload_url = submission['fileUploadUrl'] 617
631 618 submission_id = submission['id']
632 # Update submission 619 file_upload_url = submission['fileUploadUrl']
633 old_submission = submission['applicationPackages'][0] 620
634 old_submission['fileStatus'] = 'PendingDelete' 621 new_submission_path = '{}/{}'.format(submissions_path,
635 submission['applicationPackages'].append( 622 submission_id)
636 {'fileStatus': 'PendingUpload'}) 623
637 added_submission = submission['applicationPackages'][1] 624 request = urllib2.Request(new_submission_path, None, headers)
638 added_submission['fileName'] = os.path.basename(self.path) 625 opener.open(request).close()
639 626
640 old_min_sys_ram = old_submission['minimumSystemRam'] 627 self.upload_appx_file_to_windows_store(file_upload_url)
641 added_submission['minimumSystemRam'] = old_min_sys_ram 628
642 629 # Commit submission
643 old_directx_version = old_submission['minimumDirectXVersion'] 630 # https://msdn.microsoft.com/en-us/windows/uwp/monetize/commit-an-app-su bmission
644 added_submission['minimumDirectXVersion'] = old_directx_version 631 commit_path = '{}/commit'.format(new_submission_path)
645 632 request = urllib2.Request(commit_path, '', headers)
646 new_submission_path = '{}/{}'.format( 633 request.get_method = lambda: 'POST'
647 submissions_path, submission_id) 634 with contextlib.closing(opener.open(request)) as response:
648 635 submission = json.load(response)
649 self.get_response(connection, 'PUT', new_submission_path, 636
650 json.dumps(submission), headers) 637 if submission['status'] != 'CommitStarted':
651 638 raise Exception({'status': submission['status'],
652 self.upload_appx_file_to_windows_store(file_upload_url) 639 'statusDetails': submission['statusDetails']})
653
654 # Commit submission
655 # https://msdn.microsoft.com/en-us/windows/uwp/monetize/commit-an-ap p-submission
656 submission = json.loads(self.get_response(connection, 'POST',
657 new_submission_path + '/commit',
658 '', headers))
659
660 if submission['status'] != 'CommitStarted':
661 raise Exception({'status': submission['status'],
662 'statusDetails': submission['statusDetails']})
663 640
664 def run(self): 641 def run(self):
665 """ 642 """
666 Run the nightly build process for one extension 643 Run the nightly build process for one extension
667 """ 644 """
668 try: 645 try:
669 if self.config.type == 'ie': 646 if self.config.type == 'ie':
670 # We cannot build IE builds, simply list the builds already in 647 # We cannot build IE builds, simply list the builds already in
671 # the directory. Basename has to be deduced from the repository name. 648 # the directory. Basename has to be deduced from the repository name.
672 self.basename = os.path.basename(self.config.repository) 649 self.basename = os.path.basename(self.config.repository)
(...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after
747 except Exception as ex: 724 except Exception as ex:
748 logging.error('The build for %s failed:', repo) 725 logging.error('The build for %s failed:', repo)
749 logging.exception(ex) 726 logging.exception(ex)
750 727
751 file = open(nightlyConfigFile, 'wb') 728 file = open(nightlyConfigFile, 'wb')
752 nightlyConfig.write(file) 729 nightlyConfig.write(file)
753 730
754 731
755 if __name__ == '__main__': 732 if __name__ == '__main__':
756 main() 733 main()
LEFTRIGHT

Powered by Google App Engine
This is Rietveld