Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/createNightlies.py

Issue 29751598: Issue 6291 - Use client certificate for Windows Store uploads (Closed) Base URL: https://hg.adblockplus.org/abpssembly/file/a67d8f0e66b2
Patch Set: Addressing non-functional nits Created April 18, 2018, 12:57 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | sitescripts/extensions/utils.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-present eyeo GmbH 2 # Copyright (C) 2006-present eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 """ 16 """
17 17
18 Nightly builds generation script 18 Nightly builds generation script
19 ================================ 19 ================================
20 20
21 This script generates nightly builds of extensions, together 21 This script generates nightly builds of extensions, together
22 with changelogs and documentation. 22 with changelogs and documentation.
23 23
24 """ 24 """
25 25
26 import argparse 26 import argparse
27 import ConfigParser 27 import ConfigParser
28 import binascii
28 import base64 29 import base64
29 import hashlib 30 import hashlib
30 import hmac 31 import hmac
31 import json 32 import json
32 import logging 33 import logging
33 import os 34 import os
34 import pipes 35 import pipes
35 import random
36 import shutil 36 import shutil
37 import struct 37 import struct
38 import subprocess 38 import subprocess
39 import sys 39 import sys
40 import tempfile 40 import tempfile
41 import time 41 import time
42 import uuid
42 from urllib import urlencode 43 from urllib import urlencode
43 import urllib2 44 import urllib2
44 import urlparse 45 import urlparse
45 import zipfile 46 import zipfile
46 import contextlib 47 import contextlib
48 from xml.dom.minidom import parse as parseXml
47 49
48 from xml.dom.minidom import parse as parseXml 50 from Crypto.PublicKey import RSA
51 from Crypto.Signature import PKCS1_v1_5
52 import Crypto.Hash.SHA256
49 53
50 from sitescripts.extensions.utils import ( 54 from sitescripts.extensions.utils import (
51 compareVersions, Configuration, 55 compareVersions, Configuration,
52 writeAndroidUpdateManifest 56 writeAndroidUpdateManifest
53 ) 57 )
54 from sitescripts.utils import get_config, get_template 58 from sitescripts.utils import get_config, get_template
55 59
56 MAX_BUILDS = 50 60 MAX_BUILDS = 50
57 61
58 62
(...skipping 388 matching lines...) Expand 10 before | Expand all | Expand 10 after
447 try: 451 try:
448 for i, entry in enumerate(current[platform]): 452 for i, entry in enumerate(current[platform]):
449 if entry[filter_key] == filter_value: 453 if entry[filter_key] == filter_value:
450 del current[platform][i] 454 del current[platform][i]
451 if len(current[platform]) == 0: 455 if len(current[platform]) == 0:
452 del current[platform] 456 del current[platform]
453 except KeyError: 457 except KeyError:
454 pass 458 pass
455 self.write_downloads_lockfile(current) 459 self.write_downloads_lockfile(current)
456 460
457 def generate_jwt_request(self, issuer, secret, url, method, data=None, 461 def azure_jwt_signature_fnc(self):
458 add_headers=[]): 462 return (
459 header = { 463 'RS256',
460 'alg': 'HS256', # HMAC-SHA256 464 lambda s, m: PKCS1_v1_5.new(s).sign(Crypto.Hash.SHA256.new(m)),
461 'typ': 'JWT', 465 )
466
467 def mozilla_jwt_signature_fnc(self):
468 return (
469 'HS256',
470 lambda s, m: hmac.new(s, msg=m, digestmod=hashlib.sha256).digest(),
471 )
472
473 def sign_jwt(self, issuer, secret, url, signature_fnc, jwt_headers={}):
474 alg, fnc = signature_fnc()
475
476 header = {'typ': 'JWT'}
477 header.update(jwt_headers)
478 header.update({'alg': alg})
479
480 issued = int(time.time())
481 expires = issued + 60
482
483 payload = {
484 'aud': url,
485 'iss': issuer,
486 'sub': issuer,
487 'jti': str(uuid.uuid4()),
488 'iat': issued,
489 'nbf': issued,
490 'exp': expires,
462 } 491 }
463 492
464 issued = int(time.time()) 493 segments = [base64.urlsafe_b64encode(json.dumps(header)),
465 payload = { 494 base64.urlsafe_b64encode(json.dumps(payload))]
466 'iss': issuer,
467 'jti': random.random(),
468 'iat': issued,
469 'exp': issued + 60,
470 }
471 495
472 hmac_data = '{}.{}'.format( 496 signature = fnc(secret, b'.'.join(segments))
473 base64.b64encode(json.dumps(header)), 497 segments.append(base64.urlsafe_b64encode(signature))
474 base64.b64encode(json.dumps(payload)) 498 return b'.'.join(segments)
475 )
476 499
477 signature = hmac.new(secret, msg=hmac_data, 500 def generate_mozilla_jwt_request(self, issuer, secret, url, method,
478 digestmod=hashlib.sha256).digest() 501 data=None, add_headers=[]):
479 token = '{}.{}'.format(hmac_data, base64.b64encode(signature)) 502 signed = self.sign_jwt(issuer, secret, url,
503 self.mozilla_jwt_signature_fnc)
480 504
481 request = urllib2.Request(url, data) 505 request = urllib2.Request(url, data)
482 request.add_header('Authorization', 'JWT ' + token) 506 request.add_header('Authorization', 'JWT ' + signed)
483 for header in add_headers: 507 for header in add_headers:
484 request.add_header(*header) 508 request.add_header(*header)
485 request.get_method = lambda: method 509 request.get_method = lambda: method
486 510
487 return request 511 return request
488 512
489 def uploadToMozillaAddons(self): 513 def uploadToMozillaAddons(self):
490 import urllib3 514 import urllib3
491 515
492 config = get_config() 516 config = get_config()
493 517
494 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/' 518 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
495 'versions/{}/').format(self.extensionID, self.version) 519 'versions/{}/').format(self.extensionID, self.version)
496 520
497 with open(self.path, 'rb') as file: 521 with open(self.path, 'rb') as file:
498 data, content_type = urllib3.filepost.encode_multipart_formdata({ 522 data, content_type = urllib3.filepost.encode_multipart_formdata({
499 'upload': ( 523 'upload': (
500 os.path.basename(self.path), 524 os.path.basename(self.path),
501 file.read(), 525 file.read(),
502 'application/x-xpinstall' 526 'application/x-xpinstall'
503 ) 527 )
504 }) 528 })
505 529
506 request = self.generate_jwt_request( 530 request = self.generate_mozilla_jwt_request(
507 config.get('extensions', 'amo_key'), 531 config.get('extensions', 'amo_key'),
508 config.get('extensions', 'amo_secret'), 532 config.get('extensions', 'amo_secret'),
509 upload_url, 533 upload_url,
510 'PUT', 534 'PUT',
511 data, 535 data,
512 [('Content-Type', content_type)] 536 [('Content-Type', content_type)],
513 ) 537 )
514 538
515 try: 539 try:
516 urllib2.urlopen(request).close() 540 urllib2.urlopen(request).close()
517 except urllib2.HTTPError as e: 541 except urllib2.HTTPError as e:
518 try: 542 try:
519 logging.error(e.read()) 543 logging.error(e.read())
520 finally: 544 finally:
521 e.close() 545 e.close()
522 raise 546 raise
523 547
524 self.add_to_downloads_lockfile( 548 self.add_to_downloads_lockfile(
525 self.config.type, 549 self.config.type,
526 { 550 {
527 'buildtype': 'devbuild', 551 'buildtype': 'devbuild',
528 'app_id': self.extensionID, 552 'app_id': self.extensionID,
529 'version': self.version, 553 'version': self.version,
530 } 554 }
531 ) 555 )
532 os.remove(self.path) 556 os.remove(self.path)
533 557
534 def download_from_mozilla_addons(self, buildtype, version, app_id): 558 def download_from_mozilla_addons(self, buildtype, version, app_id):
535 config = get_config() 559 config = get_config()
536 iss = config.get('extensions', 'amo_key') 560 iss = config.get('extensions', 'amo_key')
537 secret = config.get('extensions', 'amo_secret') 561 secret = config.get('extensions', 'amo_secret')
538 562
539 url = ('https://addons.mozilla.org/api/v3/addons/{}/' 563 url = ('https://addons.mozilla.org/api/v3/addons/{}/'
540 'versions/{}/').format(app_id, version) 564 'versions/{}/').format(app_id, version)
541 565
542 request = self.generate_jwt_request(iss, secret, url, 'GET') 566 request = self.generate_mozilla_jwt_request(
567 iss, secret, url, 'GET',
568 )
543 response = json.load(urllib2.urlopen(request)) 569 response = json.load(urllib2.urlopen(request))
544 570
545 filename = '{}-{}.xpi'.format(self.basename, version) 571 filename = '{}-{}.xpi'.format(self.basename, version)
546 self.path = os.path.join( 572 self.path = os.path.join(
547 config.get('extensions', 'nightliesDirectory'), 573 config.get('extensions', 'nightliesDirectory'),
548 self.basename, 574 self.basename,
549 filename 575 filename
550 ) 576 )
551 577
552 necessary = ['passed_review', 'reviewed', 'processed', 'valid'] 578 necessary = ['passed_review', 'reviewed', 'processed', 'valid']
553 if all(response[x] for x in necessary): 579 if all(response[x] for x in necessary):
554 download_url = response['files'][0]['download_url'] 580 download_url = response['files'][0]['download_url']
555 checksum = response['files'][0]['hash'] 581 checksum = response['files'][0]['hash']
556 582
557 request = self.generate_jwt_request(iss, secret, download_url, 583 request = self.generate_mozilla_jwt_request(
558 'GET') 584 iss, secret, download_url, 'GET',
585 )
559 try: 586 try:
560 response = urllib2.urlopen(request) 587 response = urllib2.urlopen(request)
561 except urllib2.HTTPError as e: 588 except urllib2.HTTPError as e:
562 logging.error(e.read()) 589 logging.error(e.read())
563 590
564 # Verify the extension's integrity 591 # Verify the extension's integrity
565 file_content = response.read() 592 file_content = response.read()
566 sha256 = hashlib.sha256(file_content) 593 sha256 = hashlib.sha256(file_content)
567 returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest()) 594 returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest())
568 595
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after
639 request.get_method = lambda: 'POST' 666 request.get_method = lambda: 'POST'
640 request.add_header('Authorization', auth_token) 667 request.add_header('Authorization', auth_token)
641 request.add_header('x-goog-api-version', '2') 668 request.add_header('x-goog-api-version', '2')
642 request.add_header('Content-Length', '0') 669 request.add_header('Content-Length', '0')
643 670
644 response = json.load(opener.open(request)) 671 response = json.load(opener.open(request))
645 672
646 if any(status not in ('OK', 'ITEM_PENDING_REVIEW') for status in respons e['status']): 673 if any(status not in ('OK', 'ITEM_PENDING_REVIEW') for status in respons e['status']):
647 raise Exception({'status': response['status'], 'statusDetail': respo nse['statusDetail']}) 674 raise Exception({'status': response['status'], 'statusDetail': respo nse['statusDetail']})
648 675
676 def generate_certificate_token_request(self, url, private_key):
677 # Construct the token request according to
678 # https://docs.microsoft.com/en-us/azure/active-directory/develop/active -directory-certificate-credentials
679 hex_val = binascii.a2b_hex(self.config.thumbprint)
680 x5t = base64.urlsafe_b64encode(hex_val).decode()
681
682 key = RSA.importKey(private_key)
683
684 signed = self.sign_jwt(self.config.clientID, key, url,
685 self.azure_jwt_signature_fnc,
686 jwt_headers={'x5t': x5t})
687
688 # generate oauth parameters for login.microsoft.com
689 oauth_params = {
690 'grant_type': 'client_credentials',
691 'client_id': self.config.clientID,
692 'resource': 'https://graph.windows.net',
693 'client_assertion_type': 'urn:ietf:params:oauth:client-assertion-'
694 'type:jwt-bearer',
695 'client_assertion': signed,
696 }
697
698 request = urllib2.Request(url, urlencode(oauth_params))
699 request.get_method = lambda: 'POST'
700
701 return request
702
649 def get_windows_store_access_token(self): 703 def get_windows_store_access_token(self):
650 # use refresh token to obtain a valid access token 704 # use client certificate to obtain a valid access token
651 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo ry-protocols-oauth-code#refreshing-the-access-tokens 705 url_template = 'https://login.microsoftonline.com/{}/oauth2/token'
652 server = 'https://login.microsoftonline.com' 706 url = url_template.format(self.config.tenantID)
653 token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID) 707
708 with open(self.config.privateKey, 'r') as fp:
709 private_key = fp.read()
654 710
655 opener = urllib2.build_opener(HTTPErrorBodyHandler) 711 opener = urllib2.build_opener(HTTPErrorBodyHandler)
656 post_data = urlencode([ 712 request = self.generate_certificate_token_request(url, private_key)
657 ('refresh_token', self.config.refreshToken), 713
658 ('client_id', self.config.clientID),
659 ('client_secret', self.config.clientSecret),
660 ('grant_type', 'refresh_token'),
661 ('resource', 'https://graph.windows.net')
662 ])
663 request = urllib2.Request(token_path, post_data)
664 with contextlib.closing(opener.open(request)) as response: 714 with contextlib.closing(opener.open(request)) as response:
665 data = json.load(response) 715 data = json.load(response)
666 auth_token = '{0[token_type]} {0[access_token]}'.format(data) 716 auth_token = '{0[token_type]} {0[access_token]}'.format(data)
667 717
668 return auth_token 718 return auth_token
669 719
670 def upload_appx_file_to_windows_store(self, file_upload_url): 720 def upload_appx_file_to_windows_store(self, file_upload_url):
671 # Add .appx file to a .zip file 721 # Add .appx file to a .zip file
672 zip_path = os.path.splitext(self.path)[0] + '.zip' 722 zip_path = os.path.splitext(self.path)[0] + '.zip'
673 with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf: 723 with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
(...skipping 130 matching lines...) Expand 10 before | Expand all | Expand 10 after
804 854
805 # update nightlies config 855 # update nightlies config
806 self.config.latestRevision = self.revision 856 self.config.latestRevision = self.revision
807 857
808 if (self.config.type == 'gecko' and 858 if (self.config.type == 'gecko' and
809 self.config.galleryID and 859 self.config.galleryID and
810 get_config().has_option('extensions', 'amo_key')): 860 get_config().has_option('extensions', 'amo_key')):
811 self.uploadToMozillaAddons() 861 self.uploadToMozillaAddons()
812 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken: 862 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken:
813 self.uploadToChromeWebStore() 863 self.uploadToChromeWebStore()
814 elif self.config.type == 'edge' and self.config.clientID and self.co nfig.clientSecret and self.config.refreshToken and self.config.tenantID: 864 elif self.config.type == 'edge' and self.config.clientID and self.co nfig.tenantID and self.config.privateKey and self.config.thumbprint:
815 self.upload_to_windows_store() 865 self.upload_to_windows_store()
816 866
817 finally: 867 finally:
818 # clean up 868 # clean up
819 if self.tempdir: 869 if self.tempdir:
820 shutil.rmtree(self.tempdir, ignore_errors=True) 870 shutil.rmtree(self.tempdir, ignore_errors=True)
821 871
822 def download(self): 872 def download(self):
823 download_info = self.read_downloads_lockfile() 873 download_info = self.read_downloads_lockfile()
824 downloads = self.downloadable_repos.intersection(download_info.keys()) 874 downloads = self.downloadable_repos.intersection(download_info.keys())
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after
886 936
887 file = open(nightlyConfigFile, 'wb') 937 file = open(nightlyConfigFile, 'wb')
888 nightlyConfig.write(file) 938 nightlyConfig.write(file)
889 939
890 940
891 if __name__ == '__main__': 941 if __name__ == '__main__':
892 parser = argparse.ArgumentParser() 942 parser = argparse.ArgumentParser()
893 parser.add_argument('--download', action='store_true', default=False) 943 parser.add_argument('--download', action='store_true', default=False)
894 args = parser.parse_args() 944 args = parser.parse_args()
895 main(args.download) 945 main(args.download)
OLDNEW
« no previous file with comments | « no previous file | sitescripts/extensions/utils.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld