Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/createNightlies.py

Issue 29751598: Issue 6291 - Use client certificate for Windows Store uploads (Closed) Base URL: https://hg.adblockplus.org/abpssembly/file/a67d8f0e66b2
Patch Set: Created April 16, 2018, 4:36 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | sitescripts/extensions/utils.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-present eyeo GmbH 2 # Copyright (C) 2006-present eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 """ 16 """
17 17
18 Nightly builds generation script 18 Nightly builds generation script
19 ================================ 19 ================================
20 20
21 This script generates nightly builds of extensions, together 21 This script generates nightly builds of extensions, together
22 with changelogs and documentation. 22 with changelogs and documentation.
23 23
24 """ 24 """
25 25
26 import argparse 26 import argparse
27 import ConfigParser 27 import ConfigParser
28 import binascii
28 import base64 29 import base64
29 import hashlib 30 import hashlib
30 import hmac 31 import hmac
31 import json 32 import json
32 import logging 33 import logging
33 import os 34 import os
34 import pipes 35 import pipes
35 import random
36 import shutil 36 import shutil
37 import struct 37 import struct
38 import subprocess 38 import subprocess
39 import sys 39 import sys
40 import tempfile 40 import tempfile
41 import time 41 import time
42 import uuid
42 from urllib import urlencode 43 from urllib import urlencode
43 import urllib2 44 import urllib2
44 import urlparse 45 import urlparse
45 import zipfile 46 import zipfile
46 import contextlib 47 import contextlib
47 48
49 from Crypto.PublicKey import RSA
50 from Crypto.Signature import PKCS1_v1_5
51 import Crypto.Hash.SHA256
52
48 from xml.dom.minidom import parse as parseXml 53 from xml.dom.minidom import parse as parseXml
49 54
50 from sitescripts.extensions.utils import ( 55 from sitescripts.extensions.utils import (
51 compareVersions, Configuration, 56 compareVersions, Configuration,
52 writeAndroidUpdateManifest 57 writeAndroidUpdateManifest
53 ) 58 )
54 from sitescripts.utils import get_config, get_template 59 from sitescripts.utils import get_config, get_template
55 60
56 MAX_BUILDS = 50 61 MAX_BUILDS = 50
57 62
(...skipping 285 matching lines...) Expand 10 before | Expand all | Expand 10 after
343 if os.path.exists(self.path): 348 if os.path.exists(self.path):
344 os.remove(self.path) 349 os.remove(self.path)
345 raise 350 raise
346 else: 351 else:
347 env = os.environ 352 env = os.environ
348 spiderMonkeyBinary = self.config.spiderMonkeyBinary 353 spiderMonkeyBinary = self.config.spiderMonkeyBinary
349 if spiderMonkeyBinary: 354 if spiderMonkeyBinary:
350 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary) 355 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
351 356
352 command = [os.path.join(self.tempdir, 'build.py')] 357 command = [os.path.join(self.tempdir, 'build.py')]
353 if self.config.type == 'safari': 358 command.extend(['build', '-t', self.config.type, '-b',
354 command.extend(['-t', self.config.type, 'build']) 359 self.buildNum])
355 else:
356 command.extend(['build', '-t', self.config.type])
357 command.extend(['-b', self.buildNum])
358 360
359 if self.config.type not in {'gecko', 'edge'}: 361 if self.config.type not in {'gecko', 'edge'}:
360 command.extend(['-k', self.config.keyFile]) 362 command.extend(['-k', self.config.keyFile])
361 command.append(self.path) 363 command.append(self.path)
362 subprocess.check_call(command, env=env) 364 subprocess.check_call(command, env=env)
363 365
364 if not os.path.exists(self.path): 366 if not os.path.exists(self.path):
365 raise Exception("Build failed, output file hasn't been created") 367 raise Exception("Build failed, output file hasn't been created")
366 368
367 if self.config.type not in self.downloadable_repos: 369 if self.config.type not in self.downloadable_repos:
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after
450 try: 452 try:
451 for i, entry in enumerate(current[platform]): 453 for i, entry in enumerate(current[platform]):
452 if entry[filter_key] == filter_value: 454 if entry[filter_key] == filter_value:
453 del current[platform][i] 455 del current[platform][i]
454 if len(current[platform]) == 0: 456 if len(current[platform]) == 0:
455 del current[platform] 457 del current[platform]
456 except KeyError: 458 except KeyError:
457 pass 459 pass
458 self.write_downloads_lockfile(current) 460 self.write_downloads_lockfile(current)
459 461
460 def generate_jwt_request(self, issuer, secret, url, method, data=None, 462 def azure_jwt_signature_fnc(self):
461 add_headers=[]): 463 return (
462 header = { 464 'RS256',
463 'alg': 'HS256', # HMAC-SHA256 465 lambda s, m: PKCS1_v1_5.new(s).sign(Crypto.Hash.SHA256.new(m))
464 'typ': 'JWT', 466 )
467
468 def mozilla_jwt_signature_fnc(self):
469 return (
470 'HS256',
471 lambda s, m: hmac.new(s, msg=m, digestmod=hashlib.sha256).digest()
472 )
473
474 def sign_jwt(self, issuer, secret, url, signature_fnc, jwt_headers={}):
475 alg, fnc = signature_fnc()
476
477 header = {'typ': 'JWT'}
478 header.update(jwt_headers)
479 header.update({'alg': alg})
480
481 issued = int(time.mktime(time.localtime()))
482 expires = issued + 60
483
484 payload = {
485 'aud': url,
486 'iss': issuer,
487 'sub': issuer,
488 'jti': str(uuid.uuid4()),
489 'iat': issued,
490 'nbf': issued,
491 'exp': expires,
465 } 492 }
466 493
467 issued = int(time.time()) 494 segments = [base64.urlsafe_b64encode(json.dumps(header)),
468 payload = { 495 base64.urlsafe_b64encode(json.dumps(payload))]
469 'iss': issuer,
470 'jti': random.random(),
471 'iat': issued,
472 'exp': issued + 60,
473 }
474 496
475 hmac_data = '{}.{}'.format( 497 signature = fnc(secret, b'.'.join(segments))
476 base64.b64encode(json.dumps(header)), 498 segments.append(base64.urlsafe_b64encode(signature))
477 base64.b64encode(json.dumps(payload)) 499 return b'.'.join(segments)
478 )
479 500
480 signature = hmac.new(secret, msg=hmac_data, 501 def generate_mozilla_jwt_request(self, issuer, secret, url, method,
481 digestmod=hashlib.sha256).digest() 502 data=None, add_headers=[]):
482 token = '{}.{}'.format(hmac_data, base64.b64encode(signature)) 503 signed = self.sign_jwt(issuer, secret, url,
504 self.mozilla_jwt_signature_fnc)
483 505
484 request = urllib2.Request(url, data) 506 request = urllib2.Request(url, data)
485 request.add_header('Authorization', 'JWT ' + token) 507 request.add_header('Authorization', 'JWT ' + signed)
486 for header in add_headers: 508 for header in add_headers:
487 request.add_header(*header) 509 request.add_header(*header)
488 request.get_method = lambda: method 510 request.get_method = lambda: method
489 511
490 return request 512 return request
491 513
492 def uploadToMozillaAddons(self): 514 def uploadToMozillaAddons(self):
493 import urllib3 515 import urllib3
494 516
495 config = get_config() 517 config = get_config()
496 518
497 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/' 519 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
498 'versions/{}/').format(self.extensionID, self.version) 520 'versions/{}/').format(self.extensionID, self.version)
499 521
500 with open(self.path, 'rb') as file: 522 with open(self.path, 'rb') as file:
501 data, content_type = urllib3.filepost.encode_multipart_formdata({ 523 data, content_type = urllib3.filepost.encode_multipart_formdata({
502 'upload': ( 524 'upload': (
503 os.path.basename(self.path), 525 os.path.basename(self.path),
504 file.read(), 526 file.read(),
505 'application/x-xpinstall' 527 'application/x-xpinstall'
506 ) 528 )
507 }) 529 })
508 530
509 request = self.generate_jwt_request( 531 request = self.generate_mozilla_jwt_request(
510 config.get('extensions', 'amo_key'), 532 config.get('extensions', 'amo_key'),
511 config.get('extensions', 'amo_secret'), 533 config.get('extensions', 'amo_secret'),
512 upload_url, 534 upload_url,
513 'PUT', 535 'PUT',
514 data, 536 data,
515 [('Content-Type', content_type)] 537 [('Content-Type', content_type)],
516 ) 538 )
517 539
518 try: 540 try:
519 urllib2.urlopen(request).close() 541 urllib2.urlopen(request).close()
520 except urllib2.HTTPError as e: 542 except urllib2.HTTPError as e:
521 try: 543 try:
522 logging.error(e.read()) 544 logging.error(e.read())
523 finally: 545 finally:
524 e.close() 546 e.close()
525 raise 547 raise
526 548
527 self.add_to_downloads_lockfile( 549 self.add_to_downloads_lockfile(
528 self.config.type, 550 self.config.type,
529 { 551 {
530 'buildtype': 'devbuild', 552 'buildtype': 'devbuild',
531 'app_id': self.extensionID, 553 'app_id': self.extensionID,
532 'version': self.version, 554 'version': self.version,
533 } 555 }
534 ) 556 )
535 os.remove(self.path) 557 os.remove(self.path)
536 558
537 def download_from_mozilla_addons(self, buildtype, version, app_id): 559 def download_from_mozilla_addons(self, buildtype, version, app_id):
538 config = get_config() 560 config = get_config()
539 iss = config.get('extensions', 'amo_key') 561 iss = config.get('extensions', 'amo_key')
540 secret = config.get('extensions', 'amo_secret') 562 secret = config.get('extensions', 'amo_secret')
541 563
542 url = ('https://addons.mozilla.org/api/v3/addons/{}/' 564 url = ('https://addons.mozilla.org/api/v3/addons/{}/'
543 'versions/{}/').format(app_id, version) 565 'versions/{}/').format(app_id, version)
544 566
545 request = self.generate_jwt_request(iss, secret, url, 'GET') 567 request = self.generate_mozilla_jwt_request(
568 iss, secret, url, 'GET',
569 )
546 response = json.load(urllib2.urlopen(request)) 570 response = json.load(urllib2.urlopen(request))
547 571
548 filename = '{}-{}.xpi'.format(self.basename, version) 572 filename = '{}-{}.xpi'.format(self.basename, version)
549 self.path = os.path.join( 573 self.path = os.path.join(
550 config.get('extensions', 'nightliesDirectory'), 574 config.get('extensions', 'nightliesDirectory'),
551 self.basename, 575 self.basename,
552 filename 576 filename
553 ) 577 )
554 578
555 necessary = ['passed_review', 'reviewed', 'processed', 'valid'] 579 necessary = ['passed_review', 'reviewed', 'processed', 'valid']
556 if all(response[x] for x in necessary): 580 if all(response[x] for x in necessary):
557 download_url = response['files'][0]['download_url'] 581 download_url = response['files'][0]['download_url']
558 checksum = response['files'][0]['hash'] 582 checksum = response['files'][0]['hash']
559 583
560 request = self.generate_jwt_request(iss, secret, download_url, 584 request = self.generate_mozilla_jwt_request(
561 'GET') 585 iss, secret, download_url, 'GET',
586 )
562 try: 587 try:
563 response = urllib2.urlopen(request) 588 response = urllib2.urlopen(request)
564 except urllib2.HTTPError as e: 589 except urllib2.HTTPError as e:
565 logging.error(e.read()) 590 logging.error(e.read())
566 591
567 # Verify the extension's integrity 592 # Verify the extension's integrity
568 file_content = response.read() 593 file_content = response.read()
569 sha256 = hashlib.sha256(file_content) 594 sha256 = hashlib.sha256(file_content)
570 returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest()) 595 returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest())
571 596
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after
642 request.get_method = lambda: 'POST' 667 request.get_method = lambda: 'POST'
643 request.add_header('Authorization', auth_token) 668 request.add_header('Authorization', auth_token)
644 request.add_header('x-goog-api-version', '2') 669 request.add_header('x-goog-api-version', '2')
645 request.add_header('Content-Length', '0') 670 request.add_header('Content-Length', '0')
646 671
647 response = json.load(opener.open(request)) 672 response = json.load(opener.open(request))
648 673
649 if any(status not in ('OK', 'ITEM_PENDING_REVIEW') for status in respons e['status']): 674 if any(status not in ('OK', 'ITEM_PENDING_REVIEW') for status in respons e['status']):
650 raise Exception({'status': response['status'], 'statusDetail': respo nse['statusDetail']}) 675 raise Exception({'status': response['status'], 'statusDetail': respo nse['statusDetail']})
651 676
677 def generate_certificate_token_request(self, url, private_key):
678 # Construct the token request according to
679 # https://docs.microsoft.com/en-us/azure/active-directory/develop/active -directory-certificate-credentials
680 hex_val = binascii.a2b_hex(self.config.thumbprint)
681 x5t = base64.urlsafe_b64encode(hex_val).decode()
682
683 key = RSA.importKey(private_key)
684
685 signed = self.sign_jwt(self.config.clientID, key, url,
686 self.azure_jwt_signature_fnc,
687 jwt_headers={'x5t': x5t})
688
689 # generate oauth parameters for login.microsoft.com
690 oauth_params = {
691 'grant_type': 'client_credentials',
692 'client_id': self.config.clientID,
693 'resource': 'https://graph.windows.net',
694 'client_assertion_type': 'urn:ietf:params:oauth:client-assertion-'
695 'type:jwt-bearer',
696 'client_assertion': signed,
697 }
698
699 request = urllib2.Request(url, urlencode(oauth_params))
700 request.get_method = lambda: 'POST'
701
702 return request
703
652 def get_windows_store_access_token(self): 704 def get_windows_store_access_token(self):
653 # use refresh token to obtain a valid access token 705 # use client certificate to obtain a valid access token
654 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo ry-protocols-oauth-code#refreshing-the-access-tokens 706 url = 'https://login.microsoftonline.com/{}/oauth2/token'.format(
655 server = 'https://login.microsoftonline.com' 707 self.config.tenantID
656 token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID) 708 )
709
710 with open(self.config.privateKey, 'r') as fp:
711 private_key = fp.read()
657 712
658 opener = urllib2.build_opener(HTTPErrorBodyHandler) 713 opener = urllib2.build_opener(HTTPErrorBodyHandler)
659 post_data = urlencode([ 714 request = self.generate_certificate_token_request(url, private_key)
660 ('refresh_token', self.config.refreshToken), 715
661 ('client_id', self.config.clientID),
662 ('client_secret', self.config.clientSecret),
663 ('grant_type', 'refresh_token'),
664 ('resource', 'https://graph.windows.net')
665 ])
666 request = urllib2.Request(token_path, post_data)
667 with contextlib.closing(opener.open(request)) as response: 716 with contextlib.closing(opener.open(request)) as response:
668 data = json.load(response) 717 data = json.load(response)
669 auth_token = '{0[token_type]} {0[access_token]}'.format(data) 718 auth_token = '{0[token_type]} {0[access_token]}'.format(data)
670 719
671 return auth_token 720 return auth_token
672 721
673 def upload_appx_file_to_windows_store(self, file_upload_url): 722 def upload_appx_file_to_windows_store(self, file_upload_url):
674 # Add .appx file to a .zip file 723 # Add .appx file to a .zip file
675 zip_path = os.path.splitext(self.path)[0] + '.zip' 724 zip_path = os.path.splitext(self.path)[0] + '.zip'
676 with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf: 725 with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
(...skipping 130 matching lines...) Expand 10 before | Expand all | Expand 10 after
807 856
808 # update nightlies config 857 # update nightlies config
809 self.config.latestRevision = self.revision 858 self.config.latestRevision = self.revision
810 859
811 if (self.config.type == 'gecko' and 860 if (self.config.type == 'gecko' and
812 self.config.galleryID and 861 self.config.galleryID and
813 get_config().has_option('extensions', 'amo_key')): 862 get_config().has_option('extensions', 'amo_key')):
814 self.uploadToMozillaAddons() 863 self.uploadToMozillaAddons()
815 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken: 864 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken:
816 self.uploadToChromeWebStore() 865 self.uploadToChromeWebStore()
817 elif self.config.type == 'edge' and self.config.clientID and self.co nfig.clientSecret and self.config.refreshToken and self.config.tenantID: 866 elif self.config.type == 'edge' and self.config.clientID and self.co nfig.tenantID and self.config.privateKey and self.config.thumbprint:
818 self.upload_to_windows_store() 867 self.upload_to_windows_store()
819 868
820 finally: 869 finally:
821 # clean up 870 # clean up
822 if self.tempdir: 871 if self.tempdir:
823 shutil.rmtree(self.tempdir, ignore_errors=True) 872 shutil.rmtree(self.tempdir, ignore_errors=True)
824 873
825 def download(self): 874 def download(self):
826 download_info = self.read_downloads_lockfile() 875 download_info = self.read_downloads_lockfile()
827 downloads = self.downloadable_repos.intersection(download_info.keys()) 876 downloads = self.downloadable_repos.intersection(download_info.keys())
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after
889 938
890 file = open(nightlyConfigFile, 'wb') 939 file = open(nightlyConfigFile, 'wb')
891 nightlyConfig.write(file) 940 nightlyConfig.write(file)
892 941
893 942
894 if __name__ == '__main__': 943 if __name__ == '__main__':
895 parser = argparse.ArgumentParser() 944 parser = argparse.ArgumentParser()
896 parser.add_argument('--download', action='store_true', default=False) 945 parser.add_argument('--download', action='store_true', default=False)
897 args = parser.parse_args() 946 args = parser.parse_args()
898 main(args.download) 947 main(args.download)
OLDNEW
« no previous file with comments | « no previous file | sitescripts/extensions/utils.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld