| OLD | NEW |
| 1 # This file is part of the Adblock Plus web scripts, | 1 # This file is part of the Adblock Plus web scripts, |
| 2 # Copyright (C) 2006-present eyeo GmbH | 2 # Copyright (C) 2006-present eyeo GmbH |
| 3 # | 3 # |
| 4 # Adblock Plus is free software: you can redistribute it and/or modify | 4 # Adblock Plus is free software: you can redistribute it and/or modify |
| 5 # it under the terms of the GNU General Public License version 3 as | 5 # it under the terms of the GNU General Public License version 3 as |
| 6 # published by the Free Software Foundation. | 6 # published by the Free Software Foundation. |
| 7 # | 7 # |
| 8 # Adblock Plus is distributed in the hope that it will be useful, | 8 # Adblock Plus is distributed in the hope that it will be useful, |
| 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| (...skipping 30 matching lines...) Expand all Loading... |
| 41 from urllib import urlencode | 41 from urllib import urlencode |
| 42 import urllib2 | 42 import urllib2 |
| 43 import urlparse | 43 import urlparse |
| 44 import zipfile | 44 import zipfile |
| 45 import contextlib | 45 import contextlib |
| 46 | 46 |
| 47 from xml.dom.minidom import parse as parseXml | 47 from xml.dom.minidom import parse as parseXml |
| 48 | 48 |
| 49 from sitescripts.extensions.utils import ( | 49 from sitescripts.extensions.utils import ( |
| 50 compareVersions, Configuration, | 50 compareVersions, Configuration, |
| 51 writeAndroidUpdateManifest | 51 writeAndroidUpdateManifest, |
| 52 ) | 52 ) |
| 53 from sitescripts.utils import get_config, get_template | 53 from sitescripts.utils import get_config, get_template |
| 54 | 54 |
| 55 MAX_BUILDS = 50 | 55 MAX_BUILDS = 50 |
| 56 | 56 |
| 57 | 57 |
| 58 # Google and Microsoft APIs use HTTP error codes with error message in | 58 # Google and Microsoft APIs use HTTP error codes with error message in |
| 59 # body. So we add the response body to the HTTPError to get more | 59 # body. So we add the response body to the HTTPError to get more |
| 60 # meaningful error messages. | 60 # meaningful error messages. |
| 61 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler): | 61 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler): |
| (...skipping 26 matching lines...) Expand all Loading... |
| 88 | 88 |
| 89 def hasChanges(self): | 89 def hasChanges(self): |
| 90 return self.revision != self.previousRevision | 90 return self.revision != self.previousRevision |
| 91 | 91 |
| 92 def getCurrentRevision(self): | 92 def getCurrentRevision(self): |
| 93 """ | 93 """ |
| 94 retrieves the current revision ID from the repository | 94 retrieves the current revision ID from the repository |
| 95 """ | 95 """ |
| 96 command = [ | 96 command = [ |
| 97 'hg', 'id', '-i', '-r', self.config.revision, '--config', | 97 'hg', 'id', '-i', '-r', self.config.revision, '--config', |
| 98 'defaults.id=', self.config.repository | 98 'defaults.id=', self.config.repository, |
| 99 ] | 99 ] |
| 100 return subprocess.check_output(command).strip() | 100 return subprocess.check_output(command).strip() |
| 101 | 101 |
| 102 def getCurrentBuild(self): | 102 def getCurrentBuild(self): |
| 103 """ | 103 """ |
| 104 calculates the (typically numerical) build ID for the current build | 104 calculates the (typically numerical) build ID for the current build |
| 105 """ | 105 """ |
| 106 command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir] | 106 command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir] |
| 107 build = subprocess.check_output(command).strip() | 107 build = subprocess.check_output(command).strip() |
| 108 if self.config.type in {'gecko', 'gecko-webext'}: | 108 if self.config.type in {'gecko', 'gecko-webext'}: |
| 109 build += 'beta' | 109 build += 'beta' |
| 110 return build | 110 return build |
| 111 | 111 |
| 112 def getChanges(self): | 112 def getChanges(self): |
| 113 """ | 113 """ |
| 114 retrieve changes between the current and previous ("first") revision | 114 retrieve changes between the current and previous ("first") revision |
| 115 """ | 115 """ |
| 116 command = [ | 116 command = [ |
| 117 'hg', 'log', '-R', self.tempdir, '-r', | 117 'hg', 'log', '-R', self.tempdir, '-r', |
| 118 'reverse(ancestors({}))'.format(self.config.revision), '-l', '50', | 118 'reverse(ancestors({}))'.format(self.config.revision), '-l', '50', |
| 119 '--encoding', 'utf-8', '--template', | 119 '--encoding', 'utf-8', '--template', |
| 120 '{date|isodate}\\0{author|person}\\0{rev}\\0{desc}\\0\\0', | 120 '{date|isodate}\\0{author|person}\\0{rev}\\0{desc}\\0\\0', |
| 121 '--config', 'defaults.log=' | 121 '--config', 'defaults.log=', |
| 122 ] | 122 ] |
| 123 result = subprocess.check_output(command).decode('utf-8') | 123 result = subprocess.check_output(command).decode('utf-8') |
| 124 | 124 |
| 125 for change in result.split('\x00\x00'): | 125 for change in result.split('\x00\x00'): |
| 126 if change: | 126 if change: |
| 127 date, author, revision, description = change.split('\x00') | 127 date, author, revision, description = change.split('\x00') |
| 128 yield {'date': date, 'author': author, 'revision': revision, 'de
scription': description} | 128 yield {'date': date, 'author': author, 'revision': revision, 'de
scription': description} |
| 129 | 129 |
| 130 def copyRepository(self): | 130 def copyRepository(self): |
| 131 """ | 131 """ |
| (...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 265 os.makedirs(baseDir) | 265 os.makedirs(baseDir) |
| 266 | 266 |
| 267 # ABP for Android used to have its own update manifest format. We need t
o | 267 # ABP for Android used to have its own update manifest format. We need t
o |
| 268 # generate both that and the new one in the libadblockplus format as lon
g | 268 # generate both that and the new one in the libadblockplus format as lon
g |
| 269 # as a significant amount of users is on an old version. | 269 # as a significant amount of users is on an old version. |
| 270 if self.config.type == 'android': | 270 if self.config.type == 'android': |
| 271 newManifestPath = os.path.join(baseDir, 'update.json') | 271 newManifestPath = os.path.join(baseDir, 'update.json') |
| 272 writeAndroidUpdateManifest(newManifestPath, [{ | 272 writeAndroidUpdateManifest(newManifestPath, [{ |
| 273 'basename': self.basename, | 273 'basename': self.basename, |
| 274 'version': self.version, | 274 'version': self.version, |
| 275 'updateURL': self.updateURL | 275 'updateURL': self.updateURL, |
| 276 }]) | 276 }]) |
| 277 | 277 |
| 278 template = get_template(get_config().get('extensions', templateName), | 278 template = get_template(get_config().get('extensions', templateName), |
| 279 autoescape=autoescape) | 279 autoescape=autoescape) |
| 280 template.stream({'extensions': [self]}).dump(manifestPath) | 280 template.stream({'extensions': [self]}).dump(manifestPath) |
| 281 | 281 |
| 282 def writeIEUpdateManifest(self, versions): | 282 def writeIEUpdateManifest(self, versions): |
| 283 """ | 283 """ |
| 284 Writes update.json file for the latest IE build | 284 Writes update.json file for the latest IE build |
| 285 """ | 285 """ |
| 286 if len(versions) == 0: | 286 if len(versions) == 0: |
| 287 return | 287 return |
| 288 | 288 |
| 289 version = versions[0] | 289 version = versions[0] |
| 290 packageName = self.basename + '-' + version + self.config.packageSuffix | 290 packageName = self.basename + '-' + version + self.config.packageSuffix |
| 291 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '
/' + packageName + '?update') | 291 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '
/' + packageName + '?update') |
| 292 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) | 292 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) |
| 293 manifestPath = os.path.join(baseDir, 'update.json') | 293 manifestPath = os.path.join(baseDir, 'update.json') |
| 294 | 294 |
| 295 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit
e | 295 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit
e |
| 296 doWrite(manifestPath, [{ | 296 doWrite(manifestPath, [{ |
| 297 'basename': self.basename, | 297 'basename': self.basename, |
| 298 'version': version, | 298 'version': version, |
| 299 'updateURL': updateURL | 299 'updateURL': updateURL, |
| 300 }]) | 300 }]) |
| 301 | 301 |
| 302 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']: | 302 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']: |
| 303 linkPath = os.path.join(baseDir, '00latest%s' % suffix) | 303 linkPath = os.path.join(baseDir, '00latest%s' % suffix) |
| 304 outputPath = os.path.join(baseDir, self.basename + '-' + version + s
uffix) | 304 outputPath = os.path.join(baseDir, self.basename + '-' + version + s
uffix) |
| 305 if hasattr(os, 'symlink'): | 305 if hasattr(os, 'symlink'): |
| 306 if os.path.exists(linkPath): | 306 if os.path.exists(linkPath): |
| 307 os.remove(linkPath) | 307 os.remove(linkPath) |
| 308 os.symlink(os.path.basename(outputPath), linkPath) | 308 os.symlink(os.path.basename(outputPath), linkPath) |
| 309 else: | 309 else: |
| (...skipping 14 matching lines...) Expand all Loading... |
| 324 apkFile = open(self.path, 'wb') | 324 apkFile = open(self.path, 'wb') |
| 325 | 325 |
| 326 try: | 326 try: |
| 327 try: | 327 try: |
| 328 port = get_config().get('extensions', 'androidBuildPort') | 328 port = get_config().get('extensions', 'androidBuildPort') |
| 329 except ConfigParser.NoOptionError: | 329 except ConfigParser.NoOptionError: |
| 330 port = '22' | 330 port = '22' |
| 331 command = ['ssh', '-p', port, get_config().get('extensions', 'an
droidBuildHost')] | 331 command = ['ssh', '-p', port, get_config().get('extensions', 'an
droidBuildHost')] |
| 332 command.extend(map(pipes.quote, [ | 332 command.extend(map(pipes.quote, [ |
| 333 '/home/android/bin/makedebugbuild.py', '--revision', | 333 '/home/android/bin/makedebugbuild.py', '--revision', |
| 334 self.buildNum, '--version', self.version, '--stdout' | 334 self.buildNum, '--version', self.version, '--stdout', |
| 335 ])) | 335 ])) |
| 336 subprocess.check_call(command, stdout=apkFile, close_fds=True) | 336 subprocess.check_call(command, stdout=apkFile, close_fds=True) |
| 337 except: | 337 except: |
| 338 # clear broken output if any | 338 # clear broken output if any |
| 339 if os.path.exists(self.path): | 339 if os.path.exists(self.path): |
| 340 os.remove(self.path) | 340 os.remove(self.path) |
| 341 raise | 341 raise |
| 342 else: | 342 else: |
| 343 env = os.environ | 343 env = os.environ |
| 344 spiderMonkeyBinary = self.config.spiderMonkeyBinary | 344 spiderMonkeyBinary = self.config.spiderMonkeyBinary |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 399 packageFile = self.basename + '-' + version + self.config.packageSuf
fix | 399 packageFile = self.basename + '-' + version + self.config.packageSuf
fix |
| 400 changelogFile = self.basename + '-' + version + '.changelog.xhtml' | 400 changelogFile = self.basename + '-' + version + '.changelog.xhtml' |
| 401 if not os.path.exists(os.path.join(baseDir, packageFile)): | 401 if not os.path.exists(os.path.join(baseDir, packageFile)): |
| 402 # Oops | 402 # Oops |
| 403 continue | 403 continue |
| 404 | 404 |
| 405 link = { | 405 link = { |
| 406 'version': version, | 406 'version': version, |
| 407 'download': packageFile, | 407 'download': packageFile, |
| 408 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)), | 408 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)), |
| 409 'size': os.path.getsize(os.path.join(baseDir, packageFile)) | 409 'size': os.path.getsize(os.path.join(baseDir, packageFile)), |
| 410 } | 410 } |
| 411 if os.path.exists(os.path.join(baseDir, changelogFile)): | 411 if os.path.exists(os.path.join(baseDir, changelogFile)): |
| 412 link['changelog'] = changelogFile | 412 link['changelog'] = changelogFile |
| 413 links.append(link) | 413 links.append(link) |
| 414 template = get_template(get_config().get('extensions', 'nightlyIndexPage
')) | 414 template = get_template(get_config().get('extensions', 'nightlyIndexPage
')) |
| 415 template.stream({'config': self.config, 'links': links}).dump(outputPath
) | 415 template.stream({'config': self.config, 'links': links}).dump(outputPath
) |
| 416 | 416 |
| 417 def uploadToMozillaAddons(self): | 417 def uploadToMozillaAddons(self): |
| 418 import urllib3 | 418 import urllib3 |
| 419 | 419 |
| 420 header = { | 420 header = { |
| 421 'alg': 'HS256', # HMAC-SHA256 | 421 'alg': 'HS256', # HMAC-SHA256 |
| 422 'typ': 'JWT', | 422 'typ': 'JWT', |
| 423 } | 423 } |
| 424 | 424 |
| 425 issued = int(time.time()) | 425 issued = int(time.time()) |
| 426 payload = { | 426 payload = { |
| 427 'iss': get_config().get('extensions', 'amo_key'), | 427 'iss': get_config().get('extensions', 'amo_key'), |
| 428 'jti': random.random(), | 428 'jti': random.random(), |
| 429 'iat': issued, | 429 'iat': issued, |
| 430 'exp': issued + 60, | 430 'exp': issued + 60, |
| 431 } | 431 } |
| 432 | 432 |
| 433 input = '{}.{}'.format( | 433 input = '{}.{}'.format( |
| 434 base64.b64encode(json.dumps(header)), | 434 base64.b64encode(json.dumps(header)), |
| 435 base64.b64encode(json.dumps(payload)) | 435 base64.b64encode(json.dumps(payload)), |
| 436 ) | 436 ) |
| 437 | 437 |
| 438 signature = hmac.new(get_config().get('extensions', 'amo_secret'), | 438 signature = hmac.new(get_config().get('extensions', 'amo_secret'), |
| 439 msg=input, | 439 msg=input, |
| 440 digestmod=hashlib.sha256).digest() | 440 digestmod=hashlib.sha256).digest() |
| 441 token = '{}.{}'.format(input, base64.b64encode(signature)) | 441 token = '{}.{}'.format(input, base64.b64encode(signature)) |
| 442 | 442 |
| 443 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/' | 443 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/' |
| 444 'versions/{}/').format(self.extensionID, self.version) | 444 'versions/{}/').format(self.extensionID, self.version) |
| 445 | 445 |
| 446 with open(self.path, 'rb') as file: | 446 with open(self.path, 'rb') as file: |
| 447 data, content_type = urllib3.filepost.encode_multipart_formdata({ | 447 data, content_type = urllib3.filepost.encode_multipart_formdata({ |
| 448 'upload': ( | 448 'upload': ( |
| 449 os.path.basename(self.path), | 449 os.path.basename(self.path), |
| 450 file.read(), | 450 file.read(), |
| 451 'application/x-xpinstall' | 451 'application/x-xpinstall', |
| 452 ) | 452 ), |
| 453 }) | 453 }) |
| 454 | 454 |
| 455 request = urllib2.Request(upload_url, data=data) | 455 request = urllib2.Request(upload_url, data=data) |
| 456 request.add_header('Content-Type', content_type) | 456 request.add_header('Content-Type', content_type) |
| 457 request.add_header('Authorization', 'JWT ' + token) | 457 request.add_header('Authorization', 'JWT ' + token) |
| 458 request.get_method = lambda: 'PUT' | 458 request.get_method = lambda: 'PUT' |
| 459 | 459 |
| 460 try: | 460 try: |
| 461 urllib2.urlopen(request).close() | 461 urllib2.urlopen(request).close() |
| 462 except urllib2.HTTPError as e: | 462 except urllib2.HTTPError as e: |
| (...skipping 11 matching lines...) Expand all Loading... |
| 474 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh | 474 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh |
| 475 | 475 |
| 476 response = json.load(opener.open( | 476 response = json.load(opener.open( |
| 477 'https://accounts.google.com/o/oauth2/token', | 477 'https://accounts.google.com/o/oauth2/token', |
| 478 | 478 |
| 479 urlencode([ | 479 urlencode([ |
| 480 ('refresh_token', self.config.refreshToken), | 480 ('refresh_token', self.config.refreshToken), |
| 481 ('client_id', self.config.clientID), | 481 ('client_id', self.config.clientID), |
| 482 ('client_secret', self.config.clientSecret), | 482 ('client_secret', self.config.clientSecret), |
| 483 ('grant_type', 'refresh_token'), | 483 ('grant_type', 'refresh_token'), |
| 484 ]) | 484 ]), |
| 485 )) | 485 )) |
| 486 | 486 |
| 487 auth_token = '%s %s' % (response['token_type'], response['access_token']
) | 487 auth_token = '%s %s' % (response['token_type'], response['access_token']
) |
| 488 | 488 |
| 489 # upload a new version with the Chrome Web Store API | 489 # upload a new version with the Chrome Web Store API |
| 490 # https://developer.chrome.com/webstore/using_webstore_api#uploadexisitn
g | 490 # https://developer.chrome.com/webstore/using_webstore_api#uploadexisitn
g |
| 491 | 491 |
| 492 request = urllib2.Request('https://www.googleapis.com/upload/chromewebst
ore/v1.1/items/' + self.config.devbuildGalleryID) | 492 request = urllib2.Request('https://www.googleapis.com/upload/chromewebst
ore/v1.1/items/' + self.config.devbuildGalleryID) |
| 493 request.get_method = lambda: 'PUT' | 493 request.get_method = lambda: 'PUT' |
| 494 request.add_header('Authorization', auth_token) | 494 request.add_header('Authorization', auth_token) |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 528 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo
ry-protocols-oauth-code#refreshing-the-access-tokens | 528 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo
ry-protocols-oauth-code#refreshing-the-access-tokens |
| 529 server = 'https://login.microsoftonline.com' | 529 server = 'https://login.microsoftonline.com' |
| 530 token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID) | 530 token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID) |
| 531 | 531 |
| 532 opener = urllib2.build_opener(HTTPErrorBodyHandler) | 532 opener = urllib2.build_opener(HTTPErrorBodyHandler) |
| 533 post_data = urlencode([ | 533 post_data = urlencode([ |
| 534 ('refresh_token', self.config.refreshToken), | 534 ('refresh_token', self.config.refreshToken), |
| 535 ('client_id', self.config.clientID), | 535 ('client_id', self.config.clientID), |
| 536 ('client_secret', self.config.clientSecret), | 536 ('client_secret', self.config.clientSecret), |
| 537 ('grant_type', 'refresh_token'), | 537 ('grant_type', 'refresh_token'), |
| 538 ('resource', 'https://graph.windows.net') | 538 ('resource', 'https://graph.windows.net'), |
| 539 ]) | 539 ]) |
| 540 request = urllib2.Request(token_path, post_data) | 540 request = urllib2.Request(token_path, post_data) |
| 541 with contextlib.closing(opener.open(request)) as response: | 541 with contextlib.closing(opener.open(request)) as response: |
| 542 data = json.load(response) | 542 data = json.load(response) |
| 543 auth_token = '{0[token_type]} {0[access_token]}'.format(data) | 543 auth_token = '{0[token_type]} {0[access_token]}'.format(data) |
| 544 | 544 |
| 545 return auth_token | 545 return auth_token |
| 546 | 546 |
| 547 def upload_appx_file_to_windows_store(self, file_upload_url): | 547 def upload_appx_file_to_windows_store(self, file_upload_url): |
| 548 # Add .appx file to a .zip file | 548 # Add .appx file to a .zip file |
| (...skipping 20 matching lines...) Expand all Loading... |
| 569 def upload_to_windows_store(self): | 569 def upload_to_windows_store(self): |
| 570 opener = urllib2.build_opener(HTTPErrorBodyHandler) | 570 opener = urllib2.build_opener(HTTPErrorBodyHandler) |
| 571 | 571 |
| 572 headers = {'Authorization': self.get_windows_store_access_token(), | 572 headers = {'Authorization': self.get_windows_store_access_token(), |
| 573 'Content-type': 'application/json'} | 573 'Content-type': 'application/json'} |
| 574 | 574 |
| 575 # Get application | 575 # Get application |
| 576 # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app | 576 # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app |
| 577 api_path = '{}/v1.0/my/applications/{}'.format( | 577 api_path = '{}/v1.0/my/applications/{}'.format( |
| 578 'https://manage.devcenter.microsoft.com', | 578 'https://manage.devcenter.microsoft.com', |
| 579 self.config.devbuildGalleryID | 579 self.config.devbuildGalleryID, |
| 580 ) | 580 ) |
| 581 | 581 |
| 582 request = urllib2.Request(api_path, None, headers) | 582 request = urllib2.Request(api_path, None, headers) |
| 583 with contextlib.closing(opener.open(request)) as response: | 583 with contextlib.closing(opener.open(request)) as response: |
| 584 app_obj = json.load(response) | 584 app_obj = json.load(response) |
| 585 | 585 |
| 586 # Delete existing in-progress submission | 586 # Delete existing in-progress submission |
| 587 # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-su
bmission | 587 # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-su
bmission |
| 588 submissions_path = api_path + '/submissions' | 588 submissions_path = api_path + '/submissions' |
| 589 if 'pendingApplicationSubmission' in app_obj: | 589 if 'pendingApplicationSubmission' in app_obj: |
| (...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 717 except Exception as ex: | 717 except Exception as ex: |
| 718 logging.error('The build for %s failed:', repo) | 718 logging.error('The build for %s failed:', repo) |
| 719 logging.exception(ex) | 719 logging.exception(ex) |
| 720 | 720 |
| 721 file = open(nightlyConfigFile, 'wb') | 721 file = open(nightlyConfigFile, 'wb') |
| 722 nightlyConfig.write(file) | 722 nightlyConfig.write(file) |
| 723 | 723 |
| 724 | 724 |
| 725 if __name__ == '__main__': | 725 if __name__ == '__main__': |
| 726 main() | 726 main() |
| OLD | NEW |