OLD | NEW |
1 # This file is part of the Adblock Plus web scripts, | 1 # This file is part of the Adblock Plus web scripts, |
2 # Copyright (C) 2006-present eyeo GmbH | 2 # Copyright (C) 2006-present eyeo GmbH |
3 # | 3 # |
4 # Adblock Plus is free software: you can redistribute it and/or modify | 4 # Adblock Plus is free software: you can redistribute it and/or modify |
5 # it under the terms of the GNU General Public License version 3 as | 5 # it under the terms of the GNU General Public License version 3 as |
6 # published by the Free Software Foundation. | 6 # published by the Free Software Foundation. |
7 # | 7 # |
8 # Adblock Plus is distributed in the hope that it will be useful, | 8 # Adblock Plus is distributed in the hope that it will be useful, |
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
(...skipping 30 matching lines...) Expand all Loading... |
41 from urllib import urlencode | 41 from urllib import urlencode |
42 import urllib2 | 42 import urllib2 |
43 import urlparse | 43 import urlparse |
44 import zipfile | 44 import zipfile |
45 import contextlib | 45 import contextlib |
46 | 46 |
47 from xml.dom.minidom import parse as parseXml | 47 from xml.dom.minidom import parse as parseXml |
48 | 48 |
49 from sitescripts.extensions.utils import ( | 49 from sitescripts.extensions.utils import ( |
50 compareVersions, Configuration, | 50 compareVersions, Configuration, |
51 writeAndroidUpdateManifest | 51 writeAndroidUpdateManifest, |
52 ) | 52 ) |
53 from sitescripts.utils import get_config, get_template | 53 from sitescripts.utils import get_config, get_template |
54 | 54 |
55 MAX_BUILDS = 50 | 55 MAX_BUILDS = 50 |
56 | 56 |
57 | 57 |
58 # Google and Microsoft APIs use HTTP error codes with error message in | 58 # Google and Microsoft APIs use HTTP error codes with error message in |
59 # body. So we add the response body to the HTTPError to get more | 59 # body. So we add the response body to the HTTPError to get more |
60 # meaningful error messages. | 60 # meaningful error messages. |
61 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler): | 61 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler): |
(...skipping 26 matching lines...) Expand all Loading... |
88 | 88 |
89 def hasChanges(self): | 89 def hasChanges(self): |
90 return self.revision != self.previousRevision | 90 return self.revision != self.previousRevision |
91 | 91 |
92 def getCurrentRevision(self): | 92 def getCurrentRevision(self): |
93 """ | 93 """ |
94 retrieves the current revision ID from the repository | 94 retrieves the current revision ID from the repository |
95 """ | 95 """ |
96 command = [ | 96 command = [ |
97 'hg', 'id', '-i', '-r', self.config.revision, '--config', | 97 'hg', 'id', '-i', '-r', self.config.revision, '--config', |
98 'defaults.id=', self.config.repository | 98 'defaults.id=', self.config.repository, |
99 ] | 99 ] |
100 return subprocess.check_output(command).strip() | 100 return subprocess.check_output(command).strip() |
101 | 101 |
102 def getCurrentBuild(self): | 102 def getCurrentBuild(self): |
103 """ | 103 """ |
104 calculates the (typically numerical) build ID for the current build | 104 calculates the (typically numerical) build ID for the current build |
105 """ | 105 """ |
106 command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir] | 106 command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir] |
107 build = subprocess.check_output(command).strip() | 107 build = subprocess.check_output(command).strip() |
108 if self.config.type in {'gecko', 'gecko-webext'}: | 108 if self.config.type in {'gecko', 'gecko-webext'}: |
109 build += 'beta' | 109 build += 'beta' |
110 return build | 110 return build |
111 | 111 |
112 def getChanges(self): | 112 def getChanges(self): |
113 """ | 113 """ |
114 retrieve changes between the current and previous ("first") revision | 114 retrieve changes between the current and previous ("first") revision |
115 """ | 115 """ |
116 command = [ | 116 command = [ |
117 'hg', 'log', '-R', self.tempdir, '-r', | 117 'hg', 'log', '-R', self.tempdir, '-r', |
118 'reverse(ancestors({}))'.format(self.config.revision), '-l', '50', | 118 'reverse(ancestors({}))'.format(self.config.revision), '-l', '50', |
119 '--encoding', 'utf-8', '--template', | 119 '--encoding', 'utf-8', '--template', |
120 '{date|isodate}\\0{author|person}\\0{rev}\\0{desc}\\0\\0', | 120 '{date|isodate}\\0{author|person}\\0{rev}\\0{desc}\\0\\0', |
121 '--config', 'defaults.log=' | 121 '--config', 'defaults.log=', |
122 ] | 122 ] |
123 result = subprocess.check_output(command).decode('utf-8') | 123 result = subprocess.check_output(command).decode('utf-8') |
124 | 124 |
125 for change in result.split('\x00\x00'): | 125 for change in result.split('\x00\x00'): |
126 if change: | 126 if change: |
127 date, author, revision, description = change.split('\x00') | 127 date, author, revision, description = change.split('\x00') |
128 yield {'date': date, 'author': author, 'revision': revision, 'de
scription': description} | 128 yield {'date': date, 'author': author, 'revision': revision, 'de
scription': description} |
129 | 129 |
130 def copyRepository(self): | 130 def copyRepository(self): |
131 """ | 131 """ |
(...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
265 os.makedirs(baseDir) | 265 os.makedirs(baseDir) |
266 | 266 |
267 # ABP for Android used to have its own update manifest format. We need t
o | 267 # ABP for Android used to have its own update manifest format. We need t
o |
268 # generate both that and the new one in the libadblockplus format as lon
g | 268 # generate both that and the new one in the libadblockplus format as lon
g |
269 # as a significant amount of users is on an old version. | 269 # as a significant amount of users is on an old version. |
270 if self.config.type == 'android': | 270 if self.config.type == 'android': |
271 newManifestPath = os.path.join(baseDir, 'update.json') | 271 newManifestPath = os.path.join(baseDir, 'update.json') |
272 writeAndroidUpdateManifest(newManifestPath, [{ | 272 writeAndroidUpdateManifest(newManifestPath, [{ |
273 'basename': self.basename, | 273 'basename': self.basename, |
274 'version': self.version, | 274 'version': self.version, |
275 'updateURL': self.updateURL | 275 'updateURL': self.updateURL, |
276 }]) | 276 }]) |
277 | 277 |
278 template = get_template(get_config().get('extensions', templateName), | 278 template = get_template(get_config().get('extensions', templateName), |
279 autoescape=autoescape) | 279 autoescape=autoescape) |
280 template.stream({'extensions': [self]}).dump(manifestPath) | 280 template.stream({'extensions': [self]}).dump(manifestPath) |
281 | 281 |
282 def writeIEUpdateManifest(self, versions): | 282 def writeIEUpdateManifest(self, versions): |
283 """ | 283 """ |
284 Writes update.json file for the latest IE build | 284 Writes update.json file for the latest IE build |
285 """ | 285 """ |
286 if len(versions) == 0: | 286 if len(versions) == 0: |
287 return | 287 return |
288 | 288 |
289 version = versions[0] | 289 version = versions[0] |
290 packageName = self.basename + '-' + version + self.config.packageSuffix | 290 packageName = self.basename + '-' + version + self.config.packageSuffix |
291 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '
/' + packageName + '?update') | 291 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '
/' + packageName + '?update') |
292 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) | 292 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) |
293 manifestPath = os.path.join(baseDir, 'update.json') | 293 manifestPath = os.path.join(baseDir, 'update.json') |
294 | 294 |
295 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit
e | 295 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit
e |
296 doWrite(manifestPath, [{ | 296 doWrite(manifestPath, [{ |
297 'basename': self.basename, | 297 'basename': self.basename, |
298 'version': version, | 298 'version': version, |
299 'updateURL': updateURL | 299 'updateURL': updateURL, |
300 }]) | 300 }]) |
301 | 301 |
302 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']: | 302 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']: |
303 linkPath = os.path.join(baseDir, '00latest%s' % suffix) | 303 linkPath = os.path.join(baseDir, '00latest%s' % suffix) |
304 outputPath = os.path.join(baseDir, self.basename + '-' + version + s
uffix) | 304 outputPath = os.path.join(baseDir, self.basename + '-' + version + s
uffix) |
305 if hasattr(os, 'symlink'): | 305 if hasattr(os, 'symlink'): |
306 if os.path.exists(linkPath): | 306 if os.path.exists(linkPath): |
307 os.remove(linkPath) | 307 os.remove(linkPath) |
308 os.symlink(os.path.basename(outputPath), linkPath) | 308 os.symlink(os.path.basename(outputPath), linkPath) |
309 else: | 309 else: |
(...skipping 14 matching lines...) Expand all Loading... |
324 apkFile = open(self.path, 'wb') | 324 apkFile = open(self.path, 'wb') |
325 | 325 |
326 try: | 326 try: |
327 try: | 327 try: |
328 port = get_config().get('extensions', 'androidBuildPort') | 328 port = get_config().get('extensions', 'androidBuildPort') |
329 except ConfigParser.NoOptionError: | 329 except ConfigParser.NoOptionError: |
330 port = '22' | 330 port = '22' |
331 command = ['ssh', '-p', port, get_config().get('extensions', 'an
droidBuildHost')] | 331 command = ['ssh', '-p', port, get_config().get('extensions', 'an
droidBuildHost')] |
332 command.extend(map(pipes.quote, [ | 332 command.extend(map(pipes.quote, [ |
333 '/home/android/bin/makedebugbuild.py', '--revision', | 333 '/home/android/bin/makedebugbuild.py', '--revision', |
334 self.buildNum, '--version', self.version, '--stdout' | 334 self.buildNum, '--version', self.version, '--stdout', |
335 ])) | 335 ])) |
336 subprocess.check_call(command, stdout=apkFile, close_fds=True) | 336 subprocess.check_call(command, stdout=apkFile, close_fds=True) |
337 except: | 337 except: |
338 # clear broken output if any | 338 # clear broken output if any |
339 if os.path.exists(self.path): | 339 if os.path.exists(self.path): |
340 os.remove(self.path) | 340 os.remove(self.path) |
341 raise | 341 raise |
342 else: | 342 else: |
343 env = os.environ | 343 env = os.environ |
344 spiderMonkeyBinary = self.config.spiderMonkeyBinary | 344 spiderMonkeyBinary = self.config.spiderMonkeyBinary |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
399 packageFile = self.basename + '-' + version + self.config.packageSuf
fix | 399 packageFile = self.basename + '-' + version + self.config.packageSuf
fix |
400 changelogFile = self.basename + '-' + version + '.changelog.xhtml' | 400 changelogFile = self.basename + '-' + version + '.changelog.xhtml' |
401 if not os.path.exists(os.path.join(baseDir, packageFile)): | 401 if not os.path.exists(os.path.join(baseDir, packageFile)): |
402 # Oops | 402 # Oops |
403 continue | 403 continue |
404 | 404 |
405 link = { | 405 link = { |
406 'version': version, | 406 'version': version, |
407 'download': packageFile, | 407 'download': packageFile, |
408 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)), | 408 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)), |
409 'size': os.path.getsize(os.path.join(baseDir, packageFile)) | 409 'size': os.path.getsize(os.path.join(baseDir, packageFile)), |
410 } | 410 } |
411 if os.path.exists(os.path.join(baseDir, changelogFile)): | 411 if os.path.exists(os.path.join(baseDir, changelogFile)): |
412 link['changelog'] = changelogFile | 412 link['changelog'] = changelogFile |
413 links.append(link) | 413 links.append(link) |
414 template = get_template(get_config().get('extensions', 'nightlyIndexPage
')) | 414 template = get_template(get_config().get('extensions', 'nightlyIndexPage
')) |
415 template.stream({'config': self.config, 'links': links}).dump(outputPath
) | 415 template.stream({'config': self.config, 'links': links}).dump(outputPath
) |
416 | 416 |
417 def uploadToMozillaAddons(self): | 417 def uploadToMozillaAddons(self): |
418 import urllib3 | 418 import urllib3 |
419 | 419 |
420 header = { | 420 header = { |
421 'alg': 'HS256', # HMAC-SHA256 | 421 'alg': 'HS256', # HMAC-SHA256 |
422 'typ': 'JWT', | 422 'typ': 'JWT', |
423 } | 423 } |
424 | 424 |
425 issued = int(time.time()) | 425 issued = int(time.time()) |
426 payload = { | 426 payload = { |
427 'iss': get_config().get('extensions', 'amo_key'), | 427 'iss': get_config().get('extensions', 'amo_key'), |
428 'jti': random.random(), | 428 'jti': random.random(), |
429 'iat': issued, | 429 'iat': issued, |
430 'exp': issued + 60, | 430 'exp': issued + 60, |
431 } | 431 } |
432 | 432 |
433 input = '{}.{}'.format( | 433 input = '{}.{}'.format( |
434 base64.b64encode(json.dumps(header)), | 434 base64.b64encode(json.dumps(header)), |
435 base64.b64encode(json.dumps(payload)) | 435 base64.b64encode(json.dumps(payload)), |
436 ) | 436 ) |
437 | 437 |
438 signature = hmac.new(get_config().get('extensions', 'amo_secret'), | 438 signature = hmac.new(get_config().get('extensions', 'amo_secret'), |
439 msg=input, | 439 msg=input, |
440 digestmod=hashlib.sha256).digest() | 440 digestmod=hashlib.sha256).digest() |
441 token = '{}.{}'.format(input, base64.b64encode(signature)) | 441 token = '{}.{}'.format(input, base64.b64encode(signature)) |
442 | 442 |
443 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/' | 443 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/' |
444 'versions/{}/').format(self.extensionID, self.version) | 444 'versions/{}/').format(self.extensionID, self.version) |
445 | 445 |
446 with open(self.path, 'rb') as file: | 446 with open(self.path, 'rb') as file: |
447 data, content_type = urllib3.filepost.encode_multipart_formdata({ | 447 data, content_type = urllib3.filepost.encode_multipart_formdata({ |
448 'upload': ( | 448 'upload': ( |
449 os.path.basename(self.path), | 449 os.path.basename(self.path), |
450 file.read(), | 450 file.read(), |
451 'application/x-xpinstall' | 451 'application/x-xpinstall', |
452 ) | 452 ), |
453 }) | 453 }) |
454 | 454 |
455 request = urllib2.Request(upload_url, data=data) | 455 request = urllib2.Request(upload_url, data=data) |
456 request.add_header('Content-Type', content_type) | 456 request.add_header('Content-Type', content_type) |
457 request.add_header('Authorization', 'JWT ' + token) | 457 request.add_header('Authorization', 'JWT ' + token) |
458 request.get_method = lambda: 'PUT' | 458 request.get_method = lambda: 'PUT' |
459 | 459 |
460 try: | 460 try: |
461 urllib2.urlopen(request).close() | 461 urllib2.urlopen(request).close() |
462 except urllib2.HTTPError as e: | 462 except urllib2.HTTPError as e: |
(...skipping 11 matching lines...) Expand all Loading... |
474 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh | 474 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh |
475 | 475 |
476 response = json.load(opener.open( | 476 response = json.load(opener.open( |
477 'https://accounts.google.com/o/oauth2/token', | 477 'https://accounts.google.com/o/oauth2/token', |
478 | 478 |
479 urlencode([ | 479 urlencode([ |
480 ('refresh_token', self.config.refreshToken), | 480 ('refresh_token', self.config.refreshToken), |
481 ('client_id', self.config.clientID), | 481 ('client_id', self.config.clientID), |
482 ('client_secret', self.config.clientSecret), | 482 ('client_secret', self.config.clientSecret), |
483 ('grant_type', 'refresh_token'), | 483 ('grant_type', 'refresh_token'), |
484 ]) | 484 ]), |
485 )) | 485 )) |
486 | 486 |
487 auth_token = '%s %s' % (response['token_type'], response['access_token']
) | 487 auth_token = '%s %s' % (response['token_type'], response['access_token']
) |
488 | 488 |
489 # upload a new version with the Chrome Web Store API | 489 # upload a new version with the Chrome Web Store API |
490 # https://developer.chrome.com/webstore/using_webstore_api#uploadexisitn
g | 490 # https://developer.chrome.com/webstore/using_webstore_api#uploadexisitn
g |
491 | 491 |
492 request = urllib2.Request('https://www.googleapis.com/upload/chromewebst
ore/v1.1/items/' + self.config.devbuildGalleryID) | 492 request = urllib2.Request('https://www.googleapis.com/upload/chromewebst
ore/v1.1/items/' + self.config.devbuildGalleryID) |
493 request.get_method = lambda: 'PUT' | 493 request.get_method = lambda: 'PUT' |
494 request.add_header('Authorization', auth_token) | 494 request.add_header('Authorization', auth_token) |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
528 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo
ry-protocols-oauth-code#refreshing-the-access-tokens | 528 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo
ry-protocols-oauth-code#refreshing-the-access-tokens |
529 server = 'https://login.microsoftonline.com' | 529 server = 'https://login.microsoftonline.com' |
530 token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID) | 530 token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID) |
531 | 531 |
532 opener = urllib2.build_opener(HTTPErrorBodyHandler) | 532 opener = urllib2.build_opener(HTTPErrorBodyHandler) |
533 post_data = urlencode([ | 533 post_data = urlencode([ |
534 ('refresh_token', self.config.refreshToken), | 534 ('refresh_token', self.config.refreshToken), |
535 ('client_id', self.config.clientID), | 535 ('client_id', self.config.clientID), |
536 ('client_secret', self.config.clientSecret), | 536 ('client_secret', self.config.clientSecret), |
537 ('grant_type', 'refresh_token'), | 537 ('grant_type', 'refresh_token'), |
538 ('resource', 'https://graph.windows.net') | 538 ('resource', 'https://graph.windows.net'), |
539 ]) | 539 ]) |
540 request = urllib2.Request(token_path, post_data) | 540 request = urllib2.Request(token_path, post_data) |
541 with contextlib.closing(opener.open(request)) as response: | 541 with contextlib.closing(opener.open(request)) as response: |
542 data = json.load(response) | 542 data = json.load(response) |
543 auth_token = '{0[token_type]} {0[access_token]}'.format(data) | 543 auth_token = '{0[token_type]} {0[access_token]}'.format(data) |
544 | 544 |
545 return auth_token | 545 return auth_token |
546 | 546 |
547 def upload_appx_file_to_windows_store(self, file_upload_url): | 547 def upload_appx_file_to_windows_store(self, file_upload_url): |
548 # Add .appx file to a .zip file | 548 # Add .appx file to a .zip file |
(...skipping 20 matching lines...) Expand all Loading... |
569 def upload_to_windows_store(self): | 569 def upload_to_windows_store(self): |
570 opener = urllib2.build_opener(HTTPErrorBodyHandler) | 570 opener = urllib2.build_opener(HTTPErrorBodyHandler) |
571 | 571 |
572 headers = {'Authorization': self.get_windows_store_access_token(), | 572 headers = {'Authorization': self.get_windows_store_access_token(), |
573 'Content-type': 'application/json'} | 573 'Content-type': 'application/json'} |
574 | 574 |
575 # Get application | 575 # Get application |
576 # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app | 576 # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app |
577 api_path = '{}/v1.0/my/applications/{}'.format( | 577 api_path = '{}/v1.0/my/applications/{}'.format( |
578 'https://manage.devcenter.microsoft.com', | 578 'https://manage.devcenter.microsoft.com', |
579 self.config.devbuildGalleryID | 579 self.config.devbuildGalleryID, |
580 ) | 580 ) |
581 | 581 |
582 request = urllib2.Request(api_path, None, headers) | 582 request = urllib2.Request(api_path, None, headers) |
583 with contextlib.closing(opener.open(request)) as response: | 583 with contextlib.closing(opener.open(request)) as response: |
584 app_obj = json.load(response) | 584 app_obj = json.load(response) |
585 | 585 |
586 # Delete existing in-progress submission | 586 # Delete existing in-progress submission |
587 # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-su
bmission | 587 # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-su
bmission |
588 submissions_path = api_path + '/submissions' | 588 submissions_path = api_path + '/submissions' |
589 if 'pendingApplicationSubmission' in app_obj: | 589 if 'pendingApplicationSubmission' in app_obj: |
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
717 except Exception as ex: | 717 except Exception as ex: |
718 logging.error('The build for %s failed:', repo) | 718 logging.error('The build for %s failed:', repo) |
719 logging.exception(ex) | 719 logging.exception(ex) |
720 | 720 |
721 file = open(nightlyConfigFile, 'wb') | 721 file = open(nightlyConfigFile, 'wb') |
722 nightlyConfig.write(file) | 722 nightlyConfig.write(file) |
723 | 723 |
724 | 724 |
725 if __name__ == '__main__': | 725 if __name__ == '__main__': |
726 main() | 726 main() |
OLD | NEW |