Left: | ||
Right: |
OLD | NEW |
---|---|
1 # This file is part of the Adblock Plus web scripts, | 1 # This file is part of the Adblock Plus web scripts, |
2 # Copyright (C) 2006-2016 Eyeo GmbH | 2 # Copyright (C) 2006-2016 Eyeo GmbH |
3 # | 3 # |
4 # Adblock Plus is free software: you can redistribute it and/or modify | 4 # Adblock Plus is free software: you can redistribute it and/or modify |
5 # it under the terms of the GNU General Public License version 3 as | 5 # it under the terms of the GNU General Public License version 3 as |
6 # published by the Free Software Foundation. | 6 # published by the Free Software Foundation. |
7 # | 7 # |
8 # Adblock Plus is distributed in the hope that it will be useful, | 8 # Adblock Plus is distributed in the hope that it will be useful, |
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
11 # GNU General Public License for more details. | 11 # GNU General Public License for more details. |
12 # | 12 # |
13 # You should have received a copy of the GNU General Public License | 13 # You should have received a copy of the GNU General Public License |
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. | 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. |
15 | 15 |
16 """ | 16 """ |
17 | 17 |
18 Nightly builds generation script | 18 Nightly builds generation script |
19 ================================ | 19 ================================ |
20 | 20 |
21 This script generates nightly builds of extensions, together | 21 This script generates nightly builds of extensions, together |
22 with changelogs and documentation. | 22 with changelogs and documentation. |
23 | 23 |
24 """ | 24 """ |
25 | 25 |
26 import ConfigParser | 26 import ConfigParser |
27 import cookielib | 27 import base64 |
28 from datetime import datetime | 28 from datetime import datetime |
29 import hashlib | 29 import hashlib |
30 import HTMLParser | 30 import hmac |
31 import json | 31 import json |
32 import logging | 32 import logging |
33 import os | 33 import os |
34 import pipes | 34 import pipes |
35 import random | |
35 import shutil | 36 import shutil |
36 import struct | 37 import struct |
37 import subprocess | 38 import subprocess |
38 import sys | 39 import sys |
39 import tempfile | 40 import tempfile |
40 import time | 41 import time |
41 from urllib import urlencode | 42 from urllib import urlencode |
42 import urllib2 | 43 import urllib2 |
43 import urlparse | 44 import urlparse |
44 from xml.dom.minidom import parse as parseXml | 45 from xml.dom.minidom import parse as parseXml |
(...skipping 339 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
384 } | 385 } |
385 if os.path.exists(os.path.join(baseDir, changelogFile)): | 386 if os.path.exists(os.path.join(baseDir, changelogFile)): |
386 link['changelog'] = changelogFile | 387 link['changelog'] = changelogFile |
387 links.append(link) | 388 links.append(link) |
388 template = get_template(get_config().get('extensions', 'nightlyIndexPage ')) | 389 template = get_template(get_config().get('extensions', 'nightlyIndexPage ')) |
389 template.stream({'config': self.config, 'links': links}).dump(outputPath ) | 390 template.stream({'config': self.config, 'links': links}).dump(outputPath ) |
390 | 391 |
391 def uploadToMozillaAddons(self): | 392 def uploadToMozillaAddons(self): |
392 import urllib3 | 393 import urllib3 |
393 | 394 |
394 username = get_config().get('extensions', 'amo_username') | 395 header = { |
395 password = get_config().get('extensions', 'amo_password') | 396 'alg': 'HS256', # HMAC-SHA256 |
397 'typ': 'JWT', | |
398 } | |
396 | 399 |
397 slug = self.config.galleryID | 400 issued = int(time.time()) |
398 login_url = 'https://addons.mozilla.org/en-US/firefox/users/login' | 401 payload = { |
399 upload_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/uploa d' % slug | 402 'iss': get_config().get('extensions', 'amo_key'), |
400 add_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/versions /add' % slug | 403 'jti': random.random(), |
404 'iat': issued, | |
405 'exp': issued + 60, | |
406 } | |
401 | 407 |
402 cookie_jar = cookielib.CookieJar() | 408 input = '.'.join([ |
Sebastian Noack
2016/09/13 15:03:12
'{}.{}'.format(...) seems more appropriate rather
Wladimir Palant
2016/09/13 15:16:54
Done.
| |
403 opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar)) | 409 base64.b64encode(json.dumps(header)), |
410 base64.b64encode(json.dumps(payload)) | |
411 ]) | |
404 | 412 |
405 def load_url(url, data=None): | 413 signature = hmac.new(get_config().get('extensions', 'amo_secret'), |
406 content_type = 'application/x-www-form-urlencoded' | 414 msg=input, |
407 if isinstance(data, dict): | 415 digestmod=hashlib.sha256).digest() |
408 if any(isinstance(v, tuple) for v in data.itervalues()): | 416 token = '.'.join([input, base64.b64encode(signature)]) |
409 data, content_type = urllib3.filepost.encode_multipart_formd ata(data) | |
410 else: | |
411 data = urlencode(data.items()) | |
412 | 417 |
413 request = urllib2.Request(url, data, headers={'Content-Type': conten t_type}) | 418 upload_url = ('https://addons.mozilla.org/api/v3/addons/{0}/' |
Sebastian Noack
2016/09/13 15:03:12
Nit: Note that the indices in the placeholders are
Wladimir Palant
2016/09/13 15:16:54
Done.
| |
414 response = opener.open(request) | 419 'versions/{1}/').format(self.extensionID, self.version) |
415 try: | |
416 return response.read() | |
417 finally: | |
418 response.close() | |
419 | 420 |
420 class CSRFParser(HTMLParser.HTMLParser): | 421 with open(self.path, 'rb') as file: |
421 result = None | 422 data, content_type = urllib3.filepost.encode_multipart_formdata({ |
422 dummy_exception = Exception() | 423 'upload': ( |
424 os.path.basename(self.path), | |
425 file.read(), | |
426 'application/x-xpinstall' | |
427 ) | |
428 }) | |
423 | 429 |
424 def __init__(self, data): | 430 request = urllib2.Request(upload_url, data=data) |
425 HTMLParser.HTMLParser.__init__(self) | 431 request.add_header('Content-Type', content_type) |
426 try: | 432 request.add_header('Authorization', 'JWT ' + token) |
427 self.feed(data) | 433 request.get_method = lambda: 'PUT' |
428 self.close() | |
429 except Exception, e: | |
430 if e != self.dummy_exception: | |
431 raise | |
432 | 434 |
433 if not self.result: | 435 try: |
434 raise Exception('Failed to extract CSRF token') | 436 urllib2.urlopen(request).close() |
435 | 437 except urllib2.HTTPError as e: |
436 def set_result(self, value): | 438 logging.error(e.read()) |
Sebastian Noack
2016/09/13 15:03:12
The error response should be closed as well.
Wladimir Palant
2016/09/13 15:16:54
Done.
| |
437 self.result = value | 439 raise |
438 raise self.dummy_exception | |
439 | |
440 def handle_starttag(self, tag, attrs): | |
441 attrs = dict(attrs) | |
442 if tag == 'meta' and attrs.get('name') == 'csrf': | |
443 self.set_result(attrs.get('content')) | |
444 if tag == 'input' and attrs.get('name') == 'csrfmiddlewaretoken' : | |
445 self.set_result(attrs.get('value')) | |
446 | |
447 # Extract anonymous CSRF token | |
448 login_page = load_url(login_url) | |
449 csrf_token = CSRFParser(login_page).result | |
450 | |
451 # Log in and get session's CSRF token | |
452 main_page = load_url( | |
453 login_url, | |
454 { | |
455 'csrfmiddlewaretoken': csrf_token, | |
456 'username': username, | |
457 'password': password, | |
458 } | |
459 ) | |
460 csrf_token = CSRFParser(main_page).result | |
461 | |
462 # Upload build | |
463 with open(self.path, 'rb') as file: | |
464 upload_response = json.loads(load_url( | |
465 upload_url, | |
466 { | |
467 'csrfmiddlewaretoken': csrf_token, | |
468 'upload': (os.path.basename(self.path), file.read(), 'applic ation/x-xpinstall'), | |
469 } | |
470 )) | |
471 | |
472 # Wait for validation to finish | |
473 while not upload_response.get('validation'): | |
474 time.sleep(2) | |
475 upload_response = json.loads(load_url( | |
476 upload_url + '/' + upload_response.get('upload') | |
477 )) | |
478 | |
479 if upload_response['validation'].get('errors', 0): | |
480 raise Exception('Build failed AMO validation, see https://addons.moz illa.org%s' % upload_response.get('full_report_url')) | |
481 | |
482 # Add version | |
483 add_response = json.loads(load_url( | |
484 add_url, | |
485 { | |
486 'csrfmiddlewaretoken': csrf_token, | |
487 'upload': upload_response.get('upload'), | |
488 'source': ('', '', 'application/octet-stream'), | |
489 'beta': 'on', | |
490 'supported_platforms': 1, # PLATFORM_ANY.id | |
491 } | |
492 )) | |
493 | 440 |
494 def uploadToChromeWebStore(self): | 441 def uploadToChromeWebStore(self): |
495 # Google APIs use HTTP error codes with error message in body. So we add | 442 # Google APIs use HTTP error codes with error message in body. So we add |
496 # the response body to the HTTPError to get more meaningful error messag es. | 443 # the response body to the HTTPError to get more meaningful error messag es. |
497 | 444 |
498 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler): | 445 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler): |
499 def http_error_default(self, req, fp, code, msg, hdrs): | 446 def http_error_default(self, req, fp, code, msg, hdrs): |
500 raise urllib2.HTTPError(req.get_full_url(), code, '%s\n%s' % (ms g, fp.read()), hdrs, fp) | 447 raise urllib2.HTTPError(req.get_full_url(), code, '%s\n%s' % (ms g, fp.read()), hdrs, fp) |
501 | 448 |
502 opener = urllib2.build_opener(HTTPErrorBodyHandler) | 449 opener = urllib2.build_opener(HTTPErrorBodyHandler) |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
593 | 540 |
594 if self.config.type == 'ie': | 541 if self.config.type == 'ie': |
595 self.writeIEUpdateManifest(versions) | 542 self.writeIEUpdateManifest(versions) |
596 | 543 |
597 # update index page | 544 # update index page |
598 self.updateIndex(versions) | 545 self.updateIndex(versions) |
599 | 546 |
600 # update nightlies config | 547 # update nightlies config |
601 self.config.latestRevision = self.revision | 548 self.config.latestRevision = self.revision |
602 | 549 |
603 if self.config.type == 'gecko' and self.config.galleryID and get_con fig().get('extensions', 'amo_username'): | 550 if self.config.type == 'gecko' and self.config.galleryID and get_con fig().has_option('extensions', 'amo_key'): |
604 self.uploadToMozillaAddons() | 551 self.uploadToMozillaAddons() |
605 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken: | 552 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken: |
606 self.uploadToChromeWebStore() | 553 self.uploadToChromeWebStore() |
607 finally: | 554 finally: |
608 # clean up | 555 # clean up |
609 if self.tempdir: | 556 if self.tempdir: |
610 shutil.rmtree(self.tempdir, ignore_errors=True) | 557 shutil.rmtree(self.tempdir, ignore_errors=True) |
611 | 558 |
612 | 559 |
613 def main(): | 560 def main(): |
(...skipping 17 matching lines...) Expand all Loading... | |
631 except Exception, ex: | 578 except Exception, ex: |
632 logging.error('The build for %s failed:', repo) | 579 logging.error('The build for %s failed:', repo) |
633 logging.exception(ex) | 580 logging.exception(ex) |
634 | 581 |
635 file = open(nightlyConfigFile, 'wb') | 582 file = open(nightlyConfigFile, 'wb') |
636 nightlyConfig.write(file) | 583 nightlyConfig.write(file) |
637 | 584 |
638 | 585 |
639 if __name__ == '__main__': | 586 if __name__ == '__main__': |
640 main() | 587 main() |
OLD | NEW |