Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/createNightlies.py

Issue 29352919: Issue 4419 - Use proper API in order to upload devbuilds to AMO (Closed) Base URL: https://hg.adblockplus.org/sitescripts
Patch Set: Got rid of unnecessary opener object Created Sept. 13, 2016, 2:18 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « .sitescripts.example ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-2016 Eyeo GmbH 2 # Copyright (C) 2006-2016 Eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 """ 16 """
17 17
18 Nightly builds generation script 18 Nightly builds generation script
19 ================================ 19 ================================
20 20
21 This script generates nightly builds of extensions, together 21 This script generates nightly builds of extensions, together
22 with changelogs and documentation. 22 with changelogs and documentation.
23 23
24 """ 24 """
25 25
26 import ConfigParser 26 import ConfigParser
27 import cookielib 27 import base64
28 from datetime import datetime 28 from datetime import datetime
29 import hashlib 29 import hashlib
30 import HTMLParser 30 import hmac
31 import json 31 import json
32 import logging 32 import logging
33 import os 33 import os
34 import pipes 34 import pipes
35 import random
35 import shutil 36 import shutil
36 import struct 37 import struct
37 import subprocess 38 import subprocess
38 import sys 39 import sys
39 import tempfile 40 import tempfile
40 import time 41 import time
41 from urllib import urlencode 42 from urllib import urlencode
42 import urllib2 43 import urllib2
43 import urlparse 44 import urlparse
44 from xml.dom.minidom import parse as parseXml 45 from xml.dom.minidom import parse as parseXml
(...skipping 339 matching lines...) Expand 10 before | Expand all | Expand 10 after
384 } 385 }
385 if os.path.exists(os.path.join(baseDir, changelogFile)): 386 if os.path.exists(os.path.join(baseDir, changelogFile)):
386 link['changelog'] = changelogFile 387 link['changelog'] = changelogFile
387 links.append(link) 388 links.append(link)
388 template = get_template(get_config().get('extensions', 'nightlyIndexPage ')) 389 template = get_template(get_config().get('extensions', 'nightlyIndexPage '))
389 template.stream({'config': self.config, 'links': links}).dump(outputPath ) 390 template.stream({'config': self.config, 'links': links}).dump(outputPath )
390 391
391 def uploadToMozillaAddons(self): 392 def uploadToMozillaAddons(self):
392 import urllib3 393 import urllib3
393 394
394 username = get_config().get('extensions', 'amo_username') 395 header = {
395 password = get_config().get('extensions', 'amo_password') 396 'alg': 'HS256', # HMAC-SHA256
397 'typ': 'JWT',
398 }
396 399
397 slug = self.config.galleryID 400 issued = int(time.time())
398 login_url = 'https://addons.mozilla.org/en-US/firefox/users/login' 401 payload = {
399 upload_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/uploa d' % slug 402 'iss': get_config().get('extensions', 'amo_key'),
400 add_url = 'https://addons.mozilla.org/en-US/developers/addon/%s/versions /add' % slug 403 'jti': random.random(),
404 'iat': issued,
405 'exp': issued + 60,
406 }
401 407
402 cookie_jar = cookielib.CookieJar() 408 input = '.'.join([
Sebastian Noack 2016/09/13 15:03:12 '{}.{}'.format(...) seems more appropriate rather
Wladimir Palant 2016/09/13 15:16:54 Done.
403 opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar)) 409 base64.b64encode(json.dumps(header)),
410 base64.b64encode(json.dumps(payload))
411 ])
404 412
405 def load_url(url, data=None): 413 signature = hmac.new(get_config().get('extensions', 'amo_secret'),
406 content_type = 'application/x-www-form-urlencoded' 414 msg=input,
407 if isinstance(data, dict): 415 digestmod=hashlib.sha256).digest()
408 if any(isinstance(v, tuple) for v in data.itervalues()): 416 token = '.'.join([input, base64.b64encode(signature)])
409 data, content_type = urllib3.filepost.encode_multipart_formd ata(data)
410 else:
411 data = urlencode(data.items())
412 417
413 request = urllib2.Request(url, data, headers={'Content-Type': conten t_type}) 418 upload_url = ('https://addons.mozilla.org/api/v3/addons/{0}/'
Sebastian Noack 2016/09/13 15:03:12 Nit: Note that the indices in the placeholders are
Wladimir Palant 2016/09/13 15:16:54 Done.
414 response = opener.open(request) 419 'versions/{1}/').format(self.extensionID, self.version)
415 try:
416 return response.read()
417 finally:
418 response.close()
419 420
420 class CSRFParser(HTMLParser.HTMLParser): 421 with open(self.path, 'rb') as file:
421 result = None 422 data, content_type = urllib3.filepost.encode_multipart_formdata({
422 dummy_exception = Exception() 423 'upload': (
424 os.path.basename(self.path),
425 file.read(),
426 'application/x-xpinstall'
427 )
428 })
423 429
424 def __init__(self, data): 430 request = urllib2.Request(upload_url, data=data)
425 HTMLParser.HTMLParser.__init__(self) 431 request.add_header('Content-Type', content_type)
426 try: 432 request.add_header('Authorization', 'JWT ' + token)
427 self.feed(data) 433 request.get_method = lambda: 'PUT'
428 self.close()
429 except Exception, e:
430 if e != self.dummy_exception:
431 raise
432 434
433 if not self.result: 435 try:
434 raise Exception('Failed to extract CSRF token') 436 urllib2.urlopen(request).close()
435 437 except urllib2.HTTPError as e:
436 def set_result(self, value): 438 logging.error(e.read())
Sebastian Noack 2016/09/13 15:03:12 The error response should be closed as well.
Wladimir Palant 2016/09/13 15:16:54 Done.
437 self.result = value 439 raise
438 raise self.dummy_exception
439
440 def handle_starttag(self, tag, attrs):
441 attrs = dict(attrs)
442 if tag == 'meta' and attrs.get('name') == 'csrf':
443 self.set_result(attrs.get('content'))
444 if tag == 'input' and attrs.get('name') == 'csrfmiddlewaretoken' :
445 self.set_result(attrs.get('value'))
446
447 # Extract anonymous CSRF token
448 login_page = load_url(login_url)
449 csrf_token = CSRFParser(login_page).result
450
451 # Log in and get session's CSRF token
452 main_page = load_url(
453 login_url,
454 {
455 'csrfmiddlewaretoken': csrf_token,
456 'username': username,
457 'password': password,
458 }
459 )
460 csrf_token = CSRFParser(main_page).result
461
462 # Upload build
463 with open(self.path, 'rb') as file:
464 upload_response = json.loads(load_url(
465 upload_url,
466 {
467 'csrfmiddlewaretoken': csrf_token,
468 'upload': (os.path.basename(self.path), file.read(), 'applic ation/x-xpinstall'),
469 }
470 ))
471
472 # Wait for validation to finish
473 while not upload_response.get('validation'):
474 time.sleep(2)
475 upload_response = json.loads(load_url(
476 upload_url + '/' + upload_response.get('upload')
477 ))
478
479 if upload_response['validation'].get('errors', 0):
480 raise Exception('Build failed AMO validation, see https://addons.moz illa.org%s' % upload_response.get('full_report_url'))
481
482 # Add version
483 add_response = json.loads(load_url(
484 add_url,
485 {
486 'csrfmiddlewaretoken': csrf_token,
487 'upload': upload_response.get('upload'),
488 'source': ('', '', 'application/octet-stream'),
489 'beta': 'on',
490 'supported_platforms': 1, # PLATFORM_ANY.id
491 }
492 ))
493 440
494 def uploadToChromeWebStore(self): 441 def uploadToChromeWebStore(self):
495 # Google APIs use HTTP error codes with error message in body. So we add 442 # Google APIs use HTTP error codes with error message in body. So we add
496 # the response body to the HTTPError to get more meaningful error messag es. 443 # the response body to the HTTPError to get more meaningful error messag es.
497 444
498 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler): 445 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
499 def http_error_default(self, req, fp, code, msg, hdrs): 446 def http_error_default(self, req, fp, code, msg, hdrs):
500 raise urllib2.HTTPError(req.get_full_url(), code, '%s\n%s' % (ms g, fp.read()), hdrs, fp) 447 raise urllib2.HTTPError(req.get_full_url(), code, '%s\n%s' % (ms g, fp.read()), hdrs, fp)
501 448
502 opener = urllib2.build_opener(HTTPErrorBodyHandler) 449 opener = urllib2.build_opener(HTTPErrorBodyHandler)
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after
593 540
594 if self.config.type == 'ie': 541 if self.config.type == 'ie':
595 self.writeIEUpdateManifest(versions) 542 self.writeIEUpdateManifest(versions)
596 543
597 # update index page 544 # update index page
598 self.updateIndex(versions) 545 self.updateIndex(versions)
599 546
600 # update nightlies config 547 # update nightlies config
601 self.config.latestRevision = self.revision 548 self.config.latestRevision = self.revision
602 549
603 if self.config.type == 'gecko' and self.config.galleryID and get_con fig().get('extensions', 'amo_username'): 550 if self.config.type == 'gecko' and self.config.galleryID and get_con fig().has_option('extensions', 'amo_key'):
604 self.uploadToMozillaAddons() 551 self.uploadToMozillaAddons()
605 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken: 552 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken:
606 self.uploadToChromeWebStore() 553 self.uploadToChromeWebStore()
607 finally: 554 finally:
608 # clean up 555 # clean up
609 if self.tempdir: 556 if self.tempdir:
610 shutil.rmtree(self.tempdir, ignore_errors=True) 557 shutil.rmtree(self.tempdir, ignore_errors=True)
611 558
612 559
613 def main(): 560 def main():
(...skipping 17 matching lines...) Expand all
631 except Exception, ex: 578 except Exception, ex:
632 logging.error('The build for %s failed:', repo) 579 logging.error('The build for %s failed:', repo)
633 logging.exception(ex) 580 logging.exception(ex)
634 581
635 file = open(nightlyConfigFile, 'wb') 582 file = open(nightlyConfigFile, 'wb')
636 nightlyConfig.write(file) 583 nightlyConfig.write(file)
637 584
638 585
639 if __name__ == '__main__': 586 if __name__ == '__main__':
640 main() 587 main()
OLDNEW
« no previous file with comments | « .sitescripts.example ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld