Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: sitescripts/extensions/bin/createNightlies.py

Issue 29716693: Issue 6371 - Update buildtools dep. to c830dfa08e2f, use AMO-signing API (Closed) Base URL: https://hg.adblockplus.org/abpssembly/file/1e38c3375fa3
Left Patch Set: Created March 8, 2018, 9:22 a.m.
Right Patch Set: NO CHANGE - Rebase against current master Created March 12, 2018, 7:29 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « ensure_dependencies.py ('k') | sitescripts/extensions/template/gecko.json » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-present eyeo GmbH 2 # Copyright (C) 2006-present eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
(...skipping 330 matching lines...) Expand 10 before | Expand all | Expand 10 after
341 except: 341 except:
342 # clear broken output if any 342 # clear broken output if any
343 if os.path.exists(self.path): 343 if os.path.exists(self.path):
344 os.remove(self.path) 344 os.remove(self.path)
345 raise 345 raise
346 else: 346 else:
347 env = os.environ 347 env = os.environ
348 spiderMonkeyBinary = self.config.spiderMonkeyBinary 348 spiderMonkeyBinary = self.config.spiderMonkeyBinary
349 if spiderMonkeyBinary: 349 if spiderMonkeyBinary:
350 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary) 350 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
351
352 command = [os.path.join(self.tempdir, 'build.py')]
351 if self.config.type == 'safari': 353 if self.config.type == 'safari':
352 cmd_order = ['-t', self.config.type, 'build'] 354 command.extend(['-t', self.config.type, 'build'])
353 else: 355 else:
354 cmd_order = ['build', '-t', self.config.type] 356 command.extend(['build', '-t', self.config.type])
355 command = [os.path.join(self.tempdir, 'build.py')] + cmd_order 357 command.extend(['-b', self.buildNum])
356 command += ['-b', self.buildNum]
Sebastian Noack 2018/03/09 00:33:18 You could use list.extend() in order to modify the
tlucas 2018/03/09 08:12:57 Done.
357 358
358 if self.config.type not in {'gecko', 'edge'}: 359 if self.config.type not in {'gecko', 'edge'}:
359 command.extend(['-k', self.config.keyFile]) 360 command.extend(['-k', self.config.keyFile])
360 command.append(self.path) 361 command.append(self.path)
361 subprocess.check_call(command, env=env) 362 subprocess.check_call(command, env=env)
362 363
363 if not os.path.exists(self.path): 364 if not os.path.exists(self.path):
364 raise Exception("Build failed, output file hasn't been created") 365 raise Exception("Build failed, output file hasn't been created")
365 366
366 if self.config.type not in self.downloadable_repos: 367 if self.config.type not in self.downloadable_repos:
367 linkPath = os.path.join(baseDir, 368 linkPath = os.path.join(baseDir,
368 '00latest%s' % self.config.packageSuffix) 369 '00latest' + self.config.packageSuffix)
Sebastian Noack 2018/03/09 00:33:19 Please use .format() instead of %. flake8-eyeo wou
tlucas 2018/03/09 08:12:58 This is flawed, yes - but .format() should be used
Sebastian Noack 2018/03/09 16:16:27 You are right, since we just append in the end, +
369 self.symlink_or_copy(self.path, linkPath) 370 self.symlink_or_copy(self.path, linkPath)
370 371
371 def retireBuilds(self): 372 def retireBuilds(self):
372 """ 373 """
373 removes outdated builds, returns the sorted version numbers of remaini ng 374 removes outdated builds, returns the sorted version numbers of remaini ng
374 builds 375 builds
375 """ 376 """
376 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 377 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
377 versions = [] 378 versions = []
378 prefix = self.basename + '-' 379 prefix = self.basename + '-'
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after
415 } 416 }
416 if os.path.exists(os.path.join(baseDir, changelogFile)): 417 if os.path.exists(os.path.join(baseDir, changelogFile)):
417 link['changelog'] = changelogFile 418 link['changelog'] = changelogFile
418 links.append(link) 419 links.append(link)
419 template = get_template(get_config().get('extensions', 'nightlyIndexPage ')) 420 template = get_template(get_config().get('extensions', 'nightlyIndexPage '))
420 template.stream({'config': self.config, 'links': links}).dump(outputPath ) 421 template.stream({'config': self.config, 'links': links}).dump(outputPath )
421 422
422 def read_downloads_lockfile(self): 423 def read_downloads_lockfile(self):
423 path = get_config().get('extensions', 'downloadLockFile') 424 path = get_config().get('extensions', 'downloadLockFile')
424 try: 425 try:
425 current = {}
426 with open(path, 'r') as fp: 426 with open(path, 'r') as fp:
427 current = json.load(fp) 427 current = json.load(fp)
428 except IOError: 428 except IOError:
429 logging.warning('No lockfile found. Creating ' + path) 429 logging.warning('No lockfile found. Creating ' + path)
430 current = {}
430 431
431 return current 432 return current
432 433
433 def write_downloads_lockfile(self, values): 434 def write_downloads_lockfile(self, values):
434 path = get_config().get('extensions', 'downloadLockFile') 435 path = get_config().get('extensions', 'downloadLockFile')
435 with open(path, 'w') as fp: 436 with open(path, 'w') as fp:
436 json.dump(values, fp) 437 json.dump(values, fp)
437 438
438 def add_to_downloads_lockfile(self, platform, values): 439 def add_to_downloads_lockfile(self, platform, values):
439 current = self.read_downloads_lockfile() 440 current = self.read_downloads_lockfile()
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after
504 'application/x-xpinstall' 505 'application/x-xpinstall'
505 ) 506 )
506 }) 507 })
507 508
508 request = self.generate_jwt_request( 509 request = self.generate_jwt_request(
509 config.get('extensions', 'amo_key'), 510 config.get('extensions', 'amo_key'),
510 config.get('extensions', 'amo_secret'), 511 config.get('extensions', 'amo_secret'),
511 upload_url, 512 upload_url,
512 'PUT', 513 'PUT',
513 data, 514 data,
514 (('Content-Type', content_type),), 515 [('Content-Type', content_type)]
515 ) 516 )
516 517
517 try: 518 try:
518 urllib2.urlopen(request).close() 519 urllib2.urlopen(request).close()
519 except urllib2.HTTPError as e: 520 except urllib2.HTTPError as e:
520 try: 521 try:
521 logging.error(e.read()) 522 logging.error(e.read())
522 finally: 523 finally:
523 e.close() 524 e.close()
524 raise 525 raise
(...skipping 21 matching lines...) Expand all
546 547
547 necessary = ['passed_review', 'reviewed', 'processed', 'valid'] 548 necessary = ['passed_review', 'reviewed', 'processed', 'valid']
548 if all(response[x] for x in necessary): 549 if all(response[x] for x in necessary):
549 download_url = response['files'][0]['download_url'] 550 download_url = response['files'][0]['download_url']
550 checksum = response['files'][0]['hash'] 551 checksum = response['files'][0]['hash']
551 552
552 filename = '{}-{}.xpi'.format(self.basename, version) 553 filename = '{}-{}.xpi'.format(self.basename, version)
553 file_path = os.path.join( 554 file_path = os.path.join(
554 config.get('extensions', 'nightliesDirectory'), 555 config.get('extensions', 'nightliesDirectory'),
555 self.basename, 556 self.basename,
556 filename, 557 filename
Sebastian Noack 2018/03/09 00:33:19 Nit: Redundant comma after last argument.
tlucas 2018/03/09 08:12:57 Done.
557 ) 558 )
558 559
559 request = self.generate_jwt_request(iss, secret, download_url, 560 request = self.generate_jwt_request(iss, secret, download_url,
560 'GET') 561 'GET')
561 try: 562 try:
562 response = urllib2.urlopen(request) 563 response = urllib2.urlopen(request)
563 except urllib2.HTTPError as e: 564 except urllib2.HTTPError as e:
564 logging.error(e.read()) 565 logging.error(e.read())
565 566
566 # Verify the extension's integrity 567 # Verify the extension's integrity
567 file_content = response.read() 568 file_content = response.read()
568 sha256 = hashlib.sha256(file_content) 569 sha256 = hashlib.sha256(file_content)
569 returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest()) 570 returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest())
570 571
571 if returned_checksum != checksum: 572 if returned_checksum != checksum:
572 logging.error('Checksum could not be verified: {} vs {}' 573 logging.error('Checksum could not be verified: {} vs {}'
573 ''.format(checksum, returned_checksum)) 574 ''.format(checksum, returned_checksum))
574 575
575 with open(file_path, 'w') as fp: 576 with open(file_path, 'w') as fp:
576 fp.write(file_content) 577 fp.write(file_content)
577 578
578 self.update_link = os.path.join( 579 self.update_link = os.path.join(
579 config.get('extensions', 'nightliesURL'), 580 config.get('extensions', 'nightliesURL'),
580 self.basename, 581 self.basename,
581 filename, 582 filename
Sebastian Noack 2018/03/09 00:33:19 Nit: Redundant comma after last argument.
tlucas 2018/03/09 08:12:57 Done.
582 ) 583 )
583 584
584 self.remove_from_downloads_lockfile(self.config.type, 585 self.remove_from_downloads_lockfile(self.config.type,
585 'version', 586 'version',
586 version) 587 version)
587 elif not response['passed_review'] or not response['valid']: 588 elif not response['passed_review'] or not response['valid']:
588 # When the review failed for any reason, we want to know about it 589 # When the review failed for any reason, we want to know about it
589 logging.error(json.dumps(response, indent=4)) 590 logging.error(json.dumps(response, indent=4))
590 self.remove_from_downloads_lockfile(self.config.type, 591 self.remove_from_downloads_lockfile(self.config.type,
591 'version', 592 'version',
(...skipping 288 matching lines...) Expand 10 before | Expand all | Expand 10 after
880 881
881 file = open(nightlyConfigFile, 'wb') 882 file = open(nightlyConfigFile, 'wb')
882 nightlyConfig.write(file) 883 nightlyConfig.write(file)
883 884
884 885
885 if __name__ == '__main__': 886 if __name__ == '__main__':
886 parser = argparse.ArgumentParser() 887 parser = argparse.ArgumentParser()
887 parser.add_argument('--download', action='store_true', default=False) 888 parser.add_argument('--download', action='store_true', default=False)
888 args = parser.parse_args() 889 args = parser.parse_args()
889 main(args.download) 890 main(args.download)
LEFTRIGHT

Powered by Google App Engine
This is Rietveld