Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/createNightlies.py

Issue 29756624: Noissue - Adapt best practices for trailing commas (sitescripts) (Closed)
Patch Set: Re-run script on Python 2, added flake8-commas extension Created April 19, 2018, 2:35 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-present eyeo GmbH 2 # Copyright (C) 2006-present eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
(...skipping 30 matching lines...) Expand all
41 from urllib import urlencode 41 from urllib import urlencode
42 import urllib2 42 import urllib2
43 import urlparse 43 import urlparse
44 import zipfile 44 import zipfile
45 import contextlib 45 import contextlib
46 46
47 from xml.dom.minidom import parse as parseXml 47 from xml.dom.minidom import parse as parseXml
48 48
49 from sitescripts.extensions.utils import ( 49 from sitescripts.extensions.utils import (
50 compareVersions, Configuration, 50 compareVersions, Configuration,
51 writeAndroidUpdateManifest 51 writeAndroidUpdateManifest,
52 ) 52 )
53 from sitescripts.utils import get_config, get_template 53 from sitescripts.utils import get_config, get_template
54 54
55 MAX_BUILDS = 50 55 MAX_BUILDS = 50
56 56
57 57
58 # Google and Microsoft APIs use HTTP error codes with error message in 58 # Google and Microsoft APIs use HTTP error codes with error message in
59 # body. So we add the response body to the HTTPError to get more 59 # body. So we add the response body to the HTTPError to get more
60 # meaningful error messages. 60 # meaningful error messages.
61 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler): 61 class HTTPErrorBodyHandler(urllib2.HTTPDefaultErrorHandler):
(...skipping 26 matching lines...) Expand all
88 88
89 def hasChanges(self): 89 def hasChanges(self):
90 return self.revision != self.previousRevision 90 return self.revision != self.previousRevision
91 91
92 def getCurrentRevision(self): 92 def getCurrentRevision(self):
93 """ 93 """
94 retrieves the current revision ID from the repository 94 retrieves the current revision ID from the repository
95 """ 95 """
96 command = [ 96 command = [
97 'hg', 'id', '-i', '-r', self.config.revision, '--config', 97 'hg', 'id', '-i', '-r', self.config.revision, '--config',
98 'defaults.id=', self.config.repository 98 'defaults.id=', self.config.repository,
99 ] 99 ]
100 return subprocess.check_output(command).strip() 100 return subprocess.check_output(command).strip()
101 101
102 def getCurrentBuild(self): 102 def getCurrentBuild(self):
103 """ 103 """
104 calculates the (typically numerical) build ID for the current build 104 calculates the (typically numerical) build ID for the current build
105 """ 105 """
106 command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir] 106 command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
107 build = subprocess.check_output(command).strip() 107 build = subprocess.check_output(command).strip()
108 if self.config.type in {'gecko', 'gecko-webext'}: 108 if self.config.type in {'gecko', 'gecko-webext'}:
109 build += 'beta' 109 build += 'beta'
110 return build 110 return build
111 111
112 def getChanges(self): 112 def getChanges(self):
113 """ 113 """
114 retrieve changes between the current and previous ("first") revision 114 retrieve changes between the current and previous ("first") revision
115 """ 115 """
116 command = [ 116 command = [
117 'hg', 'log', '-R', self.tempdir, '-r', 117 'hg', 'log', '-R', self.tempdir, '-r',
118 'reverse(ancestors({}))'.format(self.config.revision), '-l', '50', 118 'reverse(ancestors({}))'.format(self.config.revision), '-l', '50',
119 '--encoding', 'utf-8', '--template', 119 '--encoding', 'utf-8', '--template',
120 '{date|isodate}\\0{author|person}\\0{rev}\\0{desc}\\0\\0', 120 '{date|isodate}\\0{author|person}\\0{rev}\\0{desc}\\0\\0',
121 '--config', 'defaults.log=' 121 '--config', 'defaults.log=',
122 ] 122 ]
123 result = subprocess.check_output(command).decode('utf-8') 123 result = subprocess.check_output(command).decode('utf-8')
124 124
125 for change in result.split('\x00\x00'): 125 for change in result.split('\x00\x00'):
126 if change: 126 if change:
127 date, author, revision, description = change.split('\x00') 127 date, author, revision, description = change.split('\x00')
128 yield {'date': date, 'author': author, 'revision': revision, 'de scription': description} 128 yield {'date': date, 'author': author, 'revision': revision, 'de scription': description}
129 129
130 def copyRepository(self): 130 def copyRepository(self):
131 """ 131 """
(...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after
265 os.makedirs(baseDir) 265 os.makedirs(baseDir)
266 266
267 # ABP for Android used to have its own update manifest format. We need t o 267 # ABP for Android used to have its own update manifest format. We need t o
268 # generate both that and the new one in the libadblockplus format as lon g 268 # generate both that and the new one in the libadblockplus format as lon g
269 # as a significant amount of users is on an old version. 269 # as a significant amount of users is on an old version.
270 if self.config.type == 'android': 270 if self.config.type == 'android':
271 newManifestPath = os.path.join(baseDir, 'update.json') 271 newManifestPath = os.path.join(baseDir, 'update.json')
272 writeAndroidUpdateManifest(newManifestPath, [{ 272 writeAndroidUpdateManifest(newManifestPath, [{
273 'basename': self.basename, 273 'basename': self.basename,
274 'version': self.version, 274 'version': self.version,
275 'updateURL': self.updateURL 275 'updateURL': self.updateURL,
276 }]) 276 }])
277 277
278 template = get_template(get_config().get('extensions', templateName), 278 template = get_template(get_config().get('extensions', templateName),
279 autoescape=autoescape) 279 autoescape=autoescape)
280 template.stream({'extensions': [self]}).dump(manifestPath) 280 template.stream({'extensions': [self]}).dump(manifestPath)
281 281
282 def writeIEUpdateManifest(self, versions): 282 def writeIEUpdateManifest(self, versions):
283 """ 283 """
284 Writes update.json file for the latest IE build 284 Writes update.json file for the latest IE build
285 """ 285 """
286 if len(versions) == 0: 286 if len(versions) == 0:
287 return 287 return
288 288
289 version = versions[0] 289 version = versions[0]
290 packageName = self.basename + '-' + version + self.config.packageSuffix 290 packageName = self.basename + '-' + version + self.config.packageSuffix
291 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + ' /' + packageName + '?update') 291 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + ' /' + packageName + '?update')
292 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 292 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
293 manifestPath = os.path.join(baseDir, 'update.json') 293 manifestPath = os.path.join(baseDir, 'update.json')
294 294
295 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit e 295 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit e
296 doWrite(manifestPath, [{ 296 doWrite(manifestPath, [{
297 'basename': self.basename, 297 'basename': self.basename,
298 'version': version, 298 'version': version,
299 'updateURL': updateURL 299 'updateURL': updateURL,
300 }]) 300 }])
301 301
302 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']: 302 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']:
303 linkPath = os.path.join(baseDir, '00latest%s' % suffix) 303 linkPath = os.path.join(baseDir, '00latest%s' % suffix)
304 outputPath = os.path.join(baseDir, self.basename + '-' + version + s uffix) 304 outputPath = os.path.join(baseDir, self.basename + '-' + version + s uffix)
305 if hasattr(os, 'symlink'): 305 if hasattr(os, 'symlink'):
306 if os.path.exists(linkPath): 306 if os.path.exists(linkPath):
307 os.remove(linkPath) 307 os.remove(linkPath)
308 os.symlink(os.path.basename(outputPath), linkPath) 308 os.symlink(os.path.basename(outputPath), linkPath)
309 else: 309 else:
(...skipping 14 matching lines...) Expand all
324 apkFile = open(self.path, 'wb') 324 apkFile = open(self.path, 'wb')
325 325
326 try: 326 try:
327 try: 327 try:
328 port = get_config().get('extensions', 'androidBuildPort') 328 port = get_config().get('extensions', 'androidBuildPort')
329 except ConfigParser.NoOptionError: 329 except ConfigParser.NoOptionError:
330 port = '22' 330 port = '22'
331 command = ['ssh', '-p', port, get_config().get('extensions', 'an droidBuildHost')] 331 command = ['ssh', '-p', port, get_config().get('extensions', 'an droidBuildHost')]
332 command.extend(map(pipes.quote, [ 332 command.extend(map(pipes.quote, [
333 '/home/android/bin/makedebugbuild.py', '--revision', 333 '/home/android/bin/makedebugbuild.py', '--revision',
334 self.buildNum, '--version', self.version, '--stdout' 334 self.buildNum, '--version', self.version, '--stdout',
335 ])) 335 ]))
336 subprocess.check_call(command, stdout=apkFile, close_fds=True) 336 subprocess.check_call(command, stdout=apkFile, close_fds=True)
337 except: 337 except:
338 # clear broken output if any 338 # clear broken output if any
339 if os.path.exists(self.path): 339 if os.path.exists(self.path):
340 os.remove(self.path) 340 os.remove(self.path)
341 raise 341 raise
342 else: 342 else:
343 env = os.environ 343 env = os.environ
344 spiderMonkeyBinary = self.config.spiderMonkeyBinary 344 spiderMonkeyBinary = self.config.spiderMonkeyBinary
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after
399 packageFile = self.basename + '-' + version + self.config.packageSuf fix 399 packageFile = self.basename + '-' + version + self.config.packageSuf fix
400 changelogFile = self.basename + '-' + version + '.changelog.xhtml' 400 changelogFile = self.basename + '-' + version + '.changelog.xhtml'
401 if not os.path.exists(os.path.join(baseDir, packageFile)): 401 if not os.path.exists(os.path.join(baseDir, packageFile)):
402 # Oops 402 # Oops
403 continue 403 continue
404 404
405 link = { 405 link = {
406 'version': version, 406 'version': version,
407 'download': packageFile, 407 'download': packageFile,
408 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)), 408 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)),
409 'size': os.path.getsize(os.path.join(baseDir, packageFile)) 409 'size': os.path.getsize(os.path.join(baseDir, packageFile)),
410 } 410 }
411 if os.path.exists(os.path.join(baseDir, changelogFile)): 411 if os.path.exists(os.path.join(baseDir, changelogFile)):
412 link['changelog'] = changelogFile 412 link['changelog'] = changelogFile
413 links.append(link) 413 links.append(link)
414 template = get_template(get_config().get('extensions', 'nightlyIndexPage ')) 414 template = get_template(get_config().get('extensions', 'nightlyIndexPage '))
415 template.stream({'config': self.config, 'links': links}).dump(outputPath ) 415 template.stream({'config': self.config, 'links': links}).dump(outputPath )
416 416
417 def uploadToMozillaAddons(self): 417 def uploadToMozillaAddons(self):
418 import urllib3 418 import urllib3
419 419
420 header = { 420 header = {
421 'alg': 'HS256', # HMAC-SHA256 421 'alg': 'HS256', # HMAC-SHA256
422 'typ': 'JWT', 422 'typ': 'JWT',
423 } 423 }
424 424
425 issued = int(time.time()) 425 issued = int(time.time())
426 payload = { 426 payload = {
427 'iss': get_config().get('extensions', 'amo_key'), 427 'iss': get_config().get('extensions', 'amo_key'),
428 'jti': random.random(), 428 'jti': random.random(),
429 'iat': issued, 429 'iat': issued,
430 'exp': issued + 60, 430 'exp': issued + 60,
431 } 431 }
432 432
433 input = '{}.{}'.format( 433 input = '{}.{}'.format(
434 base64.b64encode(json.dumps(header)), 434 base64.b64encode(json.dumps(header)),
435 base64.b64encode(json.dumps(payload)) 435 base64.b64encode(json.dumps(payload)),
436 ) 436 )
437 437
438 signature = hmac.new(get_config().get('extensions', 'amo_secret'), 438 signature = hmac.new(get_config().get('extensions', 'amo_secret'),
439 msg=input, 439 msg=input,
440 digestmod=hashlib.sha256).digest() 440 digestmod=hashlib.sha256).digest()
441 token = '{}.{}'.format(input, base64.b64encode(signature)) 441 token = '{}.{}'.format(input, base64.b64encode(signature))
442 442
443 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/' 443 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
444 'versions/{}/').format(self.extensionID, self.version) 444 'versions/{}/').format(self.extensionID, self.version)
445 445
446 with open(self.path, 'rb') as file: 446 with open(self.path, 'rb') as file:
447 data, content_type = urllib3.filepost.encode_multipart_formdata({ 447 data, content_type = urllib3.filepost.encode_multipart_formdata({
448 'upload': ( 448 'upload': (
449 os.path.basename(self.path), 449 os.path.basename(self.path),
450 file.read(), 450 file.read(),
451 'application/x-xpinstall' 451 'application/x-xpinstall',
452 ) 452 ),
453 }) 453 })
454 454
455 request = urllib2.Request(upload_url, data=data) 455 request = urllib2.Request(upload_url, data=data)
456 request.add_header('Content-Type', content_type) 456 request.add_header('Content-Type', content_type)
457 request.add_header('Authorization', 'JWT ' + token) 457 request.add_header('Authorization', 'JWT ' + token)
458 request.get_method = lambda: 'PUT' 458 request.get_method = lambda: 'PUT'
459 459
460 try: 460 try:
461 urllib2.urlopen(request).close() 461 urllib2.urlopen(request).close()
462 except urllib2.HTTPError as e: 462 except urllib2.HTTPError as e:
(...skipping 11 matching lines...) Expand all
474 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh 474 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh
475 475
476 response = json.load(opener.open( 476 response = json.load(opener.open(
477 'https://accounts.google.com/o/oauth2/token', 477 'https://accounts.google.com/o/oauth2/token',
478 478
479 urlencode([ 479 urlencode([
480 ('refresh_token', self.config.refreshToken), 480 ('refresh_token', self.config.refreshToken),
481 ('client_id', self.config.clientID), 481 ('client_id', self.config.clientID),
482 ('client_secret', self.config.clientSecret), 482 ('client_secret', self.config.clientSecret),
483 ('grant_type', 'refresh_token'), 483 ('grant_type', 'refresh_token'),
484 ]) 484 ]),
485 )) 485 ))
486 486
487 auth_token = '%s %s' % (response['token_type'], response['access_token'] ) 487 auth_token = '%s %s' % (response['token_type'], response['access_token'] )
488 488
489 # upload a new version with the Chrome Web Store API 489 # upload a new version with the Chrome Web Store API
490 # https://developer.chrome.com/webstore/using_webstore_api#uploadexisitn g 490 # https://developer.chrome.com/webstore/using_webstore_api#uploadexisitn g
491 491
492 request = urllib2.Request('https://www.googleapis.com/upload/chromewebst ore/v1.1/items/' + self.config.devbuildGalleryID) 492 request = urllib2.Request('https://www.googleapis.com/upload/chromewebst ore/v1.1/items/' + self.config.devbuildGalleryID)
493 request.get_method = lambda: 'PUT' 493 request.get_method = lambda: 'PUT'
494 request.add_header('Authorization', auth_token) 494 request.add_header('Authorization', auth_token)
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after
528 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo ry-protocols-oauth-code#refreshing-the-access-tokens 528 # https://docs.microsoft.com/en-us/azure/active-directory/active-directo ry-protocols-oauth-code#refreshing-the-access-tokens
529 server = 'https://login.microsoftonline.com' 529 server = 'https://login.microsoftonline.com'
530 token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID) 530 token_path = '{}/{}/oauth2/token'.format(server, self.config.tenantID)
531 531
532 opener = urllib2.build_opener(HTTPErrorBodyHandler) 532 opener = urllib2.build_opener(HTTPErrorBodyHandler)
533 post_data = urlencode([ 533 post_data = urlencode([
534 ('refresh_token', self.config.refreshToken), 534 ('refresh_token', self.config.refreshToken),
535 ('client_id', self.config.clientID), 535 ('client_id', self.config.clientID),
536 ('client_secret', self.config.clientSecret), 536 ('client_secret', self.config.clientSecret),
537 ('grant_type', 'refresh_token'), 537 ('grant_type', 'refresh_token'),
538 ('resource', 'https://graph.windows.net') 538 ('resource', 'https://graph.windows.net'),
539 ]) 539 ])
540 request = urllib2.Request(token_path, post_data) 540 request = urllib2.Request(token_path, post_data)
541 with contextlib.closing(opener.open(request)) as response: 541 with contextlib.closing(opener.open(request)) as response:
542 data = json.load(response) 542 data = json.load(response)
543 auth_token = '{0[token_type]} {0[access_token]}'.format(data) 543 auth_token = '{0[token_type]} {0[access_token]}'.format(data)
544 544
545 return auth_token 545 return auth_token
546 546
547 def upload_appx_file_to_windows_store(self, file_upload_url): 547 def upload_appx_file_to_windows_store(self, file_upload_url):
548 # Add .appx file to a .zip file 548 # Add .appx file to a .zip file
(...skipping 20 matching lines...) Expand all
569 def upload_to_windows_store(self): 569 def upload_to_windows_store(self):
570 opener = urllib2.build_opener(HTTPErrorBodyHandler) 570 opener = urllib2.build_opener(HTTPErrorBodyHandler)
571 571
572 headers = {'Authorization': self.get_windows_store_access_token(), 572 headers = {'Authorization': self.get_windows_store_access_token(),
573 'Content-type': 'application/json'} 573 'Content-type': 'application/json'}
574 574
575 # Get application 575 # Get application
576 # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app 576 # https://docs.microsoft.com/en-us/windows/uwp/monetize/get-an-app
577 api_path = '{}/v1.0/my/applications/{}'.format( 577 api_path = '{}/v1.0/my/applications/{}'.format(
578 'https://manage.devcenter.microsoft.com', 578 'https://manage.devcenter.microsoft.com',
579 self.config.devbuildGalleryID 579 self.config.devbuildGalleryID,
580 ) 580 )
581 581
582 request = urllib2.Request(api_path, None, headers) 582 request = urllib2.Request(api_path, None, headers)
583 with contextlib.closing(opener.open(request)) as response: 583 with contextlib.closing(opener.open(request)) as response:
584 app_obj = json.load(response) 584 app_obj = json.load(response)
585 585
586 # Delete existing in-progress submission 586 # Delete existing in-progress submission
587 # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-su bmission 587 # https://docs.microsoft.com/en-us/windows/uwp/monetize/delete-an-app-su bmission
588 submissions_path = api_path + '/submissions' 588 submissions_path = api_path + '/submissions'
589 if 'pendingApplicationSubmission' in app_obj: 589 if 'pendingApplicationSubmission' in app_obj:
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after
717 except Exception as ex: 717 except Exception as ex:
718 logging.error('The build for %s failed:', repo) 718 logging.error('The build for %s failed:', repo)
719 logging.exception(ex) 719 logging.exception(ex)
720 720
721 file = open(nightlyConfigFile, 'wb') 721 file = open(nightlyConfigFile, 'wb')
722 nightlyConfig.write(file) 722 nightlyConfig.write(file)
723 723
724 724
725 if __name__ == '__main__': 725 if __name__ == '__main__':
726 main() 726 main()
OLDNEW
« no previous file with comments | « sitescripts/content_blocker_lists/bin/generate_lists.py ('k') | sitescripts/extensions/bin/updateRecommendations.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld