Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/createNightlies.py

Issue 29716693: Issue 6371 - Update buildtools dep. to c830dfa08e2f, use AMO-signing API (Closed) Base URL: https://hg.adblockplus.org/abpssembly/file/1e38c3375fa3
Patch Set: Created March 7, 2018, 10:05 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-present eyeo GmbH 2 # Copyright (C) 2006-present eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 """ 16 """
17 17
18 Nightly builds generation script 18 Nightly builds generation script
19 ================================ 19 ================================
20 20
21 This script generates nightly builds of extensions, together 21 This script generates nightly builds of extensions, together
22 with changelogs and documentation. 22 with changelogs and documentation.
23 23
24 """ 24 """
25 25
26 import argparse
26 import ConfigParser 27 import ConfigParser
27 import base64 28 import base64
28 import hashlib 29 import hashlib
29 import hmac 30 import hmac
30 import json 31 import json
31 import logging 32 import logging
32 import os 33 import os
33 import pipes 34 import pipes
34 import random 35 import random
35 import shutil 36 import shutil
(...skipping 27 matching lines...) Expand all
63 raise urllib2.HTTPError(req.get_full_url(), code, 64 raise urllib2.HTTPError(req.get_full_url(), code,
64 '{}\n{}'.format(msg, fp.read()), hdrs, fp) 65 '{}\n{}'.format(msg, fp.read()), hdrs, fp)
65 66
66 67
67 class NightlyBuild(object): 68 class NightlyBuild(object):
68 """ 69 """
69 Performs the build process for an extension, 70 Performs the build process for an extension,
70 generating changelogs and documentation. 71 generating changelogs and documentation.
71 """ 72 """
72 73
74 downloadable_repos = {'gecko'}
75
73 def __init__(self, config): 76 def __init__(self, config):
74 """ 77 """
75 Creates a NightlyBuild instance; we are simply 78 Creates a NightlyBuild instance; we are simply
76 recording the configuration settings here. 79 recording the configuration settings here.
77 """ 80 """
78 self.config = config 81 self.config = config
79 self.revision = self.getCurrentRevision() 82 self.revision = self.getCurrentRevision()
80 try: 83 try:
81 self.previousRevision = config.latestRevision 84 self.previousRevision = config.latestRevision
82 except: 85 except:
(...skipping 15 matching lines...) Expand all
98 'defaults.id=', self.config.repository 101 'defaults.id=', self.config.repository
99 ] 102 ]
100 return subprocess.check_output(command).strip() 103 return subprocess.check_output(command).strip()
101 104
102 def getCurrentBuild(self): 105 def getCurrentBuild(self):
103 """ 106 """
104 calculates the (typically numerical) build ID for the current build 107 calculates the (typically numerical) build ID for the current build
105 """ 108 """
106 command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir] 109 command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
107 build = subprocess.check_output(command).strip() 110 build = subprocess.check_output(command).strip()
108 if self.config.type == 'gecko':
109 build += 'beta'
110 return build 111 return build
111 112
112 def getChanges(self): 113 def getChanges(self):
113 """ 114 """
114 retrieve changes between the current and previous ("first") revision 115 retrieve changes between the current and previous ("first") revision
115 """ 116 """
116 command = [ 117 command = [
117 'hg', 'log', '-R', self.tempdir, '-r', 118 'hg', 'log', '-R', self.tempdir, '-r',
118 'reverse(ancestors({}))'.format(self.config.revision), '-l', '50', 119 'reverse(ancestors({}))'.format(self.config.revision), '-l', '50',
119 '--encoding', 'utf-8', '--template', 120 '--encoding', 'utf-8', '--template',
(...skipping 14 matching lines...) Expand all
134 self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName) 135 self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName)
135 command = ['hg', 'clone', '-q', self.config.repository, '-u', 136 command = ['hg', 'clone', '-q', self.config.repository, '-u',
136 self.config.revision, self.tempdir] 137 self.config.revision, self.tempdir]
137 subprocess.check_call(command) 138 subprocess.check_call(command)
138 139
139 # Make sure to run ensure_dependencies.py if present 140 # Make sure to run ensure_dependencies.py if present
140 depscript = os.path.join(self.tempdir, 'ensure_dependencies.py') 141 depscript = os.path.join(self.tempdir, 'ensure_dependencies.py')
141 if os.path.isfile(depscript): 142 if os.path.isfile(depscript):
142 subprocess.check_call([sys.executable, depscript, '-q']) 143 subprocess.check_call([sys.executable, depscript, '-q'])
143 144
145 def symlink_or_copy(self, source, target):
146 if hasattr(os, 'symlink'):
147 if os.path.exists(target):
148 os.remove(target)
149 os.symlink(os.path.basename(source), target)
150 else:
151 shutil.copyfile(source, target)
152
144 def writeChangelog(self, changes): 153 def writeChangelog(self, changes):
145 """ 154 """
146 write the changelog file into the cloned repository 155 write the changelog file into the cloned repository
147 """ 156 """
148 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 157 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
149 if not os.path.exists(baseDir): 158 if not os.path.exists(baseDir):
150 os.makedirs(baseDir) 159 os.makedirs(baseDir)
151 changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version) 160 changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version)
152 changelogPath = os.path.join(baseDir, changelogFile) 161 changelogPath = os.path.join(baseDir, changelogFile)
153 self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.base name + '/' + changelogFile) 162 self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.base name + '/' + changelogFile)
154 163
155 template = get_template(get_config().get('extensions', 'changelogTemplat e')) 164 template = get_template(get_config().get('extensions', 'changelogTemplat e'))
156 template.stream({'changes': changes}).dump(changelogPath, encoding='utf- 8') 165 template.stream({'changes': changes}).dump(changelogPath, encoding='utf- 8')
157 166
158 linkPath = os.path.join(baseDir, '00latest.changelog.xhtml') 167 linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
159 if hasattr(os, 'symlink'): 168 self.symlink_or_copy(changelogPath, linkPath)
160 if os.path.exists(linkPath):
161 os.remove(linkPath)
162 os.symlink(os.path.basename(changelogPath), linkPath)
163 else:
164 shutil.copyfile(changelogPath, linkPath)
165 169
166 def readGeckoMetadata(self): 170 def readGeckoMetadata(self):
167 """ 171 """
168 read Gecko-specific metadata file from a cloned repository 172 read Gecko-specific metadata file from a cloned repository
169 and parse id, version, basename and the compat section 173 and parse id, version, basename and the compat section
170 out of the file 174 out of the file
171 """ 175 """
172 import buildtools.packagerChrome as packager 176 import buildtools.packagerChrome as packager
173 metadata = packager.readMetadata(self.tempdir, self.config.type) 177 metadata = packager.readMetadata(self.tempdir, self.config.type)
174 self.extensionID = metadata.get('general', 'id') 178 self.extensionID = packager.get_app_id(False, metadata)
175 self.version = packager.getBuildVersion(self.tempdir, metadata, False, 179 self.version = packager.getBuildVersion(self.tempdir, metadata, False,
176 self.buildNum) 180 self.buildNum)
177 self.basename = metadata.get('general', 'basename') 181 self.basename = metadata.get('general', 'basename')
182 self.min_version = metadata.get('compat', 'gecko')
178 183
179 def readAndroidMetadata(self): 184 def readAndroidMetadata(self):
180 """ 185 """
181 Read Android-specific metadata from AndroidManifest.xml file. 186 Read Android-specific metadata from AndroidManifest.xml file.
182 """ 187 """
183 manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), ' r') 188 manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), ' r')
184 manifest = parseXml(manifestFile) 189 manifest = parseXml(manifestFile)
185 manifestFile.close() 190 manifestFile.close()
186 191
187 root = manifest.documentElement 192 root = manifest.documentElement
(...skipping 26 matching lines...) Expand all
214 self.version = packager.getBuildVersion(self.tempdir, metadata, False, 219 self.version = packager.getBuildVersion(self.tempdir, metadata, False,
215 self.buildNum) 220 self.buildNum)
216 self.basename = metadata.get('general', 'basename') 221 self.basename = metadata.get('general', 'basename')
217 222
218 self.compat = [] 223 self.compat = []
219 if metadata.has_section('compat') and metadata.has_option('compat', 'chr ome'): 224 if metadata.has_section('compat') and metadata.has_option('compat', 'chr ome'):
220 self.compat.append({'id': 'chrome', 'minVersion': metadata.get('comp at', 'chrome')}) 225 self.compat.append({'id': 'chrome', 'minVersion': metadata.get('comp at', 'chrome')})
221 226
222 def readSafariMetadata(self): 227 def readSafariMetadata(self):
223 import sitescripts.extensions.bin.legacy.packagerSafari as packager 228 import sitescripts.extensions.bin.legacy.packagerSafari as packager
224 from buildtools import xarfile 229 from sitescripts.extensions.bin.legacy import xarfile
225 metadata = packager.readMetadata(self.tempdir, self.config.type) 230 metadata = packager.readMetadata(self.tempdir, self.config.type)
226 certs = xarfile.read_certificates_and_key(self.config.keyFile)[0] 231 certs = xarfile.read_certificates_and_key(self.config.keyFile)[0]
227 232
228 self.certificateID = packager.get_developer_identifier(certs) 233 self.certificateID = packager.get_developer_identifier(certs)
229 self.version = packager.getBuildVersion(self.tempdir, metadata, False, 234 self.version = packager.getBuildVersion(self.tempdir, metadata, False,
230 self.buildNum) 235 self.buildNum)
231 self.shortVersion = metadata.get('general', 'version') 236 self.shortVersion = metadata.get('general', 'version')
232 self.basename = metadata.get('general', 'basename') 237 self.basename = metadata.get('general', 'basename')
233 self.updatedFromGallery = False 238 self.updatedFromGallery = False
234 239
(...skipping 16 matching lines...) Expand all
251 """ 256 """
252 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 257 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
253 if self.config.type == 'safari': 258 if self.config.type == 'safari':
254 manifestPath = os.path.join(baseDir, 'updates.plist') 259 manifestPath = os.path.join(baseDir, 'updates.plist')
255 templateName = 'safariUpdateManifest' 260 templateName = 'safariUpdateManifest'
256 autoescape = True 261 autoescape = True
257 elif self.config.type == 'android': 262 elif self.config.type == 'android':
258 manifestPath = os.path.join(baseDir, 'updates.xml') 263 manifestPath = os.path.join(baseDir, 'updates.xml')
259 templateName = 'androidUpdateManifest' 264 templateName = 'androidUpdateManifest'
260 autoescape = True 265 autoescape = True
266 elif self.config.type == 'gecko':
267 manifestPath = os.path.join(baseDir, 'updates.json')
268 templateName = 'geckoUpdateManifest'
269 autoescape = True
261 else: 270 else:
262 return 271 return
263 272
264 if not os.path.exists(baseDir): 273 if not os.path.exists(baseDir):
265 os.makedirs(baseDir) 274 os.makedirs(baseDir)
266 275
267 # ABP for Android used to have its own update manifest format. We need t o 276 # ABP for Android used to have its own update manifest format. We need t o
268 # generate both that and the new one in the libadblockplus format as lon g 277 # generate both that and the new one in the libadblockplus format as lon g
269 # as a significant amount of users is on an old version. 278 # as a significant amount of users is on an old version.
270 if self.config.type == 'android': 279 if self.config.type == 'android':
(...skipping 24 matching lines...) Expand all
295 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit e 304 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit e
296 doWrite(manifestPath, [{ 305 doWrite(manifestPath, [{
297 'basename': self.basename, 306 'basename': self.basename,
298 'version': version, 307 'version': version,
299 'updateURL': updateURL 308 'updateURL': updateURL
300 }]) 309 }])
301 310
302 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']: 311 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']:
303 linkPath = os.path.join(baseDir, '00latest%s' % suffix) 312 linkPath = os.path.join(baseDir, '00latest%s' % suffix)
304 outputPath = os.path.join(baseDir, self.basename + '-' + version + s uffix) 313 outputPath = os.path.join(baseDir, self.basename + '-' + version + s uffix)
305 if hasattr(os, 'symlink'): 314 self.symlink_or_copy(outputPath, linkPath)
306 if os.path.exists(linkPath):
307 os.remove(linkPath)
308 os.symlink(os.path.basename(outputPath), linkPath)
309 else:
310 shutil.copyfile(outputPath, linkPath)
311 315
312 def build(self): 316 def build(self):
313 """ 317 """
314 run the build command in the tempdir 318 run the build command in the tempdir
315 """ 319 """
316 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 320 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
317 if not os.path.exists(baseDir): 321 if not os.path.exists(baseDir):
318 os.makedirs(baseDir) 322 os.makedirs(baseDir)
319 outputFile = '%s-%s%s' % (self.basename, self.version, self.config.packa geSuffix) 323 outputFile = '%s-%s%s' % (self.basename, self.version, self.config.packa geSuffix)
320 self.path = os.path.join(baseDir, outputFile) 324 self.path = os.path.join(baseDir, outputFile)
(...skipping 16 matching lines...) Expand all
337 except: 341 except:
338 # clear broken output if any 342 # clear broken output if any
339 if os.path.exists(self.path): 343 if os.path.exists(self.path):
340 os.remove(self.path) 344 os.remove(self.path)
341 raise 345 raise
342 else: 346 else:
343 env = os.environ 347 env = os.environ
344 spiderMonkeyBinary = self.config.spiderMonkeyBinary 348 spiderMonkeyBinary = self.config.spiderMonkeyBinary
345 if spiderMonkeyBinary: 349 if spiderMonkeyBinary:
346 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary) 350 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
351 if self.config.type == 'safari':
tlucas 2018/03/07 22:14:19 Fully adapting to #6021 here (since the 'safari'-t
352 cmd_order = ['-t', self.config.type, 'build']
353 else:
354 cmd_order = ['build', '-t', self.config.type]
355 command = [os.path.join(self.tempdir, 'build.py')] + cmd_order
356 command += ['-b', self.buildNum]
347 357
348 command = [os.path.join(self.tempdir, 'build.py'),
349 'build', '-t', self.config.type, '-b', self.buildNum]
350 if self.config.type not in {'gecko', 'edge'}: 358 if self.config.type not in {'gecko', 'edge'}:
351 command.extend(['-k', self.config.keyFile]) 359 command.extend(['-k', self.config.keyFile])
352 command.append(self.path) 360 command.append(self.path)
353 subprocess.check_call(command, env=env) 361 subprocess.check_call(command, env=env)
354 362
355 if not os.path.exists(self.path): 363 if not os.path.exists(self.path):
356 raise Exception("Build failed, output file hasn't been created") 364 raise Exception("Build failed, output file hasn't been created")
357 365
358 linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffi x) 366 if self.config.type not in self.downloadable_repos:
359 if hasattr(os, 'symlink'): 367 linkPath = os.path.join(baseDir,
360 if os.path.exists(linkPath): 368 '00latest%s' % self.config.packageSuffix)
361 os.remove(linkPath) 369 self.symlink_or_copy(self.path, linkPath)
362 os.symlink(os.path.basename(self.path), linkPath)
363 else:
364 shutil.copyfile(self.path, linkPath)
365 370
366 def retireBuilds(self): 371 def retireBuilds(self):
367 """ 372 """
368 removes outdated builds, returns the sorted version numbers of remaini ng 373 removes outdated builds, returns the sorted version numbers of remaini ng
369 builds 374 builds
370 """ 375 """
371 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 376 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
372 versions = [] 377 versions = []
373 prefix = self.basename + '-' 378 prefix = self.basename + '-'
374 suffix = self.config.packageSuffix 379 suffix = self.config.packageSuffix
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after
407 'download': packageFile, 412 'download': packageFile,
408 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)), 413 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)),
409 'size': os.path.getsize(os.path.join(baseDir, packageFile)) 414 'size': os.path.getsize(os.path.join(baseDir, packageFile))
410 } 415 }
411 if os.path.exists(os.path.join(baseDir, changelogFile)): 416 if os.path.exists(os.path.join(baseDir, changelogFile)):
412 link['changelog'] = changelogFile 417 link['changelog'] = changelogFile
413 links.append(link) 418 links.append(link)
414 template = get_template(get_config().get('extensions', 'nightlyIndexPage ')) 419 template = get_template(get_config().get('extensions', 'nightlyIndexPage '))
415 template.stream({'config': self.config, 'links': links}).dump(outputPath ) 420 template.stream({'config': self.config, 'links': links}).dump(outputPath )
416 421
417 def uploadToMozillaAddons(self): 422 def open_downloads_loackfile(self):
Sebastian Noack 2018/03/08 02:39:21 Shouldn't this be "read" rather than "open"?
Sebastian Noack 2018/03/08 02:39:21 Typo: loackfile => lockfile (or lock_file)
tlucas 2018/03/08 09:23:04 Yes, you're right - Done.
418 import urllib3 423 path = get_config().get('extensions', 'downloadLockFile')
424 current = {}
Sebastian Noack 2018/03/08 02:39:22 This can go into the except block.
tlucas 2018/03/08 09:23:05 Done.
Sebastian Noack 2018/03/09 00:33:18 You put it in the try-block, not in the except-blo
tlucas 2018/03/09 08:12:57 Done.
425 try:
426 with open(path, 'r') as fp:
427 current = json.load(fp)
428 except IOError:
429 logging.warning('No lockfile found. Creating ' + path)
419 430
431 return current
432
433 def write_download_lockfile(self, values):
434 path = get_config().get('extensions', 'downloadLockFile')
435 with open(path, 'w') as fp:
436 json.dump(values, fp)
437
438 def add_to_downloads_lockfile(self, platform, values):
439 current = self.open_downloads_loackfile()
440
441 current.setdefault(platform, [])
442 current[platform].append(values)
443
444 self.write_download_lockfile(current)
445
446 def remove_from_downloads_lockfile(self, platform, filter_key,
447 filter_value):
448 current = self.open_downloads_loackfile()
449 try:
450 for i, entry in enumerate(current[platform]):
451 if entry[filter_key] == filter_value:
452 del current[platform][i]
453 if len(current[platform]) == 0:
454 current.pop(platform)
Sebastian Noack 2018/03/08 02:39:22 You can use the del statement. We don't need it's
tlucas 2018/03/08 09:23:05 Done.
455 except KeyError:
456 pass
457 self.write_download_lockfile(current)
458
459 def generate_jwt_request(self, issuer, secret, url, method, data=None,
460 add_headers=[]):
420 header = { 461 header = {
421 'alg': 'HS256', # HMAC-SHA256 462 'alg': 'HS256', # HMAC-SHA256
422 'typ': 'JWT', 463 'typ': 'JWT',
423 } 464 }
424 465
425 issued = int(time.time()) 466 issued = int(time.time())
426 payload = { 467 payload = {
427 'iss': get_config().get('extensions', 'amo_key'), 468 'iss': issuer,
428 'jti': random.random(), 469 'jti': random.random(),
429 'iat': issued, 470 'iat': issued,
430 'exp': issued + 60, 471 'exp': issued + 60,
431 } 472 }
432 473
433 input = '{}.{}'.format( 474 hmac_data = '{}.{}'.format(
434 base64.b64encode(json.dumps(header)), 475 base64.b64encode(json.dumps(header)),
435 base64.b64encode(json.dumps(payload)) 476 base64.b64encode(json.dumps(payload))
436 ) 477 )
437 478
438 signature = hmac.new(get_config().get('extensions', 'amo_secret'), 479 signature = hmac.new(secret, msg=hmac_data,
439 msg=input,
440 digestmod=hashlib.sha256).digest() 480 digestmod=hashlib.sha256).digest()
441 token = '{}.{}'.format(input, base64.b64encode(signature)) 481 token = '{}.{}'.format(hmac_data, base64.b64encode(signature))
482
483 request = urllib2.Request(url, data)
484 request.add_header('Authorization', 'JWT ' + token)
485 for header in add_headers:
486 request.add_header(*header)
487 request.get_method = lambda: method
488
489 return request
490
491 def uploadToMozillaAddons(self):
492 import urllib3
493
494 config = get_config()
442 495
443 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/' 496 upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
444 'versions/{}/').format(self.extensionID, self.version) 497 'versions/{}/').format(self.extensionID, self.version)
445 498
446 with open(self.path, 'rb') as file: 499 with open(self.path, 'rb') as file:
447 data, content_type = urllib3.filepost.encode_multipart_formdata({ 500 data, content_type = urllib3.filepost.encode_multipart_formdata({
448 'upload': ( 501 'upload': (
449 os.path.basename(self.path), 502 os.path.basename(self.path),
450 file.read(), 503 file.read(),
451 'application/x-xpinstall' 504 'application/x-xpinstall'
452 ) 505 )
453 }) 506 })
454 507
455 request = urllib2.Request(upload_url, data=data) 508 request = self.generate_jwt_request(
456 request.add_header('Content-Type', content_type) 509 config.get('extensions', 'amo_key'),
457 request.add_header('Authorization', 'JWT ' + token) 510 config.get('extensions', 'amo_secret'),
458 request.get_method = lambda: 'PUT' 511 upload_url,
512 'PUT',
513 data,
514 (('Content-Type', content_type),),
515 )
459 516
460 try: 517 try:
461 urllib2.urlopen(request).close() 518 urllib2.urlopen(request).close()
462 except urllib2.HTTPError as e: 519 except urllib2.HTTPError as e:
463 try: 520 try:
464 logging.error(e.read()) 521 logging.error(e.read())
465 finally: 522 finally:
466 e.close() 523 e.close()
467 raise 524 raise
468 525
526 self.add_to_downloads_lockfile(
527 self.config.type,
528 {
529 'buildtype': 'devbuild',
530 'app_id': self.extensionID,
531 'version': self.version,
532 }
533 )
534 os.remove(self.path)
535
536 def download_from_mozilla_addons(self, buildtype, version, app_id):
537 config = get_config()
538 iss = config.get('extensions', 'amo_key')
539 secret = config.get('extensions', 'amo_secret')
540
541 url = ('https://addons.mozilla.org/api/v3/addons/{}/'
542 'versions/{}/').format(app_id, version)
543
544 request = self.generate_jwt_request(iss, secret, url, 'GET')
545
Sebastian Noack 2018/03/08 02:39:22 Nit: This blank line seems redundant.
tlucas 2018/03/08 09:23:05 Done.
546 response = json.load(urllib2.urlopen(request))
547
548 necessary = ['passed_review', 'reviewed', 'processed', 'valid']
549 if all(response[x] for x in necessary):
550 download_url = response['files'][0]['download_url']
551 checksum = response['files'][0]['hash']
552
553 basename = self.basename
554 filename = '{}-{}.xpi'.format(self.basename, version)
555 file_path = os.path.join(
556 config.get('extensions', 'nightliesDirectory'), basename)
Sebastian Noack 2018/03/08 02:39:22 Nit: When wrapping right after opening parenthesis
tlucas 2018/03/08 09:23:04 Done.
557 file_path = os.path.join(file_path, filename)
558
559 request = self.generate_jwt_request(iss, secret, download_url,
560 'GET')
561 try:
562 response = urllib2.urlopen(request)
563 except urllib2.HTTPError as e:
564 logging.error(e.read())
565
566 # Verify the extension's integrity
567 file_content = response.read()
568 sha256 = hashlib.sha256()
569 sha256.update(file_content)
Sebastian Noack 2018/03/08 02:39:22 You can pass the data to the sha256 constructor. N
tlucas 2018/03/08 09:23:04 Done.
570 returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest())
571
572 if returned_checksum != checksum:
573 logging.error('Checksum could not be verified: {} vs {}'
574 ''.format(checksum, returned_checksum))
575
576 with open(file_path, 'w') as fp:
577 fp.write(file_content)
578
579 self.update_link = os.path.join(
580 config.get('extensions', 'nightliesURL'), self.basename,
Sebastian Noack 2018/03/08 02:39:22 Nit: This is easier to read if you put each argume
tlucas 2018/03/08 09:23:04 Done.
581 filename)
582
583 self.remove_from_downloads_lockfile(self.config.type,
584 'version',
585 version)
586 elif not response['passed_review'] or not response['valid']:
587 # When the review failed for any reason, we want to know about it
588 logging.error(json.dumps(response, indent=4))
589 self.remove_from_downloads_lockfile(self.config.type,
590 'version',
591 version)
592
469 def uploadToChromeWebStore(self): 593 def uploadToChromeWebStore(self):
470 594
471 opener = urllib2.build_opener(HTTPErrorBodyHandler) 595 opener = urllib2.build_opener(HTTPErrorBodyHandler)
472 596
473 # use refresh token to obtain a valid access token 597 # use refresh token to obtain a valid access token
474 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh 598 # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh
475 599
476 response = json.load(opener.open( 600 response = json.load(opener.open(
477 'https://accounts.google.com/o/oauth2/token', 601 'https://accounts.google.com/o/oauth2/token',
478 602
(...skipping 176 matching lines...) Expand 10 before | Expand all | Expand 10 after
655 self.readSafariMetadata() 779 self.readSafariMetadata()
656 elif self.config.type == 'gecko': 780 elif self.config.type == 'gecko':
657 self.readGeckoMetadata() 781 self.readGeckoMetadata()
658 elif self.config.type == 'edge': 782 elif self.config.type == 'edge':
659 self.read_edge_metadata() 783 self.read_edge_metadata()
660 else: 784 else:
661 raise Exception('Unknown build type {}' % self.config.type) 785 raise Exception('Unknown build type {}' % self.config.type)
662 786
663 # create development build 787 # create development build
664 self.build() 788 self.build()
789 if self.config.type not in self.downloadable_repos:
790 # write out changelog
791 self.writeChangelog(self.getChanges())
665 792
666 # write out changelog 793 # write update manifest
667 self.writeChangelog(self.getChanges()) 794 self.writeUpdateManifest()
668
669 # write update manifest
670 self.writeUpdateManifest()
671 795
672 # retire old builds 796 # retire old builds
673 versions = self.retireBuilds() 797 versions = self.retireBuilds()
674 798
675 if self.config.type == 'ie': 799 if self.config.type == 'ie':
676 self.writeIEUpdateManifest(versions) 800 self.writeIEUpdateManifest(versions)
677 801
678 # update index page 802 if self.config.type not in self.downloadable_repos:
679 self.updateIndex(versions) 803 # update index page
804 self.updateIndex(versions)
680 805
681 # update nightlies config 806 # update nightlies config
682 self.config.latestRevision = self.revision 807 self.config.latestRevision = self.revision
683 808
684 if (self.config.type == 'gecko' and 809 if (self.config.type == 'gecko' and
685 self.config.galleryID and 810 self.config.galleryID and
686 get_config().has_option('extensions', 'amo_key')): 811 get_config().has_option('extensions', 'amo_key')):
687 self.uploadToMozillaAddons() 812 self.uploadToMozillaAddons()
688 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken: 813 elif self.config.type == 'chrome' and self.config.clientID and self. config.clientSecret and self.config.refreshToken:
689 self.uploadToChromeWebStore() 814 self.uploadToChromeWebStore()
690 elif self.config.type == 'edge' and self.config.clientID and self.co nfig.clientSecret and self.config.refreshToken and self.config.tenantID: 815 elif self.config.type == 'edge' and self.config.clientID and self.co nfig.clientSecret and self.config.refreshToken and self.config.tenantID:
691 self.upload_to_windows_store() 816 self.upload_to_windows_store()
692 817
693 finally: 818 finally:
694 # clean up 819 # clean up
695 if self.tempdir: 820 if self.tempdir:
696 shutil.rmtree(self.tempdir, ignore_errors=True) 821 shutil.rmtree(self.tempdir, ignore_errors=True)
697 822
823 def download(self):
824 with open(get_config().get('extensions', 'downloadLockFile')) as fp:
825 download_info = json.load(fp)
698 826
699 def main(): 827 downloads = self.downloadable_repos.intersection(download_info.keys())
828
829 if self.config.type in downloads:
830 try:
831 self.copyRepository()
832 self.readGeckoMetadata()
833
834 for data in download_info[self.config.type]:
835 self.version = data['version']
836
837 self.download_from_mozilla_addons(**data)
838
839 # write out changelog
840 self.writeChangelog(self.getChanges())
841
842 # write update manifest
843 self.writeUpdateManifest()
844
845 # retire old builds
846 versions = self.retireBuilds()
847 # update index page
848 self.updateIndex(versions)
849 finally:
850 # clean up
851 if self.tempdir:
852 shutil.rmtree(self.tempdir, ignore_errors=True)
853
854
855 def main(download=False):
700 """ 856 """
701 main function for createNightlies.py 857 main function for createNightlies.py
702 """ 858 """
703 nightlyConfig = ConfigParser.SafeConfigParser() 859 nightlyConfig = ConfigParser.SafeConfigParser()
704 nightlyConfigFile = get_config().get('extensions', 'nightliesData') 860 nightlyConfigFile = get_config().get('extensions', 'nightliesData')
861
705 if os.path.exists(nightlyConfigFile): 862 if os.path.exists(nightlyConfigFile):
706 nightlyConfig.read(nightlyConfigFile) 863 nightlyConfig.read(nightlyConfigFile)
707 864
708 # build all extensions specified in the configuration file 865 # build all extensions specified in the configuration file
709 # and generate changelogs and documentations for each: 866 # and generate changelogs and documentations for each:
710 data = None 867 data = None
711 for repo in Configuration.getRepositoryConfigurations(nightlyConfig): 868 for repo in Configuration.getRepositoryConfigurations(nightlyConfig):
712 build = None 869 build = None
713 try: 870 try:
714 build = NightlyBuild(repo) 871 build = NightlyBuild(repo)
715 if build.hasChanges(): 872 if download:
873 build.download()
874 elif build.hasChanges():
716 build.run() 875 build.run()
717 except Exception as ex: 876 except Exception as ex:
718 logging.error('The build for %s failed:', repo) 877 logging.error('The build for %s failed:', repo)
719 logging.exception(ex) 878 logging.exception(ex)
720 879
721 file = open(nightlyConfigFile, 'wb') 880 file = open(nightlyConfigFile, 'wb')
722 nightlyConfig.write(file) 881 nightlyConfig.write(file)
723 882
724 883
725 if __name__ == '__main__': 884 if __name__ == '__main__':
726 main() 885 parser = argparse.ArgumentParser()
886
Sebastian Noack 2018/03/08 02:39:21 Nit: This blank line seems redundant.
tlucas 2018/03/08 09:23:04 Done.
887 parser.add_argument('--download', action='store_true', default=False)
888 args = parser.parse_args()
889 main(args.download)
OLDNEW

Powered by Google App Engine
This is Rietveld