Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Unified Diff: sitescripts/extensions/bin/createNightlies.py

Issue 29716693: Issue 6371 - Update buildtools dep. to c830dfa08e2f, use AMO-signing API (Closed) Base URL: https://hg.adblockplus.org/abpssembly/file/1e38c3375fa3
Patch Set: Created March 7, 2018, 10:05 p.m.
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: sitescripts/extensions/bin/createNightlies.py
diff --git a/sitescripts/extensions/bin/createNightlies.py b/sitescripts/extensions/bin/createNightlies.py
index 9ea72c2ce25be7aee8ca5c1040a6bb33e85bc730..2941c4063d53719799e846786850274e623df0e4 100644
--- a/sitescripts/extensions/bin/createNightlies.py
+++ b/sitescripts/extensions/bin/createNightlies.py
@@ -23,6 +23,7 @@ Nightly builds generation script
"""
+import argparse
import ConfigParser
import base64
import hashlib
@@ -70,6 +71,8 @@ class NightlyBuild(object):
generating changelogs and documentation.
"""
+ downloadable_repos = {'gecko'}
+
def __init__(self, config):
"""
Creates a NightlyBuild instance; we are simply
@@ -105,8 +108,6 @@ class NightlyBuild(object):
"""
command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
build = subprocess.check_output(command).strip()
- if self.config.type == 'gecko':
- build += 'beta'
return build
def getChanges(self):
@@ -141,6 +142,14 @@ class NightlyBuild(object):
if os.path.isfile(depscript):
subprocess.check_call([sys.executable, depscript, '-q'])
+ def symlink_or_copy(self, source, target):
+ if hasattr(os, 'symlink'):
+ if os.path.exists(target):
+ os.remove(target)
+ os.symlink(os.path.basename(source), target)
+ else:
+ shutil.copyfile(source, target)
+
def writeChangelog(self, changes):
"""
write the changelog file into the cloned repository
@@ -156,12 +165,7 @@ class NightlyBuild(object):
template.stream({'changes': changes}).dump(changelogPath, encoding='utf-8')
linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
- if hasattr(os, 'symlink'):
- if os.path.exists(linkPath):
- os.remove(linkPath)
- os.symlink(os.path.basename(changelogPath), linkPath)
- else:
- shutil.copyfile(changelogPath, linkPath)
+ self.symlink_or_copy(changelogPath, linkPath)
def readGeckoMetadata(self):
"""
@@ -171,10 +175,11 @@ class NightlyBuild(object):
"""
import buildtools.packagerChrome as packager
metadata = packager.readMetadata(self.tempdir, self.config.type)
- self.extensionID = metadata.get('general', 'id')
+ self.extensionID = packager.get_app_id(False, metadata)
self.version = packager.getBuildVersion(self.tempdir, metadata, False,
self.buildNum)
self.basename = metadata.get('general', 'basename')
+ self.min_version = metadata.get('compat', 'gecko')
def readAndroidMetadata(self):
"""
@@ -221,7 +226,7 @@ class NightlyBuild(object):
def readSafariMetadata(self):
import sitescripts.extensions.bin.legacy.packagerSafari as packager
- from buildtools import xarfile
+ from sitescripts.extensions.bin.legacy import xarfile
metadata = packager.readMetadata(self.tempdir, self.config.type)
certs = xarfile.read_certificates_and_key(self.config.keyFile)[0]
@@ -258,6 +263,10 @@ class NightlyBuild(object):
manifestPath = os.path.join(baseDir, 'updates.xml')
templateName = 'androidUpdateManifest'
autoescape = True
+ elif self.config.type == 'gecko':
+ manifestPath = os.path.join(baseDir, 'updates.json')
+ templateName = 'geckoUpdateManifest'
+ autoescape = True
else:
return
@@ -302,12 +311,7 @@ class NightlyBuild(object):
for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']:
linkPath = os.path.join(baseDir, '00latest%s' % suffix)
outputPath = os.path.join(baseDir, self.basename + '-' + version + suffix)
- if hasattr(os, 'symlink'):
- if os.path.exists(linkPath):
- os.remove(linkPath)
- os.symlink(os.path.basename(outputPath), linkPath)
- else:
- shutil.copyfile(outputPath, linkPath)
+ self.symlink_or_copy(outputPath, linkPath)
def build(self):
"""
@@ -344,9 +348,13 @@ class NightlyBuild(object):
spiderMonkeyBinary = self.config.spiderMonkeyBinary
if spiderMonkeyBinary:
env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
+ if self.config.type == 'safari':
tlucas 2018/03/07 22:14:19 Fully adapting to #6021 here (since the 'safari'-t
+ cmd_order = ['-t', self.config.type, 'build']
+ else:
+ cmd_order = ['build', '-t', self.config.type]
+ command = [os.path.join(self.tempdir, 'build.py')] + cmd_order
+ command += ['-b', self.buildNum]
- command = [os.path.join(self.tempdir, 'build.py'),
- 'build', '-t', self.config.type, '-b', self.buildNum]
if self.config.type not in {'gecko', 'edge'}:
command.extend(['-k', self.config.keyFile])
command.append(self.path)
@@ -355,13 +363,10 @@ class NightlyBuild(object):
if not os.path.exists(self.path):
raise Exception("Build failed, output file hasn't been created")
- linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
- if hasattr(os, 'symlink'):
- if os.path.exists(linkPath):
- os.remove(linkPath)
- os.symlink(os.path.basename(self.path), linkPath)
- else:
- shutil.copyfile(self.path, linkPath)
+ if self.config.type not in self.downloadable_repos:
+ linkPath = os.path.join(baseDir,
+ '00latest%s' % self.config.packageSuffix)
+ self.symlink_or_copy(self.path, linkPath)
def retireBuilds(self):
"""
@@ -414,9 +419,45 @@ class NightlyBuild(object):
template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
template.stream({'config': self.config, 'links': links}).dump(outputPath)
- def uploadToMozillaAddons(self):
- import urllib3
+ def open_downloads_loackfile(self):
Sebastian Noack 2018/03/08 02:39:21 Shouldn't this be "read" rather than "open"?
Sebastian Noack 2018/03/08 02:39:21 Typo: loackfile => lockfile (or lock_file)
tlucas 2018/03/08 09:23:04 Yes, you're right - Done.
+ path = get_config().get('extensions', 'downloadLockFile')
+ current = {}
Sebastian Noack 2018/03/08 02:39:22 This can go into the except block.
tlucas 2018/03/08 09:23:05 Done.
Sebastian Noack 2018/03/09 00:33:18 You put it in the try-block, not in the except-blo
tlucas 2018/03/09 08:12:57 Done.
+ try:
+ with open(path, 'r') as fp:
+ current = json.load(fp)
+ except IOError:
+ logging.warning('No lockfile found. Creating ' + path)
+
+ return current
+ def write_download_lockfile(self, values):
+ path = get_config().get('extensions', 'downloadLockFile')
+ with open(path, 'w') as fp:
+ json.dump(values, fp)
+
+ def add_to_downloads_lockfile(self, platform, values):
+ current = self.open_downloads_loackfile()
+
+ current.setdefault(platform, [])
+ current[platform].append(values)
+
+ self.write_download_lockfile(current)
+
+ def remove_from_downloads_lockfile(self, platform, filter_key,
+ filter_value):
+ current = self.open_downloads_loackfile()
+ try:
+ for i, entry in enumerate(current[platform]):
+ if entry[filter_key] == filter_value:
+ del current[platform][i]
+ if len(current[platform]) == 0:
+ current.pop(platform)
Sebastian Noack 2018/03/08 02:39:22 You can use the del statement. We don't need it's
tlucas 2018/03/08 09:23:05 Done.
+ except KeyError:
+ pass
+ self.write_download_lockfile(current)
+
+ def generate_jwt_request(self, issuer, secret, url, method, data=None,
+ add_headers=[]):
header = {
'alg': 'HS256', # HMAC-SHA256
'typ': 'JWT',
@@ -424,21 +465,33 @@ class NightlyBuild(object):
issued = int(time.time())
payload = {
- 'iss': get_config().get('extensions', 'amo_key'),
+ 'iss': issuer,
'jti': random.random(),
'iat': issued,
'exp': issued + 60,
}
- input = '{}.{}'.format(
+ hmac_data = '{}.{}'.format(
base64.b64encode(json.dumps(header)),
base64.b64encode(json.dumps(payload))
)
- signature = hmac.new(get_config().get('extensions', 'amo_secret'),
- msg=input,
+ signature = hmac.new(secret, msg=hmac_data,
digestmod=hashlib.sha256).digest()
- token = '{}.{}'.format(input, base64.b64encode(signature))
+ token = '{}.{}'.format(hmac_data, base64.b64encode(signature))
+
+ request = urllib2.Request(url, data)
+ request.add_header('Authorization', 'JWT ' + token)
+ for header in add_headers:
+ request.add_header(*header)
+ request.get_method = lambda: method
+
+ return request
+
+ def uploadToMozillaAddons(self):
+ import urllib3
+
+ config = get_config()
upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
'versions/{}/').format(self.extensionID, self.version)
@@ -452,10 +505,14 @@ class NightlyBuild(object):
)
})
- request = urllib2.Request(upload_url, data=data)
- request.add_header('Content-Type', content_type)
- request.add_header('Authorization', 'JWT ' + token)
- request.get_method = lambda: 'PUT'
+ request = self.generate_jwt_request(
+ config.get('extensions', 'amo_key'),
+ config.get('extensions', 'amo_secret'),
+ upload_url,
+ 'PUT',
+ data,
+ (('Content-Type', content_type),),
+ )
try:
urllib2.urlopen(request).close()
@@ -466,6 +523,73 @@ class NightlyBuild(object):
e.close()
raise
+ self.add_to_downloads_lockfile(
+ self.config.type,
+ {
+ 'buildtype': 'devbuild',
+ 'app_id': self.extensionID,
+ 'version': self.version,
+ }
+ )
+ os.remove(self.path)
+
+ def download_from_mozilla_addons(self, buildtype, version, app_id):
+ config = get_config()
+ iss = config.get('extensions', 'amo_key')
+ secret = config.get('extensions', 'amo_secret')
+
+ url = ('https://addons.mozilla.org/api/v3/addons/{}/'
+ 'versions/{}/').format(app_id, version)
+
+ request = self.generate_jwt_request(iss, secret, url, 'GET')
+
Sebastian Noack 2018/03/08 02:39:22 Nit: This blank line seems redundant.
tlucas 2018/03/08 09:23:05 Done.
+ response = json.load(urllib2.urlopen(request))
+
+ necessary = ['passed_review', 'reviewed', 'processed', 'valid']
+ if all(response[x] for x in necessary):
+ download_url = response['files'][0]['download_url']
+ checksum = response['files'][0]['hash']
+
+ basename = self.basename
+ filename = '{}-{}.xpi'.format(self.basename, version)
+ file_path = os.path.join(
+ config.get('extensions', 'nightliesDirectory'), basename)
Sebastian Noack 2018/03/08 02:39:22 Nit: When wrapping right after opening parenthesis
tlucas 2018/03/08 09:23:04 Done.
+ file_path = os.path.join(file_path, filename)
+
+ request = self.generate_jwt_request(iss, secret, download_url,
+ 'GET')
+ try:
+ response = urllib2.urlopen(request)
+ except urllib2.HTTPError as e:
+ logging.error(e.read())
+
+ # Verify the extension's integrity
+ file_content = response.read()
+ sha256 = hashlib.sha256()
+ sha256.update(file_content)
Sebastian Noack 2018/03/08 02:39:22 You can pass the data to the sha256 constructor. N
tlucas 2018/03/08 09:23:04 Done.
+ returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest())
+
+ if returned_checksum != checksum:
+ logging.error('Checksum could not be verified: {} vs {}'
+ ''.format(checksum, returned_checksum))
+
+ with open(file_path, 'w') as fp:
+ fp.write(file_content)
+
+ self.update_link = os.path.join(
+ config.get('extensions', 'nightliesURL'), self.basename,
Sebastian Noack 2018/03/08 02:39:22 Nit: This is easier to read if you put each argume
tlucas 2018/03/08 09:23:04 Done.
+ filename)
+
+ self.remove_from_downloads_lockfile(self.config.type,
+ 'version',
+ version)
+ elif not response['passed_review'] or not response['valid']:
+ # When the review failed for any reason, we want to know about it
+ logging.error(json.dumps(response, indent=4))
+ self.remove_from_downloads_lockfile(self.config.type,
+ 'version',
+ version)
+
def uploadToChromeWebStore(self):
opener = urllib2.build_opener(HTTPErrorBodyHandler)
@@ -662,12 +786,12 @@ class NightlyBuild(object):
# create development build
self.build()
+ if self.config.type not in self.downloadable_repos:
+ # write out changelog
+ self.writeChangelog(self.getChanges())
- # write out changelog
- self.writeChangelog(self.getChanges())
-
- # write update manifest
- self.writeUpdateManifest()
+ # write update manifest
+ self.writeUpdateManifest()
# retire old builds
versions = self.retireBuilds()
@@ -675,8 +799,9 @@ class NightlyBuild(object):
if self.config.type == 'ie':
self.writeIEUpdateManifest(versions)
- # update index page
- self.updateIndex(versions)
+ if self.config.type not in self.downloadable_repos:
+ # update index page
+ self.updateIndex(versions)
# update nightlies config
self.config.latestRevision = self.revision
@@ -695,13 +820,45 @@ class NightlyBuild(object):
if self.tempdir:
shutil.rmtree(self.tempdir, ignore_errors=True)
+ def download(self):
+ with open(get_config().get('extensions', 'downloadLockFile')) as fp:
+ download_info = json.load(fp)
+
+ downloads = self.downloadable_repos.intersection(download_info.keys())
+
+ if self.config.type in downloads:
+ try:
+ self.copyRepository()
+ self.readGeckoMetadata()
+
+ for data in download_info[self.config.type]:
+ self.version = data['version']
+
+ self.download_from_mozilla_addons(**data)
-def main():
+ # write out changelog
+ self.writeChangelog(self.getChanges())
+
+ # write update manifest
+ self.writeUpdateManifest()
+
+ # retire old builds
+ versions = self.retireBuilds()
+ # update index page
+ self.updateIndex(versions)
+ finally:
+ # clean up
+ if self.tempdir:
+ shutil.rmtree(self.tempdir, ignore_errors=True)
+
+
+def main(download=False):
"""
main function for createNightlies.py
"""
nightlyConfig = ConfigParser.SafeConfigParser()
nightlyConfigFile = get_config().get('extensions', 'nightliesData')
+
if os.path.exists(nightlyConfigFile):
nightlyConfig.read(nightlyConfigFile)
@@ -712,7 +869,9 @@ def main():
build = None
try:
build = NightlyBuild(repo)
- if build.hasChanges():
+ if download:
+ build.download()
+ elif build.hasChanges():
build.run()
except Exception as ex:
logging.error('The build for %s failed:', repo)
@@ -723,4 +882,8 @@ def main():
if __name__ == '__main__':
- main()
+ parser = argparse.ArgumentParser()
+
Sebastian Noack 2018/03/08 02:39:21 Nit: This blank line seems redundant.
tlucas 2018/03/08 09:23:04 Done.
+ parser.add_argument('--download', action='store_true', default=False)
+ args = parser.parse_args()
+ main(args.download)

Powered by Google App Engine
This is Rietveld