Issue 6371 - Update buildtools dep. to c830dfa08e2f, use AMO-signing API

sitescripts/extensions/bin/createNightlies.py

Index: sitescripts/extensions/bin/createNightlies.py
diff --git a/sitescripts/extensions/bin/createNightlies.py b/sitescripts/extensions/bin/createNightlies.py
index 9ea72c2ce25be7aee8ca5c1040a6bb33e85bc730..2941c4063d53719799e846786850274e623df0e4 100644
--- a/sitescripts/extensions/bin/createNightlies.py
+++ b/sitescripts/extensions/bin/createNightlies.py
@@ -23,6 +23,7 @@ Nightly builds generation script
 
 """
 
+import argparse
 import ConfigParser
 import base64
 import hashlib
@@ -70,6 +71,8 @@ class NightlyBuild(object):
       generating changelogs and documentation.
     """
 
+    downloadable_repos = {'gecko'}
+
     def __init__(self, config):
         """
           Creates a NightlyBuild instance; we are simply
@@ -105,8 +108,6 @@ class NightlyBuild(object):
         """
         command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
         build = subprocess.check_output(command).strip()
-        if self.config.type == 'gecko':
-            build += 'beta'
         return build
 
     def getChanges(self):
@@ -141,6 +142,14 @@ class NightlyBuild(object):
         if os.path.isfile(depscript):
             subprocess.check_call([sys.executable, depscript, '-q'])
 
+    def symlink_or_copy(self, source, target):
+        if hasattr(os, 'symlink'):
+            if os.path.exists(target):
+                os.remove(target)
+            os.symlink(os.path.basename(source), target)
+        else:
+            shutil.copyfile(source, target)
+
     def writeChangelog(self, changes):
         """
           write the changelog file into the cloned repository
@@ -156,12 +165,7 @@ class NightlyBuild(object):
         template.stream({'changes': changes}).dump(changelogPath, encoding='utf-8')
 
         linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
-        if hasattr(os, 'symlink'):
-            if os.path.exists(linkPath):
-                os.remove(linkPath)
-            os.symlink(os.path.basename(changelogPath), linkPath)
-        else:
-            shutil.copyfile(changelogPath, linkPath)
+        self.symlink_or_copy(changelogPath, linkPath)
 
     def readGeckoMetadata(self):
         """
@@ -171,10 +175,11 @@ class NightlyBuild(object):
         """
         import buildtools.packagerChrome as packager
         metadata = packager.readMetadata(self.tempdir, self.config.type)
-        self.extensionID = metadata.get('general', 'id')
+        self.extensionID = packager.get_app_id(False, metadata)
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.basename = metadata.get('general', 'basename')
+        self.min_version = metadata.get('compat', 'gecko')
 
     def readAndroidMetadata(self):
         """
@@ -221,7 +226,7 @@ class NightlyBuild(object):
 
     def readSafariMetadata(self):
         import sitescripts.extensions.bin.legacy.packagerSafari as packager
-        from buildtools import xarfile
+        from sitescripts.extensions.bin.legacy import xarfile
         metadata = packager.readMetadata(self.tempdir, self.config.type)
         certs = xarfile.read_certificates_and_key(self.config.keyFile)[0]
 
@@ -258,6 +263,10 @@ class NightlyBuild(object):
             manifestPath = os.path.join(baseDir, 'updates.xml')
             templateName = 'androidUpdateManifest'
             autoescape = True
+        elif self.config.type == 'gecko':
+            manifestPath = os.path.join(baseDir, 'updates.json')
+            templateName = 'geckoUpdateManifest'
+            autoescape = True
         else:
             return
 
@@ -302,12 +311,7 @@ class NightlyBuild(object):
         for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']:
             linkPath = os.path.join(baseDir, '00latest%s' % suffix)
             outputPath = os.path.join(baseDir, self.basename + '-' + version + suffix)
-            if hasattr(os, 'symlink'):
-                if os.path.exists(linkPath):
-                    os.remove(linkPath)
-                os.symlink(os.path.basename(outputPath), linkPath)
-            else:
-                shutil.copyfile(outputPath, linkPath)
+            self.symlink_or_copy(outputPath, linkPath)
 
     def build(self):
         """
@@ -344,9 +348,13 @@ class NightlyBuild(object):
             spiderMonkeyBinary = self.config.spiderMonkeyBinary
             if spiderMonkeyBinary:
                 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
+            if self.config.type == 'safari':

[tlucas, 2018/03/07 22:14:19]

Fully adapting to #6021 here (since the 'safari'-target does not yet have it
included)

+                cmd_order = ['-t', self.config.type, 'build']
+            else:
+                cmd_order = ['build', '-t', self.config.type]
+            command = [os.path.join(self.tempdir, 'build.py')] + cmd_order
+            command += ['-b', self.buildNum]
 
-            command = [os.path.join(self.tempdir, 'build.py'),
-                       'build', '-t', self.config.type, '-b', self.buildNum]
             if self.config.type not in {'gecko', 'edge'}:
                 command.extend(['-k', self.config.keyFile])
             command.append(self.path)
@@ -355,13 +363,10 @@ class NightlyBuild(object):
         if not os.path.exists(self.path):
             raise Exception("Build failed, output file hasn't been created")
 
-        linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
-        if hasattr(os, 'symlink'):
-            if os.path.exists(linkPath):
-                os.remove(linkPath)
-            os.symlink(os.path.basename(self.path), linkPath)
-        else:
-            shutil.copyfile(self.path, linkPath)
+        if self.config.type not in self.downloadable_repos:
+            linkPath = os.path.join(baseDir,
+                                    '00latest%s' % self.config.packageSuffix)
+            self.symlink_or_copy(self.path, linkPath)
 
     def retireBuilds(self):
         """
@@ -414,9 +419,45 @@ class NightlyBuild(object):
         template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
         template.stream({'config': self.config, 'links': links}).dump(outputPath)
 
-    def uploadToMozillaAddons(self):
-        import urllib3
+    def open_downloads_loackfile(self):

[Sebastian Noack, 2018/03/08 02:39:21]

Shouldn't this be "read" rather than "open"?

[Sebastian Noack, 2018/03/08 02:39:21]

Typo: loackfile => lockfile (or lock_file)

[tlucas, 2018/03/08 09:23:04]

Yes, you're right - Done.

+        path = get_config().get('extensions', 'downloadLockFile')
+        current = {}

[Sebastian Noack, 2018/03/08 02:39:22]

This can go into the except block.

[tlucas, 2018/03/08 09:23:05]

Done.

[Sebastian Noack, 2018/03/09 00:33:18]

You put it in the try-block, not in the except-block.

[tlucas, 2018/03/09 08:12:57]

Done.

+        try:
+            with open(path, 'r') as fp:
+                current = json.load(fp)
+        except IOError:
+            logging.warning('No lockfile found. Creating ' + path)
+
+        return current
 
+    def write_download_lockfile(self, values):
+        path = get_config().get('extensions', 'downloadLockFile')
+        with open(path, 'w') as fp:
+            json.dump(values, fp)
+
+    def add_to_downloads_lockfile(self, platform, values):
+        current = self.open_downloads_loackfile()
+
+        current.setdefault(platform, [])
+        current[platform].append(values)
+
+        self.write_download_lockfile(current)
+
+    def remove_from_downloads_lockfile(self, platform, filter_key,
+                                       filter_value):
+        current = self.open_downloads_loackfile()
+        try:
+            for i, entry in enumerate(current[platform]):
+                if entry[filter_key] == filter_value:
+                    del current[platform][i]
+                if len(current[platform]) == 0:
+                    current.pop(platform)

[Sebastian Noack, 2018/03/08 02:39:22]

You can use the del statement. We don't need it's value (returned by pop).

[tlucas, 2018/03/08 09:23:05]

Done.

+        except KeyError:
+            pass
+        self.write_download_lockfile(current)
+
+    def generate_jwt_request(self, issuer, secret, url, method, data=None,
+                             add_headers=[]):
         header = {
             'alg': 'HS256',     # HMAC-SHA256
             'typ': 'JWT',
@@ -424,21 +465,33 @@ class NightlyBuild(object):
 
         issued = int(time.time())
         payload = {
-            'iss': get_config().get('extensions', 'amo_key'),
+            'iss': issuer,
             'jti': random.random(),
             'iat': issued,
             'exp': issued + 60,
         }
 
-        input = '{}.{}'.format(
+        hmac_data = '{}.{}'.format(
             base64.b64encode(json.dumps(header)),
             base64.b64encode(json.dumps(payload))
         )
 
-        signature = hmac.new(get_config().get('extensions', 'amo_secret'),
-                             msg=input,
+        signature = hmac.new(secret, msg=hmac_data,
                              digestmod=hashlib.sha256).digest()
-        token = '{}.{}'.format(input, base64.b64encode(signature))
+        token = '{}.{}'.format(hmac_data, base64.b64encode(signature))
+
+        request = urllib2.Request(url, data)
+        request.add_header('Authorization', 'JWT ' + token)
+        for header in add_headers:
+            request.add_header(*header)
+        request.get_method = lambda: method
+
+        return request
+
+    def uploadToMozillaAddons(self):
+        import urllib3
+
+        config = get_config()
 
         upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
                       'versions/{}/').format(self.extensionID, self.version)
@@ -452,10 +505,14 @@ class NightlyBuild(object):
                 )
             })
 
-        request = urllib2.Request(upload_url, data=data)
-        request.add_header('Content-Type', content_type)
-        request.add_header('Authorization', 'JWT ' + token)
-        request.get_method = lambda: 'PUT'
+        request = self.generate_jwt_request(
+            config.get('extensions', 'amo_key'),
+            config.get('extensions', 'amo_secret'),
+            upload_url,
+            'PUT',
+            data,
+            (('Content-Type', content_type),),
+        )
 
         try:
             urllib2.urlopen(request).close()
@@ -466,6 +523,73 @@ class NightlyBuild(object):
                 e.close()
             raise
 
+        self.add_to_downloads_lockfile(
+            self.config.type,
+            {
+                'buildtype': 'devbuild',
+                'app_id': self.extensionID,
+                'version': self.version,
+            }
+        )
+        os.remove(self.path)
+
+    def download_from_mozilla_addons(self, buildtype, version, app_id):
+        config = get_config()
+        iss = config.get('extensions', 'amo_key')
+        secret = config.get('extensions', 'amo_secret')
+
+        url = ('https://addons.mozilla.org/api/v3/addons/{}/'
+               'versions/{}/').format(app_id, version)
+
+        request = self.generate_jwt_request(iss, secret, url, 'GET')
+

[Sebastian Noack, 2018/03/08 02:39:22]

Nit: This blank line seems redundant.

[tlucas, 2018/03/08 09:23:05]

Done.

+        response = json.load(urllib2.urlopen(request))
+
+        necessary = ['passed_review', 'reviewed', 'processed', 'valid']
+        if all(response[x] for x in necessary):
+            download_url = response['files'][0]['download_url']
+            checksum = response['files'][0]['hash']
+
+            basename = self.basename
+            filename = '{}-{}.xpi'.format(self.basename, version)
+            file_path = os.path.join(
+                config.get('extensions', 'nightliesDirectory'), basename)

[Sebastian Noack, 2018/03/08 02:39:22]

Nit: When wrapping right after opening parenthesis, please put closing
parenthesis in the same column where the line with the opening parentheses
start. This is easier to read.

  file_path = os.path.join(
      config.get('extensions', 'nightliesDirectory'), basename
  )

Same below.

[tlucas, 2018/03/08 09:23:04]

Done.

+            file_path = os.path.join(file_path, filename)
+
+            request = self.generate_jwt_request(iss, secret, download_url,
+                                                'GET')
+            try:
+                response = urllib2.urlopen(request)
+            except urllib2.HTTPError as e:
+                logging.error(e.read())
+
+            # Verify the extension's integrity
+            file_content = response.read()
+            sha256 = hashlib.sha256()
+            sha256.update(file_content)

[Sebastian Noack, 2018/03/08 02:39:22]

You can pass the data to the sha256 constructor. No need for a separate call to
update().

[tlucas, 2018/03/08 09:23:04]

Done.

+            returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest())
+
+            if returned_checksum != checksum:
+                logging.error('Checksum could not be verified: {} vs {}'
+                              ''.format(checksum, returned_checksum))
+
+            with open(file_path, 'w') as fp:
+                fp.write(file_content)
+
+            self.update_link = os.path.join(
+                config.get('extensions', 'nightliesURL'), self.basename,

[Sebastian Noack, 2018/03/08 02:39:22]

Nit: This is easier to read if you put each argument on a separate line.

[tlucas, 2018/03/08 09:23:04]

Done.

+                filename)
+
+            self.remove_from_downloads_lockfile(self.config.type,
+                                                'version',
+                                                version)
+        elif not response['passed_review'] or not response['valid']:
+            # When the review failed for any reason, we want to know about it
+            logging.error(json.dumps(response, indent=4))
+            self.remove_from_downloads_lockfile(self.config.type,
+                                                'version',
+                                                version)
+
     def uploadToChromeWebStore(self):
 
         opener = urllib2.build_opener(HTTPErrorBodyHandler)
@@ -662,12 +786,12 @@ class NightlyBuild(object):
 
                 # create development build
                 self.build()
+                if self.config.type not in self.downloadable_repos:
+                    # write out changelog
+                    self.writeChangelog(self.getChanges())
 
-                # write out changelog
-                self.writeChangelog(self.getChanges())
-
-                # write update manifest
-                self.writeUpdateManifest()
+                    # write update manifest
+                    self.writeUpdateManifest()
 
             # retire old builds
             versions = self.retireBuilds()
@@ -675,8 +799,9 @@ class NightlyBuild(object):
             if self.config.type == 'ie':
                 self.writeIEUpdateManifest(versions)
 
-            # update index page
-            self.updateIndex(versions)
+            if self.config.type not in self.downloadable_repos:
+                # update index page
+                self.updateIndex(versions)
 
             # update nightlies config
             self.config.latestRevision = self.revision
@@ -695,13 +820,45 @@ class NightlyBuild(object):
             if self.tempdir:
                 shutil.rmtree(self.tempdir, ignore_errors=True)
 
+    def download(self):
+        with open(get_config().get('extensions', 'downloadLockFile')) as fp:
+            download_info = json.load(fp)
+
+        downloads = self.downloadable_repos.intersection(download_info.keys())
+
+        if self.config.type in downloads:
+            try:
+                self.copyRepository()
+                self.readGeckoMetadata()
+
+                for data in download_info[self.config.type]:
+                    self.version = data['version']
+
+                    self.download_from_mozilla_addons(**data)
 
-def main():
+                    # write out changelog
+                    self.writeChangelog(self.getChanges())
+
+                    # write update manifest
+                    self.writeUpdateManifest()
+
+                    # retire old builds
+                    versions = self.retireBuilds()
+                    # update index page
+                    self.updateIndex(versions)
+            finally:
+                # clean up
+                if self.tempdir:
+                    shutil.rmtree(self.tempdir, ignore_errors=True)
+
+
+def main(download=False):
     """
       main function for createNightlies.py
     """
     nightlyConfig = ConfigParser.SafeConfigParser()
     nightlyConfigFile = get_config().get('extensions', 'nightliesData')
+
     if os.path.exists(nightlyConfigFile):
         nightlyConfig.read(nightlyConfigFile)
 
@@ -712,7 +869,9 @@ def main():
         build = None
         try:
             build = NightlyBuild(repo)
-            if build.hasChanges():
+            if download:
+                build.download()
+            elif build.hasChanges():
                 build.run()
         except Exception as ex:
             logging.error('The build for %s failed:', repo)
@@ -723,4 +882,8 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    parser = argparse.ArgumentParser()
+

[Sebastian Noack, 2018/03/08 02:39:21]

Nit: This blank line seems redundant.

[tlucas, 2018/03/08 09:23:04]

Done.

+    parser.add_argument('--download', action='store_true', default=False)
+    args = parser.parse_args()
+    main(args.download)