Issue 6371 - Update buildtools dep. to c830dfa08e2f, use AMO-signing API

sitescripts/extensions/bin/createNightlies.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 
 Nightly builds generation script
 ================================
 
   This script generates nightly builds of extensions, together
   with changelogs and documentation.
 
 """
 
+import argparse
 import ConfigParser
 import base64
 import hashlib
 import hmac
 import json
 import logging
 import os
 import pipes
 import random
 import shutil
@@ skipping 27 matching lines @@
         raise urllib2.HTTPError(req.get_full_url(), code,
                                 '{}\n{}'.format(msg, fp.read()), hdrs, fp)
 
 
 class NightlyBuild(object):
     """
       Performs the build process for an extension,
       generating changelogs and documentation.
     """
 
+    downloadable_repos = {'gecko'}
+
     def __init__(self, config):
         """
           Creates a NightlyBuild instance; we are simply
           recording the configuration settings here.
         """
         self.config = config
         self.revision = self.getCurrentRevision()
         try:
             self.previousRevision = config.latestRevision
         except:
@@ skipping 15 matching lines @@
             'defaults.id=', self.config.repository
         ]
         return subprocess.check_output(command).strip()
 
     def getCurrentBuild(self):
         """
             calculates the (typically numerical) build ID for the current build
         """
         command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
         build = subprocess.check_output(command).strip()
-        if self.config.type == 'gecko':
-            build += 'beta'
         return build
 
     def getChanges(self):
         """
           retrieve changes between the current and previous ("first") revision
         """
         command = [
             'hg', 'log', '-R', self.tempdir, '-r',
             'reverse(ancestors({}))'.format(self.config.revision), '-l', '50',
             '--encoding', 'utf-8', '--template',
@@ skipping 14 matching lines @@
         self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName)
         command = ['hg', 'clone', '-q', self.config.repository, '-u',
                    self.config.revision, self.tempdir]
         subprocess.check_call(command)
 
         # Make sure to run ensure_dependencies.py if present
         depscript = os.path.join(self.tempdir, 'ensure_dependencies.py')
         if os.path.isfile(depscript):
             subprocess.check_call([sys.executable, depscript, '-q'])
 
+    def symlink_or_copy(self, source, target):
+        if hasattr(os, 'symlink'):
+            if os.path.exists(target):
+                os.remove(target)
+            os.symlink(os.path.basename(source), target)
+        else:
+            shutil.copyfile(source, target)
+
     def writeChangelog(self, changes):
         """
           write the changelog file into the cloned repository
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if not os.path.exists(baseDir):
             os.makedirs(baseDir)
         changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version)
         changelogPath = os.path.join(baseDir, changelogFile)
         self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + changelogFile)
 
         template = get_template(get_config().get('extensions', 'changelogTemplate'))
         template.stream({'changes': changes}).dump(changelogPath, encoding='utf-8')
 
         linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
-        if hasattr(os, 'symlink'):
-            if os.path.exists(linkPath):
-                os.remove(linkPath)
-            os.symlink(os.path.basename(changelogPath), linkPath)
-        else:
-            shutil.copyfile(changelogPath, linkPath)
+        self.symlink_or_copy(changelogPath, linkPath)
 
     def readGeckoMetadata(self):
         """
           read Gecko-specific metadata file from a cloned repository
           and parse id, version, basename and the compat section
           out of the file
         """
         import buildtools.packagerChrome as packager
         metadata = packager.readMetadata(self.tempdir, self.config.type)
-        self.extensionID = metadata.get('general', 'id')
+        self.extensionID = packager.get_app_id(False, metadata)
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.basename = metadata.get('general', 'basename')
+        self.min_version = metadata.get('compat', 'gecko')
 
     def readAndroidMetadata(self):
         """
           Read Android-specific metadata from AndroidManifest.xml file.
         """
         manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), 'r')
         manifest = parseXml(manifestFile)
         manifestFile.close()
 
         root = manifest.documentElement
@@ skipping 26 matching lines @@
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.basename = metadata.get('general', 'basename')
 
         self.compat = []
         if metadata.has_section('compat') and metadata.has_option('compat', 'chrome'):
             self.compat.append({'id': 'chrome', 'minVersion': metadata.get('compat', 'chrome')})
 
     def readSafariMetadata(self):
         import sitescripts.extensions.bin.legacy.packagerSafari as packager
-        from buildtools import xarfile
+        from sitescripts.extensions.bin.legacy import xarfile
         metadata = packager.readMetadata(self.tempdir, self.config.type)
         certs = xarfile.read_certificates_and_key(self.config.keyFile)[0]
 
         self.certificateID = packager.get_developer_identifier(certs)
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.shortVersion = metadata.get('general', 'version')
         self.basename = metadata.get('general', 'basename')
         self.updatedFromGallery = False
 
@@ skipping 16 matching lines @@
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if self.config.type == 'safari':
             manifestPath = os.path.join(baseDir, 'updates.plist')
             templateName = 'safariUpdateManifest'
             autoescape = True
         elif self.config.type == 'android':
             manifestPath = os.path.join(baseDir, 'updates.xml')
             templateName = 'androidUpdateManifest'
             autoescape = True
+        elif self.config.type == 'gecko':
+            manifestPath = os.path.join(baseDir, 'updates.json')
+            templateName = 'geckoUpdateManifest'
+            autoescape = True
         else:
             return
 
         if not os.path.exists(baseDir):
             os.makedirs(baseDir)
 
         # ABP for Android used to have its own update manifest format. We need to
         # generate both that and the new one in the libadblockplus format as long
         # as a significant amount of users is on an old version.
         if self.config.type == 'android':
@@ skipping 24 matching lines @@
         from sitescripts.extensions.utils import writeIEUpdateManifest as doWrite
         doWrite(manifestPath, [{
             'basename': self.basename,
             'version': version,
             'updateURL': updateURL
         }])
 
         for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']:
             linkPath = os.path.join(baseDir, '00latest%s' % suffix)
             outputPath = os.path.join(baseDir, self.basename + '-' + version + suffix)
-            if hasattr(os, 'symlink'):
-                if os.path.exists(linkPath):
-                    os.remove(linkPath)
-                os.symlink(os.path.basename(outputPath), linkPath)
-            else:
-                shutil.copyfile(outputPath, linkPath)
+            self.symlink_or_copy(outputPath, linkPath)
 
     def build(self):
         """
           run the build command in the tempdir
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if not os.path.exists(baseDir):
             os.makedirs(baseDir)
         outputFile = '%s-%s%s' % (self.basename, self.version, self.config.packageSuffix)
         self.path = os.path.join(baseDir, outputFile)
@@ skipping 16 matching lines @@
             except:
                 # clear broken output if any
                 if os.path.exists(self.path):
                     os.remove(self.path)
                 raise
         else:
             env = os.environ
             spiderMonkeyBinary = self.config.spiderMonkeyBinary
             if spiderMonkeyBinary:
                 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
+            if self.config.type == 'safari':
+                cmd_order = ['-t', self.config.type, 'build']
+            else:
+                cmd_order = ['build', '-t', self.config.type]
+            command = [os.path.join(self.tempdir, 'build.py')] + cmd_order
+            command += ['-b', self.buildNum]

[Sebastian Noack, 2018/03/09 00:33:18]

You could use list.extend() in order to modify the list in place:

  command = [os.path.join(self.tempdir, 'build.py')]
  if self.config.type == 'safari':
      command.extend(['-t', self.config.type, 'build'])
  else:
      command.extend(['build', '-t', self.config.type])
  command.extend(['-b', self.buildNum])

[tlucas, 2018/03/09 08:12:57]

Done.

 
-            command = [os.path.join(self.tempdir, 'build.py'),
-                       'build', '-t', self.config.type, '-b', self.buildNum]
             if self.config.type not in {'gecko', 'edge'}:
                 command.extend(['-k', self.config.keyFile])
             command.append(self.path)
             subprocess.check_call(command, env=env)
 
         if not os.path.exists(self.path):
             raise Exception("Build failed, output file hasn't been created")
 
-        linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
-        if hasattr(os, 'symlink'):
-            if os.path.exists(linkPath):
-                os.remove(linkPath)
-            os.symlink(os.path.basename(self.path), linkPath)
-        else:
-            shutil.copyfile(self.path, linkPath)
+        if self.config.type not in self.downloadable_repos:
+            linkPath = os.path.join(baseDir,
+                                    '00latest%s' % self.config.packageSuffix)

[Sebastian Noack, 2018/03/09 00:33:19]

Please use .format() instead of %. flake8-eyeo would normally warn about this,
but the error is likely ignored for this file.

[tlucas, 2018/03/09 08:12:58]

This is flawed, yes - but .format() should be used for more complex situations.
I'll simply use the + operator.

[Sebastian Noack, 2018/03/09 16:16:27]

You are right, since we just append in the end, + should be used.

+            self.symlink_or_copy(self.path, linkPath)
 
     def retireBuilds(self):
         """
           removes outdated builds, returns the sorted version numbers of remaining
           builds
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         versions = []
         prefix = self.basename + '-'
         suffix = self.config.packageSuffix
@@ skipping 32 matching lines @@
                 'download': packageFile,
                 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)),
                 'size': os.path.getsize(os.path.join(baseDir, packageFile))
             }
             if os.path.exists(os.path.join(baseDir, changelogFile)):
                 link['changelog'] = changelogFile
             links.append(link)
         template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
         template.stream({'config': self.config, 'links': links}).dump(outputPath)
 
-    def uploadToMozillaAddons(self):
-        import urllib3
+    def read_downloads_lockfile(self):
+        path = get_config().get('extensions', 'downloadLockFile')
+        try:
+            current = {}
+            with open(path, 'r') as fp:
+                current = json.load(fp)
+        except IOError:
+            logging.warning('No lockfile found. Creating ' + path)
 
+        return current
+
+    def write_downloads_lockfile(self, values):
+        path = get_config().get('extensions', 'downloadLockFile')
+        with open(path, 'w') as fp:
+            json.dump(values, fp)
+
+    def add_to_downloads_lockfile(self, platform, values):
+        current = self.read_downloads_lockfile()
+
+        current.setdefault(platform, [])
+        current[platform].append(values)
+
+        self.write_downloads_lockfile(current)
+
+    def remove_from_downloads_lockfile(self, platform, filter_key,
+                                       filter_value):
+        current = self.read_downloads_lockfile()
+        try:
+            for i, entry in enumerate(current[platform]):
+                if entry[filter_key] == filter_value:
+                    del current[platform][i]
+                if len(current[platform]) == 0:
+                    del current[platform]
+        except KeyError:
+            pass
+        self.write_downloads_lockfile(current)
+
+    def generate_jwt_request(self, issuer, secret, url, method, data=None,
+                             add_headers=[]):
         header = {
             'alg': 'HS256',     # HMAC-SHA256
             'typ': 'JWT',
         }
 
         issued = int(time.time())
         payload = {
-            'iss': get_config().get('extensions', 'amo_key'),
+            'iss': issuer,
             'jti': random.random(),
             'iat': issued,
             'exp': issued + 60,
         }
 
-        input = '{}.{}'.format(
+        hmac_data = '{}.{}'.format(
             base64.b64encode(json.dumps(header)),
             base64.b64encode(json.dumps(payload))
         )
 
-        signature = hmac.new(get_config().get('extensions', 'amo_secret'),
-                             msg=input,
+        signature = hmac.new(secret, msg=hmac_data,
                              digestmod=hashlib.sha256).digest()
-        token = '{}.{}'.format(input, base64.b64encode(signature))
+        token = '{}.{}'.format(hmac_data, base64.b64encode(signature))
+
+        request = urllib2.Request(url, data)
+        request.add_header('Authorization', 'JWT ' + token)
+        for header in add_headers:
+            request.add_header(*header)
+        request.get_method = lambda: method
+
+        return request
+
+    def uploadToMozillaAddons(self):
+        import urllib3
+
+        config = get_config()
 
         upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
                       'versions/{}/').format(self.extensionID, self.version)
 
         with open(self.path, 'rb') as file:
             data, content_type = urllib3.filepost.encode_multipart_formdata({
                 'upload': (
                     os.path.basename(self.path),
                     file.read(),
                     'application/x-xpinstall'
                 )
             })
 
-        request = urllib2.Request(upload_url, data=data)
-        request.add_header('Content-Type', content_type)
-        request.add_header('Authorization', 'JWT ' + token)
-        request.get_method = lambda: 'PUT'
+        request = self.generate_jwt_request(
+            config.get('extensions', 'amo_key'),
+            config.get('extensions', 'amo_secret'),
+            upload_url,
+            'PUT',
+            data,
+            (('Content-Type', content_type),),
+        )
 
         try:
             urllib2.urlopen(request).close()
         except urllib2.HTTPError as e:
             try:
                 logging.error(e.read())
             finally:
                 e.close()
             raise
 
+        self.add_to_downloads_lockfile(
+            self.config.type,
+            {
+                'buildtype': 'devbuild',
+                'app_id': self.extensionID,
+                'version': self.version,
+            }
+        )
+        os.remove(self.path)
+
+    def download_from_mozilla_addons(self, buildtype, version, app_id):
+        config = get_config()
+        iss = config.get('extensions', 'amo_key')
+        secret = config.get('extensions', 'amo_secret')
+
+        url = ('https://addons.mozilla.org/api/v3/addons/{}/'
+               'versions/{}/').format(app_id, version)
+
+        request = self.generate_jwt_request(iss, secret, url, 'GET')
+        response = json.load(urllib2.urlopen(request))
+
+        necessary = ['passed_review', 'reviewed', 'processed', 'valid']
+        if all(response[x] for x in necessary):
+            download_url = response['files'][0]['download_url']
+            checksum = response['files'][0]['hash']
+
+            filename = '{}-{}.xpi'.format(self.basename, version)
+            file_path = os.path.join(
+                config.get('extensions', 'nightliesDirectory'),
+                self.basename,
+                filename,

[Sebastian Noack, 2018/03/09 00:33:19]

Nit: Redundant comma after last argument.

[tlucas, 2018/03/09 08:12:57]

Done.

+            )
+
+            request = self.generate_jwt_request(iss, secret, download_url,
+                                                'GET')
+            try:
+                response = urllib2.urlopen(request)
+            except urllib2.HTTPError as e:
+                logging.error(e.read())
+
+            # Verify the extension's integrity
+            file_content = response.read()
+            sha256 = hashlib.sha256(file_content)
+            returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest())
+
+            if returned_checksum != checksum:
+                logging.error('Checksum could not be verified: {} vs {}'
+                              ''.format(checksum, returned_checksum))
+
+            with open(file_path, 'w') as fp:
+                fp.write(file_content)
+
+            self.update_link = os.path.join(
+                config.get('extensions', 'nightliesURL'),
+                self.basename,
+                filename,

[Sebastian Noack, 2018/03/09 00:33:19]

Nit: Redundant comma after last argument.

[tlucas, 2018/03/09 08:12:57]

Done.

+            )
+
+            self.remove_from_downloads_lockfile(self.config.type,
+                                                'version',
+                                                version)
+        elif not response['passed_review'] or not response['valid']:
+            # When the review failed for any reason, we want to know about it
+            logging.error(json.dumps(response, indent=4))
+            self.remove_from_downloads_lockfile(self.config.type,
+                                                'version',
+                                                version)
+
     def uploadToChromeWebStore(self):
 
         opener = urllib2.build_opener(HTTPErrorBodyHandler)
 
         # use refresh token to obtain a valid access token
         # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh
 
         response = json.load(opener.open(
             'https://accounts.google.com/o/oauth2/token',
 
@@ skipping 176 matching lines @@
                     self.readSafariMetadata()
                 elif self.config.type == 'gecko':
                     self.readGeckoMetadata()
                 elif self.config.type == 'edge':
                     self.read_edge_metadata()
                 else:
                     raise Exception('Unknown build type {}' % self.config.type)
 
                 # create development build
                 self.build()
+                if self.config.type not in self.downloadable_repos:
+                    # write out changelog
+                    self.writeChangelog(self.getChanges())
 
-                # write out changelog
-                self.writeChangelog(self.getChanges())
-
-                # write update manifest
-                self.writeUpdateManifest()
+                    # write update manifest
+                    self.writeUpdateManifest()
 
             # retire old builds
             versions = self.retireBuilds()
 
             if self.config.type == 'ie':
                 self.writeIEUpdateManifest(versions)
 
-            # update index page
-            self.updateIndex(versions)
+            if self.config.type not in self.downloadable_repos:
+                # update index page
+                self.updateIndex(versions)
 
             # update nightlies config
             self.config.latestRevision = self.revision
 
             if (self.config.type == 'gecko' and
                     self.config.galleryID and
                     get_config().has_option('extensions', 'amo_key')):
                 self.uploadToMozillaAddons()
             elif self.config.type == 'chrome' and self.config.clientID and self.config.clientSecret and self.config.refreshToken:
                 self.uploadToChromeWebStore()
             elif self.config.type == 'edge' and self.config.clientID and self.config.clientSecret and self.config.refreshToken and self.config.tenantID:
                 self.upload_to_windows_store()
 
         finally:
             # clean up
             if self.tempdir:
                 shutil.rmtree(self.tempdir, ignore_errors=True)
 
+    def download(self):
+        with open(get_config().get('extensions', 'downloadLockFile')) as fp:
+            download_info = json.load(fp)
 
-def main():
+        downloads = self.downloadable_repos.intersection(download_info.keys())
+
+        if self.config.type in downloads:
+            try:
+                self.copyRepository()
+                self.readGeckoMetadata()
+
+                for data in download_info[self.config.type]:
+                    self.version = data['version']
+
+                    self.download_from_mozilla_addons(**data)
+
+                    # write out changelog
+                    self.writeChangelog(self.getChanges())
+
+                    # write update manifest
+                    self.writeUpdateManifest()
+
+                    # retire old builds
+                    versions = self.retireBuilds()
+                    # update index page
+                    self.updateIndex(versions)
+            finally:
+                # clean up
+                if self.tempdir:
+                    shutil.rmtree(self.tempdir, ignore_errors=True)
+
+
+def main(download=False):
     """
       main function for createNightlies.py
     """
     nightlyConfig = ConfigParser.SafeConfigParser()
     nightlyConfigFile = get_config().get('extensions', 'nightliesData')
+
     if os.path.exists(nightlyConfigFile):
         nightlyConfig.read(nightlyConfigFile)
 
     # build all extensions specified in the configuration file
     # and generate changelogs and documentations for each:
     data = None
     for repo in Configuration.getRepositoryConfigurations(nightlyConfig):
         build = None
         try:
             build = NightlyBuild(repo)
-            if build.hasChanges():
+            if download:
+                build.download()
+            elif build.hasChanges():
                 build.run()
         except Exception as ex:
             logging.error('The build for %s failed:', repo)
             logging.exception(ex)
 
     file = open(nightlyConfigFile, 'wb')
     nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
-    main()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--download', action='store_true', default=False)
+    args = parser.parse_args()
+    main(args.download)