Issue 6371 - Update buildtools dep. to c830dfa08e2f, use AMO-signing API

sitescripts/extensions/bin/createNightlies.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 
 Nightly builds generation script
 ================================
 
   This script generates nightly builds of extensions, together
   with changelogs and documentation.
 
 """
 
+import argparse
 import ConfigParser
 import base64
 import hashlib
 import hmac
 import json
 import logging
 import os
 import pipes
 import random
 import shutil
@@ skipping 27 matching lines @@
         raise urllib2.HTTPError(req.get_full_url(), code,
                                 '{}\n{}'.format(msg, fp.read()), hdrs, fp)
 
 
 class NightlyBuild(object):
     """
       Performs the build process for an extension,
       generating changelogs and documentation.
     """
 
+    downloadable_repos = {'gecko'}
+
     def __init__(self, config):
         """
           Creates a NightlyBuild instance; we are simply
           recording the configuration settings here.
         """
         self.config = config
         self.revision = self.getCurrentRevision()
         try:
             self.previousRevision = config.latestRevision
         except:
@@ skipping 15 matching lines @@
             'defaults.id=', self.config.repository
         ]
         return subprocess.check_output(command).strip()
 
     def getCurrentBuild(self):
         """
             calculates the (typically numerical) build ID for the current build
         """
         command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
         build = subprocess.check_output(command).strip()
-        if self.config.type == 'gecko':
-            build += 'beta'
         return build
 
     def getChanges(self):
         """
           retrieve changes between the current and previous ("first") revision
         """
         command = [
             'hg', 'log', '-R', self.tempdir, '-r',
             'reverse(ancestors({}))'.format(self.config.revision), '-l', '50',
             '--encoding', 'utf-8', '--template',
@@ skipping 14 matching lines @@
         self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName)
         command = ['hg', 'clone', '-q', self.config.repository, '-u',
                    self.config.revision, self.tempdir]
         subprocess.check_call(command)
 
         # Make sure to run ensure_dependencies.py if present
         depscript = os.path.join(self.tempdir, 'ensure_dependencies.py')
         if os.path.isfile(depscript):
             subprocess.check_call([sys.executable, depscript, '-q'])
 
+    def symlink_or_copy(self, source, target):
+        if hasattr(os, 'symlink'):
+            if os.path.exists(target):
+                os.remove(target)
+            os.symlink(os.path.basename(source), target)
+        else:
+            shutil.copyfile(source, target)
+
     def writeChangelog(self, changes):
         """
           write the changelog file into the cloned repository
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if not os.path.exists(baseDir):
             os.makedirs(baseDir)
         changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version)
         changelogPath = os.path.join(baseDir, changelogFile)
         self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + changelogFile)
 
         template = get_template(get_config().get('extensions', 'changelogTemplate'))
         template.stream({'changes': changes}).dump(changelogPath, encoding='utf-8')
 
         linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
-        if hasattr(os, 'symlink'):
-            if os.path.exists(linkPath):
-                os.remove(linkPath)
-            os.symlink(os.path.basename(changelogPath), linkPath)
-        else:
-            shutil.copyfile(changelogPath, linkPath)
+        self.symlink_or_copy(changelogPath, linkPath)
 
     def readGeckoMetadata(self):
         """
           read Gecko-specific metadata file from a cloned repository
           and parse id, version, basename and the compat section
           out of the file
         """
         import buildtools.packagerChrome as packager
         metadata = packager.readMetadata(self.tempdir, self.config.type)
-        self.extensionID = metadata.get('general', 'id')
+        self.extensionID = packager.get_app_id(False, metadata)
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.basename = metadata.get('general', 'basename')
+        self.min_version = metadata.get('compat', 'gecko')
 
     def readAndroidMetadata(self):
         """
           Read Android-specific metadata from AndroidManifest.xml file.
         """
         manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), 'r')
         manifest = parseXml(manifestFile)
         manifestFile.close()
 
         root = manifest.documentElement
@@ skipping 26 matching lines @@
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.basename = metadata.get('general', 'basename')
 
         self.compat = []
         if metadata.has_section('compat') and metadata.has_option('compat', 'chrome'):
             self.compat.append({'id': 'chrome', 'minVersion': metadata.get('compat', 'chrome')})
 
     def readSafariMetadata(self):
         import sitescripts.extensions.bin.legacy.packagerSafari as packager
-        from buildtools import xarfile
+        from sitescripts.extensions.bin.legacy import xarfile
         metadata = packager.readMetadata(self.tempdir, self.config.type)
         certs = xarfile.read_certificates_and_key(self.config.keyFile)[0]
 
         self.certificateID = packager.get_developer_identifier(certs)
         self.version = packager.getBuildVersion(self.tempdir, metadata, False,
                                                 self.buildNum)
         self.shortVersion = metadata.get('general', 'version')
         self.basename = metadata.get('general', 'basename')
         self.updatedFromGallery = False
 
@@ skipping 16 matching lines @@
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if self.config.type == 'safari':
             manifestPath = os.path.join(baseDir, 'updates.plist')
             templateName = 'safariUpdateManifest'
             autoescape = True
         elif self.config.type == 'android':
             manifestPath = os.path.join(baseDir, 'updates.xml')
             templateName = 'androidUpdateManifest'
             autoescape = True
+        elif self.config.type == 'gecko':
+            manifestPath = os.path.join(baseDir, 'updates.json')
+            templateName = 'geckoUpdateManifest'
+            autoescape = True
         else:
             return
 
         if not os.path.exists(baseDir):
             os.makedirs(baseDir)
 
         # ABP for Android used to have its own update manifest format. We need to
         # generate both that and the new one in the libadblockplus format as long
         # as a significant amount of users is on an old version.
         if self.config.type == 'android':
@@ skipping 24 matching lines @@
         from sitescripts.extensions.utils import writeIEUpdateManifest as doWrite
         doWrite(manifestPath, [{
             'basename': self.basename,
             'version': version,
             'updateURL': updateURL
         }])
 
         for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']:
             linkPath = os.path.join(baseDir, '00latest%s' % suffix)
             outputPath = os.path.join(baseDir, self.basename + '-' + version + suffix)
-            if hasattr(os, 'symlink'):
-                if os.path.exists(linkPath):
-                    os.remove(linkPath)
-                os.symlink(os.path.basename(outputPath), linkPath)
-            else:
-                shutil.copyfile(outputPath, linkPath)
+            self.symlink_or_copy(outputPath, linkPath)
 
     def build(self):
         """
           run the build command in the tempdir
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if not os.path.exists(baseDir):
             os.makedirs(baseDir)
         outputFile = '%s-%s%s' % (self.basename, self.version, self.config.packageSuffix)
         self.path = os.path.join(baseDir, outputFile)
@@ skipping 17 matching lines @@
                 # clear broken output if any
                 if os.path.exists(self.path):
                     os.remove(self.path)
                 raise
         else:
             env = os.environ
             spiderMonkeyBinary = self.config.spiderMonkeyBinary
             if spiderMonkeyBinary:
                 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
 
-            command = [os.path.join(self.tempdir, 'build.py'),
-                       'build', '-t', self.config.type, '-b', self.buildNum]
+            command = [os.path.join(self.tempdir, 'build.py')]
+            if self.config.type == 'safari':
+                command.extend(['-t', self.config.type, 'build'])
+            else:
+                command.extend(['build', '-t', self.config.type])
+            command.extend(['-b', self.buildNum])
+
             if self.config.type not in {'gecko', 'edge'}:
                 command.extend(['-k', self.config.keyFile])
             command.append(self.path)
             subprocess.check_call(command, env=env)
 
         if not os.path.exists(self.path):
             raise Exception("Build failed, output file hasn't been created")
 
-        linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
-        if hasattr(os, 'symlink'):
-            if os.path.exists(linkPath):
-                os.remove(linkPath)
-            os.symlink(os.path.basename(self.path), linkPath)
-        else:
-            shutil.copyfile(self.path, linkPath)
+        if self.config.type not in self.downloadable_repos:
+            linkPath = os.path.join(baseDir,
+                                    '00latest' + self.config.packageSuffix)
+            self.symlink_or_copy(self.path, linkPath)
 
     def retireBuilds(self):
         """
           removes outdated builds, returns the sorted version numbers of remaining
           builds
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         versions = []
         prefix = self.basename + '-'
         suffix = self.config.packageSuffix
@@ skipping 32 matching lines @@
                 'download': packageFile,
                 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)),
                 'size': os.path.getsize(os.path.join(baseDir, packageFile))
             }
             if os.path.exists(os.path.join(baseDir, changelogFile)):
                 link['changelog'] = changelogFile
             links.append(link)
         template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
         template.stream({'config': self.config, 'links': links}).dump(outputPath)
 
-    def uploadToMozillaAddons(self):
-        import urllib3
+    def read_downloads_lockfile(self):
+        path = get_config().get('extensions', 'downloadLockFile')
+        try:
+            with open(path, 'r') as fp:
+                current = json.load(fp)
+        except IOError:
+            logging.warning('No lockfile found. Creating ' + path)
+            current = {}
 
+        return current
+
+    def write_downloads_lockfile(self, values):
+        path = get_config().get('extensions', 'downloadLockFile')
+        with open(path, 'w') as fp:
+            json.dump(values, fp)
+
+    def add_to_downloads_lockfile(self, platform, values):
+        current = self.read_downloads_lockfile()
+
+        current.setdefault(platform, [])
+        current[platform].append(values)
+
+        self.write_downloads_lockfile(current)
+
+    def remove_from_downloads_lockfile(self, platform, filter_key,
+                                       filter_value):
+        current = self.read_downloads_lockfile()
+        try:
+            for i, entry in enumerate(current[platform]):
+                if entry[filter_key] == filter_value:
+                    del current[platform][i]
+                if len(current[platform]) == 0:
+                    del current[platform]
+        except KeyError:
+            pass
+        self.write_downloads_lockfile(current)
+
+    def generate_jwt_request(self, issuer, secret, url, method, data=None,
+                             add_headers=[]):
         header = {
             'alg': 'HS256',     # HMAC-SHA256
             'typ': 'JWT',
         }
 
         issued = int(time.time())
         payload = {
-            'iss': get_config().get('extensions', 'amo_key'),
+            'iss': issuer,
             'jti': random.random(),
             'iat': issued,
             'exp': issued + 60,
         }
 
-        input = '{}.{}'.format(
+        hmac_data = '{}.{}'.format(
             base64.b64encode(json.dumps(header)),
             base64.b64encode(json.dumps(payload))
         )
 
-        signature = hmac.new(get_config().get('extensions', 'amo_secret'),
-                             msg=input,
+        signature = hmac.new(secret, msg=hmac_data,
                              digestmod=hashlib.sha256).digest()
-        token = '{}.{}'.format(input, base64.b64encode(signature))
+        token = '{}.{}'.format(hmac_data, base64.b64encode(signature))
+
+        request = urllib2.Request(url, data)
+        request.add_header('Authorization', 'JWT ' + token)
+        for header in add_headers:
+            request.add_header(*header)
+        request.get_method = lambda: method
+
+        return request
+
+    def uploadToMozillaAddons(self):
+        import urllib3
+
+        config = get_config()
 
         upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
                       'versions/{}/').format(self.extensionID, self.version)
 
         with open(self.path, 'rb') as file:
             data, content_type = urllib3.filepost.encode_multipart_formdata({
                 'upload': (
                     os.path.basename(self.path),
                     file.read(),
                     'application/x-xpinstall'
                 )
             })
 
-        request = urllib2.Request(upload_url, data=data)
-        request.add_header('Content-Type', content_type)
-        request.add_header('Authorization', 'JWT ' + token)
-        request.get_method = lambda: 'PUT'
+        request = self.generate_jwt_request(
+            config.get('extensions', 'amo_key'),
+            config.get('extensions', 'amo_secret'),
+            upload_url,
+            'PUT',
+            data,
+            (('Content-Type', content_type),),

[Sebastian Noack, 2018/03/09 16:16:27]

Nit: I missed this one before. Again, comma after last argument.

Also the outer data structure should be a list, not at tuple. We only use tuples
for data that have structure (e.g. objects that can be unpacked, like the inner
structure here), and lists for data that have order (i.e. objects that have a
variable number of elements that are processed in a uniform way).

  self.generate_jwt_request(
      config.get('extensions', 'amo_key'),
      config.get('extensions', 'amo_secret'),
      upload_url,
      'PUT',
      data,
      [('Content-Type', content_type)]
  )

[tlucas, 2018/03/12 07:22:46]

Done.

[Vasily Kuznetsov, 2018/03/13 17:49:59]

I actually prefer the comma after the last argument in such constructions. It
allows adding and removing arguments without touching irrelevant lines. This is
especially important for something like dict literals, but seems also slightly
preferable in cases like this one.

+        )
 
         try:
             urllib2.urlopen(request).close()
         except urllib2.HTTPError as e:
             try:
                 logging.error(e.read())
             finally:
                 e.close()
             raise
 
+        self.add_to_downloads_lockfile(
+            self.config.type,
+            {
+                'buildtype': 'devbuild',
+                'app_id': self.extensionID,
+                'version': self.version,
+            }
+        )
+        os.remove(self.path)
+
+    def download_from_mozilla_addons(self, buildtype, version, app_id):
+        config = get_config()
+        iss = config.get('extensions', 'amo_key')
+        secret = config.get('extensions', 'amo_secret')
+
+        url = ('https://addons.mozilla.org/api/v3/addons/{}/'
+               'versions/{}/').format(app_id, version)
+
+        request = self.generate_jwt_request(iss, secret, url, 'GET')
+        response = json.load(urllib2.urlopen(request))
+
+        necessary = ['passed_review', 'reviewed', 'processed', 'valid']
+        if all(response[x] for x in necessary):
+            download_url = response['files'][0]['download_url']
+            checksum = response['files'][0]['hash']
+
+            filename = '{}-{}.xpi'.format(self.basename, version)
+            file_path = os.path.join(
+                config.get('extensions', 'nightliesDirectory'),
+                self.basename,
+                filename
+            )
+
+            request = self.generate_jwt_request(iss, secret, download_url,
+                                                'GET')
+            try:
+                response = urllib2.urlopen(request)
+            except urllib2.HTTPError as e:
+                logging.error(e.read())
+
+            # Verify the extension's integrity
+            file_content = response.read()
+            sha256 = hashlib.sha256(file_content)
+            returned_checksum = '{}:{}'.format(sha256.name, sha256.hexdigest())
+
+            if returned_checksum != checksum:
+                logging.error('Checksum could not be verified: {} vs {}'
+                              ''.format(checksum, returned_checksum))
+
+            with open(file_path, 'w') as fp:
+                fp.write(file_content)
+
+            self.update_link = os.path.join(
+                config.get('extensions', 'nightliesURL'),
+                self.basename,
+                filename
+            )
+
+            self.remove_from_downloads_lockfile(self.config.type,
+                                                'version',
+                                                version)
+        elif not response['passed_review'] or not response['valid']:
+            # When the review failed for any reason, we want to know about it
+            logging.error(json.dumps(response, indent=4))
+            self.remove_from_downloads_lockfile(self.config.type,
+                                                'version',
+                                                version)
+
     def uploadToChromeWebStore(self):
 
         opener = urllib2.build_opener(HTTPErrorBodyHandler)
 
         # use refresh token to obtain a valid access token
         # https://developers.google.com/accounts/docs/OAuth2WebServer#refresh
 
         response = json.load(opener.open(
             'https://accounts.google.com/o/oauth2/token',
 
@@ skipping 176 matching lines @@
                     self.readSafariMetadata()
                 elif self.config.type == 'gecko':
                     self.readGeckoMetadata()
                 elif self.config.type == 'edge':
                     self.read_edge_metadata()
                 else:
                     raise Exception('Unknown build type {}' % self.config.type)
 
                 # create development build
                 self.build()
+                if self.config.type not in self.downloadable_repos:
+                    # write out changelog
+                    self.writeChangelog(self.getChanges())
 
-                # write out changelog
-                self.writeChangelog(self.getChanges())
-
-                # write update manifest
-                self.writeUpdateManifest()
+                    # write update manifest
+                    self.writeUpdateManifest()
 
             # retire old builds
             versions = self.retireBuilds()
 
             if self.config.type == 'ie':
                 self.writeIEUpdateManifest(versions)
 
-            # update index page
-            self.updateIndex(versions)
+            if self.config.type not in self.downloadable_repos:
+                # update index page
+                self.updateIndex(versions)
 
             # update nightlies config
             self.config.latestRevision = self.revision
 
             if (self.config.type == 'gecko' and
                     self.config.galleryID and
                     get_config().has_option('extensions', 'amo_key')):
                 self.uploadToMozillaAddons()
             elif self.config.type == 'chrome' and self.config.clientID and self.config.clientSecret and self.config.refreshToken:
                 self.uploadToChromeWebStore()
             elif self.config.type == 'edge' and self.config.clientID and self.config.clientSecret and self.config.refreshToken and self.config.tenantID:
                 self.upload_to_windows_store()
 
         finally:
             # clean up
             if self.tempdir:
                 shutil.rmtree(self.tempdir, ignore_errors=True)
 
+    def download(self):
+        with open(get_config().get('extensions', 'downloadLockFile')) as fp:
+            download_info = json.load(fp)
 
-def main():
+        downloads = self.downloadable_repos.intersection(download_info.keys())
+
+        if self.config.type in downloads:
+            try:
+                self.copyRepository()
+                self.readGeckoMetadata()
+
+                for data in download_info[self.config.type]:
+                    self.version = data['version']
+
+                    self.download_from_mozilla_addons(**data)
+
+                    # write out changelog
+                    self.writeChangelog(self.getChanges())
+
+                    # write update manifest
+                    self.writeUpdateManifest()
+
+                    # retire old builds
+                    versions = self.retireBuilds()
+                    # update index page
+                    self.updateIndex(versions)
+            finally:
+                # clean up
+                if self.tempdir:
+                    shutil.rmtree(self.tempdir, ignore_errors=True)
+
+
+def main(download=False):
     """
       main function for createNightlies.py
     """
     nightlyConfig = ConfigParser.SafeConfigParser()
     nightlyConfigFile = get_config().get('extensions', 'nightliesData')
+
     if os.path.exists(nightlyConfigFile):
         nightlyConfig.read(nightlyConfigFile)
 
     # build all extensions specified in the configuration file
     # and generate changelogs and documentations for each:
     data = None
     for repo in Configuration.getRepositoryConfigurations(nightlyConfig):
         build = None
         try:
             build = NightlyBuild(repo)
-            if build.hasChanges():
+            if download:
+                build.download()
+            elif build.hasChanges():
                 build.run()
         except Exception as ex:
             logging.error('The build for %s failed:', repo)
             logging.exception(ex)
 
     file = open(nightlyConfigFile, 'wb')
     nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
-    main()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--download', action='store_true', default=False)
+    args = parser.parse_args()
+    main(args.download)