Issue 4098 - Get rid of special build setup for development builds

sitescripts/extensions/bin/createNightlies.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2016 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
@@ skipping 47 matching lines @@
       generating changelogs and documentation.
     """
 
     def __init__(self, config):
         """
           Creates a NightlyBuild instance; we are simply
           recording the configuration settings here.
         """
         self.config = config
         self.revision = self.getCurrentRevision()
-        if self.config.type == 'gecko':
-            self.revision += '-beta'
         try:
             self.previousRevision = config.latestRevision
         except:
             self.previousRevision = '0'
+        self.buildNum = None

[Wladimir Palant, 2016/06/01 14:57:12]

With these changes, self.revision is a revision hash like 814673439cbc, only
used to recognize whether there are changes in the repository. self.buildNum on
the other hand is the numerical revision ID, what we had in self.revision
before. Reason is that we cannot retrieve a numerical revision ID for a remote
repository, self.buildNum has to be populated after cloning the repository.
Side-effect of this change is that all builds will be recreated after this lands
(latestRevision won't match) but that's not really an issue.

[Vasily Kuznetsov, 2016/06/01 17:14:15]

Acknowledged.

         self.tempdir = None
         self.outputFilename = None
         self.changelogFilename = None
 
     def hasChanges(self):
         return self.revision != self.previousRevision
 
     def getCurrentRevision(self):
         """
-          retrieves the current revision number from the repository
+            retrieves the current revision ID from the repository
         """
-        command = ['hg', 'log', '-R', self.config.repository, '-r', 'default',
-                   '--template', '{rev}', '--config', 'defaults.log=']
-        return subprocess.check_output(command)
+        command = [
+            'hg', 'id', '-i', '-r', 'default', '--config', 'defaults.id=',

[Vasily Kuznetsov, 2016/06/01 17:14:15]

This `--config defaults.id=` is there to make sure there are no additional
options to `hg id` coming from [defaults] section of .hgrc, right? Since
[defaults] section seems to be deprecated, I was wandering, are we using it
somewhere on the build servers? And if not, is this override needed?

[Wladimir Palant, 2016/06/03 14:14:05]

"Breaking automation is only one of the reason to not use defaults." :)

No, we don't use defaults on our servers. This was added in order to simplify
testing, because local machines (mine in particular) tend to have defaults.

[Vasily Kuznetsov, 2016/06/03 16:04:49]

Acknowledged.

+            self.config.repository
+        ]
+        return subprocess.check_output(command).strip()
+
+    def getCurrentBuild(self):
+        """
+            calculates the (typically numerical) build ID for the current build
+        """
+        command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
+        build = subprocess.check_output(command).strip()
+        if self.config.type == 'gecko':
+            build += '-beta'
+        return build
 
     def getChanges(self):
         """
           retrieve changes between the current and previous ("first") revision
         """
 
-        command = ['hg', 'log', '-R', self.config.repository, '-r', 'tip:0',
+        command = ['hg', 'log', '-R', self.tempdir, '-r', 'tip:0',
                    '-b', 'default', '-l', '50', '--encoding', 'utf-8',
                    '--template', '{date|isodate}\\0{author|person}\\0{rev}\\0{desc}\\0\\0',
                    '--config', 'defaults.log=']
         result = subprocess.check_output(command).decode('utf-8')
 
         for change in result.split('\x00\x00'):
             if change:
                 date, author, revision, description = change.split('\x00')
                 yield {'date': date, 'author': author, 'revision': revision, 'description': description}
 
     def copyRepository(self):
         """
           Create a repository copy in a temporary directory
         """
-        # We cannot use hg archive here due to
-        # http://bz.selenic.com/show_bug.cgi?id=3747, have to clone properly :-(
         self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName)
         command = ['hg', 'clone', '-q', self.config.repository, '-u', 'default', self.tempdir]
         subprocess.check_call(command)
 
-        # Make sure to process the dependencies file if it is present
-        import logging
-        logging.disable(logging.WARNING)
-        try:
-            from buildtools.ensure_dependencies import resolve_deps
-            resolve_deps(self.tempdir, self_update=False,
-                         overrideroots={'hg': os.path.abspath(
-                             os.path.join(self.config.repository, os.pardir)
-                         )},
-                         skipdependencies={'buildtools'})
-        finally:
-            logging.disable(logging.NOTSET)
+        # Make sure to run ensure_dependencies.py if present
+        command = [os.path.join(self.tempdir, 'ensure_dependencies.py'), '-q']
+        if os.path.isfile(command[0]):

[Vasily Kuznetsov, 2016/06/01 17:14:15]

Are we sure that `ensure_dependecies.py` is always executable. Maybe we should
check if it is or do ['python', '....../ensure_dependencies.py', '-q'] instead
to be more robust?

[Wladimir Palant, 2016/06/03 14:14:06]

You are right, build.py calls Python explicitly so it doesn't have to be
executable. Fixed.

+            subprocess.check_call(command)
 
     def writeChangelog(self, changes):
         """
           write the changelog file into the cloned repository
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if not os.path.exists(baseDir):
             os.makedirs(baseDir)
         changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version)
         changelogPath = os.path.join(baseDir, changelogFile)
         self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + changelogFile)
 
         template = get_template(get_config().get('extensions', 'changelogTemplate'))
         template.stream({'changes': changes}).dump(changelogPath, encoding='utf-8')
 
         linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
         if hasattr(os, 'symlink'):
             if os.path.exists(linkPath):
                 os.remove(linkPath)
             os.symlink(os.path.basename(changelogPath), linkPath)
         else:
             shutil.copyfile(changelogPath, linkPath)
 
     def readGeckoMetadata(self):
         """
           read Gecko-specific metadata file from a cloned repository
           and parse id, version, basename and the compat section
           out of the file
         """
         import buildtools.packagerGecko as packager

[Wladimir Palant, 2016/06/01 14:57:12]

We are still using latest buildtools revision to read metadata here. Eventually
we probably want to create a command line interface to retrieve this data, it's
not such a big issue however.

[Vasily Kuznetsov, 2016/06/01 17:14:15]

Acknowledged.

         metadata = packager.readMetadata(self.tempdir, self.config.type)
         self.extensionID = metadata.get('general', 'id')
-        self.version = packager.getBuildVersion(self.tempdir, metadata, False, self.revision)
+        self.version = packager.getBuildVersion(self.tempdir, metadata, False, self.buildNum)

[Vasily Kuznetsov, 2016/06/01 17:14:15]

This line is too long, although of course, it's not the fault of this change
(and it's definitely not the only one in this file). But perhaps you could wrap
it (and a few other changed long lines) while it's being changed anyway.

[Wladimir Palant, 2016/06/03 14:14:04]

This add too many unrelated changes for my taste but well - done.

         self.basename = metadata.get('general', 'basename')
         self.compat = []
         for key, value in packager.KNOWN_APPS.iteritems():
             if metadata.has_option('compat', key):
                 minVersion, maxVersion = metadata.get('compat', key).split('/')
                 self.compat.append({'id': value, 'minVersion': minVersion, 'maxVersion': maxVersion})
 
     def readAndroidMetadata(self):
         """
           Read Android-specific metadata from AndroidManifest.xml file.
         """
         manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), 'r')
         manifest = parseXml(manifestFile)
         manifestFile.close()
 
         root = manifest.documentElement
         self.version = root.attributes['android:versionName'].value
         while self.version.count('.') < 2:
             self.version += '.0'
-        self.version = '%s.%s' % (self.version, self.revision)
+        self.version = '%s.%s' % (self.version, self.buildNum)
 
         usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
         self.minSdkVersion = usesSdk.attributes['android:minSdkVersion'].value
         self.basename = os.path.basename(self.config.repository)
 
     def readChromeMetadata(self):
         """
           Read Chrome-specific metadata from metadata file. This will also
           calculate extension ID from the private key.
         """
 
         # Calculate extension ID from public key
         # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-from-your-private-key-233)
         import buildtools.packagerChrome as packager
         publicKey = packager.getPublicKey(self.config.keyFile)
         hash = hashlib.sha256()
         hash.update(publicKey)
         self.extensionID = hash.hexdigest()[0:32]
         self.extensionID = ''.join(map(lambda c: chr(97 + int(c, 16)), self.extensionID))
 
         # Now read metadata file
         metadata = packager.readMetadata(self.tempdir, self.config.type)
-        self.version = packager.getBuildVersion(self.tempdir, metadata, False, self.revision)
+        self.version = packager.getBuildVersion(self.tempdir, metadata, False, self.buildNum)
         self.basename = metadata.get('general', 'basename')
 
         self.compat = []
         if metadata.has_section('compat') and metadata.has_option('compat', 'chrome'):
             self.compat.append({'id': 'chrome', 'minVersion': metadata.get('compat', 'chrome')})
 
     def readSafariMetadata(self):
         import buildtools.packagerSafari as packager
         metadata = packager.readMetadata(self.tempdir, self.config.type)
         certs = packager.get_certificates_and_key(self.config.keyFile)[0]
 
         self.certificateID = packager.get_developer_identifier(certs)
-        self.version = packager.getBuildVersion(self.tempdir, metadata, False, self.revision)
+        self.version = packager.getBuildVersion(self.tempdir, metadata, False, self.buildNum)
         self.shortVersion = metadata.get('general', 'version')
         self.basename = metadata.get('general', 'basename')
         self.updatedFromGallery = False
 
     def writeUpdateManifest(self):
         """
           Writes update.rdf file for the current build
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if self.config.type == 'safari':
@@ skipping 65 matching lines @@
 
         if self.config.type == 'android':
             apkFile = open(self.path, 'wb')
 
             try:
                 try:
                     port = get_config().get('extensions', 'androidBuildPort')
                 except ConfigParser.NoOptionError:
                     port = '22'
                 buildCommand = ['ssh', '-p', port, get_config().get('extensions', 'androidBuildHost')]
-                buildCommand.extend(map(pipes.quote, ['/home/android/bin/makedebugbuild.py', '--revision', self.revision, '--version', self.version, '--stdout']))
+                buildCommand.extend(map(pipes.quote, ['/home/android/bin/makedebugbuild.py', '--revision', self.buildNum, '--version', self.version, '--stdout']))
                 subprocess.check_call(buildCommand, stdout=apkFile, close_fds=True)
             except:
                 # clear broken output if any
                 if os.path.exists(self.path):
                     os.remove(self.path)
                 raise
-        elif self.config.type == 'chrome':
-            import buildtools.packagerChrome as packager
-            packager.createBuild(self.tempdir, type=self.config.type, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
-        elif self.config.type == 'safari':
-            import buildtools.packagerSafari as packager
-            packager.createBuild(self.tempdir, type=self.config.type, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
         else:
-            import buildtools.packagerGecko as packager
-            packager.createBuild(self.tempdir, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
+            buildCommand = [

[Vasily Kuznetsov, 2016/06/01 17:14:15]

A nice side effect that we got rid of this branching!

[Wladimir Palant, 2016/06/03 14:14:04]

It will come back, as soon as we remove signing support for Gecko :)

[Vasily Kuznetsov, 2016/06/03 16:04:49]

:)

+                os.path.join(self.tempdir, 'build.py'), '-t', self.config.type,
+                'build', '-b', self.buildNum, '-k', self.config.keyFile,
+                self.path
+            ]
+            subprocess.check_call(buildCommand)
 
         if not os.path.exists(self.path):
             raise Exception("Build failed, output file hasn't been created")
 
         linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
         if hasattr(os, 'symlink'):
             if os.path.exists(linkPath):
                 os.remove(linkPath)
             os.symlink(os.path.basename(self.path), linkPath)
         else:
@@ skipping 221 matching lines @@
           Run the nightly build process for one extension
         """
         try:
             if self.config.type == 'ie':
                 # We cannot build IE builds, simply list the builds already in
                 # the directory. Basename has to be deduced from the repository name.
                 self.basename = os.path.basename(self.config.repository)
             else:
                 # copy the repository into a temporary directory
                 self.copyRepository()
+                self.buildNum = self.getCurrentBuild()
 
                 # get meta data from the repository
                 if self.config.type == 'android':
                     self.readAndroidMetadata()
                 elif self.config.type == 'chrome':
                     self.readChromeMetadata()
                 elif self.config.type == 'safari':
                     self.readSafariMetadata()
                 else:
                     self.readGeckoMetadata()
@@ skipping 51 matching lines @@
         except Exception, ex:
             logging.error('The build for %s failed:', repo)
             logging.exception(ex)
 
     file = open(nightlyConfigFile, 'wb')
     nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
     main()