Issue 4098 - Get rid of special build setup for development builds

sitescripts/extensions/bin/createNightlies.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2016 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
@@ skipping 47 matching lines @@
       generating changelogs and documentation.
     """
 
     def __init__(self, config):
         """
           Creates a NightlyBuild instance; we are simply
           recording the configuration settings here.
         """
         self.config = config
         self.revision = self.getCurrentRevision()
-        if self.config.type == 'gecko':
-            self.revision += '-beta'
         try:
             self.previousRevision = config.latestRevision
         except:
             self.previousRevision = '0'
+        self.buildNum = None
         self.tempdir = None
         self.outputFilename = None
         self.changelogFilename = None
 
     def hasChanges(self):
         return self.revision != self.previousRevision
 
     def getCurrentRevision(self):
         """
-          retrieves the current revision number from the repository
+            retrieves the current revision ID from the repository
         """
-        command = ['hg', 'log', '-R', self.config.repository, '-r', 'default',
-                   '--template', '{rev}', '--config', 'defaults.log=']
-        return subprocess.check_output(command)
+        command = [
+            'hg', 'id', '-i', '-r', 'default', '--config', 'defaults.id=',
+            self.config.repository
+        ]
+        return subprocess.check_output(command).strip()
+
+    def getCurrentBuild(self):
+        """
+            calculates the (typically numerical) build ID for the current build
+        """
+        command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
+        build = subprocess.check_output(command).strip()
+        if self.config.type == 'gecko':
+            build += '-beta'
+        return build
 
     def getChanges(self):
         """
           retrieve changes between the current and previous ("first") revision
         """
 
-        command = ['hg', 'log', '-R', self.config.repository, '-r', 'tip:0',
+        command = ['hg', 'log', '-R', self.tempdir, '-r', 'tip:0',
                    '-b', 'default', '-l', '50', '--encoding', 'utf-8',
                    '--template', '{date|isodate}\\0{author|person}\\0{rev}\\0{desc}\\0\\0',
                    '--config', 'defaults.log=']
         result = subprocess.check_output(command).decode('utf-8')
 
         for change in result.split('\x00\x00'):
             if change:
                 date, author, revision, description = change.split('\x00')
                 yield {'date': date, 'author': author, 'revision': revision, 'description': description}
 
     def copyRepository(self):
         """
           Create a repository copy in a temporary directory
         """
-        # We cannot use hg archive here due to
-        # http://bz.selenic.com/show_bug.cgi?id=3747, have to clone properly :-(
         self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName)
         command = ['hg', 'clone', '-q', self.config.repository, '-u', 'default', self.tempdir]
         subprocess.check_call(command)
 
-        # Make sure to process the dependencies file if it is present
-        import logging
-        logging.disable(logging.WARNING)
-        try:
-            from buildtools.ensure_dependencies import resolve_deps
-            resolve_deps(self.tempdir, self_update=False,
-                         overrideroots={'hg': os.path.abspath(
-                             os.path.join(self.config.repository, os.pardir)
-                         )},
-                         skipdependencies={'buildtools'})
-        finally:
-            logging.disable(logging.NOTSET)
+        # Make sure to run ensure_dependencies.py if present
+        command = [
+            sys.executable,
+            os.path.join(self.tempdir, 'ensure_dependencies.py'), '-q'
+        ]
+        if os.path.isfile(command[0]):

[Vasily Kuznetsov, 2016/06/03 16:04:49]

Should not this become `command[1]` now that we've inserted python as the first
item of the list?

[Wladimir Palant, 2016/06/06 11:47:46]

Ouch, you are right. In fact, I had a better idea in order to make this code
less fragile.

[Vasily Kuznetsov, 2016/06/06 12:48:16]

Lovely!

+            subprocess.check_call(command)
 
     def writeChangelog(self, changes):
         """
           write the changelog file into the cloned repository
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if not os.path.exists(baseDir):
             os.makedirs(baseDir)
         changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version)
         changelogPath = os.path.join(baseDir, changelogFile)
@@ skipping 12 matching lines @@
 
     def readGeckoMetadata(self):
         """
           read Gecko-specific metadata file from a cloned repository
           and parse id, version, basename and the compat section
           out of the file
         """
         import buildtools.packagerGecko as packager
         metadata = packager.readMetadata(self.tempdir, self.config.type)
         self.extensionID = metadata.get('general', 'id')
-        self.version = packager.getBuildVersion(self.tempdir, metadata, False, self.revision)
+        self.version = packager.getBuildVersion(self.tempdir, metadata, False,
+                                                self.buildNum)
         self.basename = metadata.get('general', 'basename')
         self.compat = []
         for key, value in packager.KNOWN_APPS.iteritems():
             if metadata.has_option('compat', key):
                 minVersion, maxVersion = metadata.get('compat', key).split('/')
                 self.compat.append({'id': value, 'minVersion': minVersion, 'maxVersion': maxVersion})
 
     def readAndroidMetadata(self):
         """
           Read Android-specific metadata from AndroidManifest.xml file.
         """
         manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), 'r')
         manifest = parseXml(manifestFile)
         manifestFile.close()
 
         root = manifest.documentElement
         self.version = root.attributes['android:versionName'].value
         while self.version.count('.') < 2:
             self.version += '.0'
-        self.version = '%s.%s' % (self.version, self.revision)
+        self.version = '%s.%s' % (self.version, self.buildNum)
 
         usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
         self.minSdkVersion = usesSdk.attributes['android:minSdkVersion'].value
         self.basename = os.path.basename(self.config.repository)
 
     def readChromeMetadata(self):
         """
           Read Chrome-specific metadata from metadata file. This will also
           calculate extension ID from the private key.
         """
 
         # Calculate extension ID from public key
         # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-from-your-private-key-233)
         import buildtools.packagerChrome as packager
         publicKey = packager.getPublicKey(self.config.keyFile)
         hash = hashlib.sha256()
         hash.update(publicKey)
         self.extensionID = hash.hexdigest()[0:32]
         self.extensionID = ''.join(map(lambda c: chr(97 + int(c, 16)), self.extensionID))
 
         # Now read metadata file
         metadata = packager.readMetadata(self.tempdir, self.config.type)
-        self.version = packager.getBuildVersion(self.tempdir, metadata, False, self.revision)
+        self.version = packager.getBuildVersion(self.tempdir, metadata, False,
+                                                self.buildNum)
         self.basename = metadata.get('general', 'basename')
 
         self.compat = []
         if metadata.has_section('compat') and metadata.has_option('compat', 'chrome'):
             self.compat.append({'id': 'chrome', 'minVersion': metadata.get('compat', 'chrome')})
 
     def readSafariMetadata(self):
         import buildtools.packagerSafari as packager
         metadata = packager.readMetadata(self.tempdir, self.config.type)
         certs = packager.get_certificates_and_key(self.config.keyFile)[0]
 
         self.certificateID = packager.get_developer_identifier(certs)
-        self.version = packager.getBuildVersion(self.tempdir, metadata, False, self.revision)
+        self.version = packager.getBuildVersion(self.tempdir, metadata, False,
+                                                self.buildNum)
         self.shortVersion = metadata.get('general', 'version')
         self.basename = metadata.get('general', 'basename')
         self.updatedFromGallery = False
 
     def writeUpdateManifest(self):
         """
           Writes update.rdf file for the current build
         """
         baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
         if self.config.type == 'safari':
@@ skipping 65 matching lines @@
 
         if self.config.type == 'android':
             apkFile = open(self.path, 'wb')
 
             try:
                 try:
                     port = get_config().get('extensions', 'androidBuildPort')
                 except ConfigParser.NoOptionError:
                     port = '22'
                 buildCommand = ['ssh', '-p', port, get_config().get('extensions', 'androidBuildHost')]
-                buildCommand.extend(map(pipes.quote, ['/home/android/bin/makedebugbuild.py', '--revision', self.revision, '--version', self.version, '--stdout']))
+                buildCommand.extend(map(pipes.quote, [
+                    '/home/android/bin/makedebugbuild.py', '--revision',
+                    self.buildNum, '--version', self.version, '--stdout'
+                ]))
                 subprocess.check_call(buildCommand, stdout=apkFile, close_fds=True)
             except:
                 # clear broken output if any
                 if os.path.exists(self.path):
                     os.remove(self.path)
                 raise
-        elif self.config.type == 'chrome':
-            import buildtools.packagerChrome as packager
-            packager.createBuild(self.tempdir, type=self.config.type, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
-        elif self.config.type == 'safari':
-            import buildtools.packagerSafari as packager
-            packager.createBuild(self.tempdir, type=self.config.type, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
         else:
-            import buildtools.packagerGecko as packager
-            packager.createBuild(self.tempdir, outFile=self.path, buildNum=self.revision, keyFile=self.config.keyFile)
+            buildCommand = [
+                os.path.join(self.tempdir, 'build.py'), '-t', self.config.type,
+                'build', '-b', self.buildNum, '-k', self.config.keyFile,
+                self.path
+            ]
+            subprocess.check_call(buildCommand)
 
         if not os.path.exists(self.path):
             raise Exception("Build failed, output file hasn't been created")
 
         linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
         if hasattr(os, 'symlink'):
             if os.path.exists(linkPath):
                 os.remove(linkPath)
             os.symlink(os.path.basename(self.path), linkPath)
         else:
@@ skipping 221 matching lines @@
           Run the nightly build process for one extension
         """
         try:
             if self.config.type == 'ie':
                 # We cannot build IE builds, simply list the builds already in
                 # the directory. Basename has to be deduced from the repository name.
                 self.basename = os.path.basename(self.config.repository)
             else:
                 # copy the repository into a temporary directory
                 self.copyRepository()
+                self.buildNum = self.getCurrentBuild()
 
                 # get meta data from the repository
                 if self.config.type == 'android':
                     self.readAndroidMetadata()
                 elif self.config.type == 'chrome':
                     self.readChromeMetadata()
                 elif self.config.type == 'safari':
                     self.readSafariMetadata()
                 else:
                     self.readGeckoMetadata()
@@ skipping 51 matching lines @@
         except Exception, ex:
             logging.error('The build for %s failed:', repo)
             logging.exception(ex)
 
     file = open(nightlyConfigFile, 'wb')
     nightlyConfig.write(file)
 
 
 if __name__ == '__main__':
     main()