Issue 5996 - Release consistent versions across WebExtensions

releaseAutomation.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function
 
 import os
+import operator
 import re
 import codecs
+import logging
 import subprocess
+import sys
 import tarfile
 import json
 
-from packager import readMetadata, getDefaultFileName
+from packager import readMetadata, getDefaultFileName, get_extension
+from localeTools import read_locale_config
+
+SOURCE_ARCHIVE = 'adblockplus-{}-source.tgz'
 
 
 def get_dependencies(prefix, repos):
     from ensure_dependencies import read_deps, safe_join
     repo = repos[prefix]
     deps = read_deps(repo)
     if deps:
         for subpath in deps:
             if subpath.startswith('_'):
                 continue
@@ skipping 81 matching lines @@
     """Run repository-checks in order to bail out early if necessary"""
     if repo_has_uncommitted():
         return False
     if repo_has_incoming(*repo_paths):
         return False
     if repo_has_outgoing():
         return continue_with_outgoing()
     return True
 
 
-def run(baseDir, type, version, keyFile, downloadsRepo):
-    if not can_safely_release(baseDir, downloadsRepo):
-        print('Aborting release.')
-        return 1
+def version_is_op_to(value, orig, op):

[Wladimir Palant, 2017/11/28 11:01:16]

For reference, sitescripts.extensions.utils has another version comparison
function. It's probably going to be an overkill here, but the function you
created here could definitely be simpler.

Typical comparison functions return a number - negative, zero or positive. This
makes comparisons simpler:

  def safe_get(list, index):
    return int(list[index]) if index < len(list) else 0

  for i in range(len(value_digits)):
    result = safe_get(value_digits, i) - safe_get(orig_digits, i)
    if result != 0:
      return result
  return 0

Nit: I consider value and orig bad parameter names for a comparison function,
these imply a special meaning of the passed in values. version1 and version2 or
a and b would be better.

[tlucas, 2017/11/28 13:28:43]

Done.

+    """Compare two version numbers (left to right) with a given operator.
 
-    if type == 'edge':
-        import buildtools.packagerEdge as packager
-    elif type == 'chrome':
-        import buildtools.packagerChrome as packager
+    If a passed version contains non-numeric values, the comparison is always
+    False.
+    """
+    def prefix_0(a, b):
+        target_len = max((len(a), len(b)))
+        a = '0' * (target_len - len(a)) + a
+        b = '0' * (target_len - len(b)) + b
+        return a, b
 
-    # Replace version number in metadata file "manually", ConfigParser will mess
-    # up the order of lines.
-    metadata = readMetadata(baseDir, type)
-    with open(metadata.option_source('general', 'version'), 'r+b') as file:
-        rawMetadata = file.read()
+    try:
+        value_digits = [int(v) for v in value.split('.')]
+        orig_digits = [int(v) for v in orig.split('.')]
+
+        value_dec = ''
+        orig_dec = ''
+
+        if len(value_digits) < len(orig_digits):
+            value_digits += [0] * (len(orig_digits) - len(value_digits))
+        elif len(orig_digits) < len(value_digits):
+            orig_digits += [0] * (len(value_digits) - len(orig_digits))
+
+        for i in range(len(value_digits)):
+            val, orig = prefix_0(str(value_digits[i]), str(orig_digits[i]))
+            value_dec += val
+            orig_dec += orig
+
+        return op(int(value_dec), int(orig_dec))
+
+    except ValueError:
+        pass

[Wladimir Palant, 2017/11/28 11:01:16]

How about only passing in tags that match r'^\d+(\.\d+)$' regular expression
instead of catching errors?

[tlucas, 2017/11/28 13:28:43]

Done, respectively only passed tags of the form "1.2.3[-chrome|-gecko|-edge]+"

+
+    return False
+
+
+def release_combination_is_possible(version, platforms, base_dir):

[tlucas, 2017/11/23 12:05:52]

This docstring reflects what was discussed in IRC.

+    """Determine whether a release for the given parameters is possible.
+
+    Examine existing tags in order to find either higher or matching versions.
+    The release is impossible if a) a higher version for a requested platform
+    exists, or if b) a matching version exists and the requested set of
+    platforms differs from what was already released.
+    """
+    considered_tags = [
+        c for c in [
+            t.split('-') for t in
+            subprocess.check_output(
+                ['hg', 'tags', '-R', base_dir, '-q']).split()
+        ] if version_is_op_to(c[0], version, operator.ge)
+    ]
+
+    for tag in considered_tags:
+        if version_is_op_to(tag[0], version, operator.eq):
+            if set(tag[1:]) != set(platforms):
+                return False, ('You have to re-release version {} for exactly '
+                               'all of {}').format(version, ', '.join(tag[1:]))
+        if version_is_op_to(tag[0], version, operator.gt):
+            intersect = set(tag[1:]).intersection(platforms)
+            if intersect:
+                return False, ('The higher version {} has already been '
+                               'released for {}').format(
+                                   tag[0], ', '.join(intersect))
+
+    return True, None
+
+
+def update_metadata(metadata, version):
+    """Replace version number in metadata file "manually".
+
+    The ConfigParser would mess up the order of lines.
+    """
+    with open(metadata.option_source('general', 'version'), 'r+b') as fp:
+        rawMetadata = fp.read()
         rawMetadata = re.sub(
             r'^(\s*version\s*=\s*).*', r'\g<1>%s' % version,
             rawMetadata, flags=re.I | re.M
         )
 
-        file.seek(0)
-        file.write(rawMetadata)
-        file.truncate()
+        fp.seek(0)
+        fp.write(rawMetadata)
+        fp.truncate()
 
-    # Read extension name from locale data
-    default_locale_path = os.path.join('_locales', packager.defaultLocale,
+
+def create_build(platform, base_dir, target_path, version, key_file=None):
+    """Create a build for the target platform and version."""
+    if platform == 'edge':
+        import buildtools.packagerEdge as packager
+    else:
+        import buildtools.packagerChrome as packager
+
+    metadata = readMetadata(base_dir, platform)
+    update_metadata(metadata, version)
+
+    build_path = os.path.join(
+        target_path,
+        getDefaultFileName(metadata, version,
+                           get_extension(platform, key_file is not None))
+    )
+
+    packager.createBuild(base_dir, type=platform, outFile=build_path,
+                         releaseBuild=True, keyFile=key_file)
+
+    return build_path
+
+
+def release_commit(base_dir, extension_name, version, platforms):
+    """Create a release commit with a representative message."""
+    subprocess.check_call([
+        'hg', 'commit', '-R', base_dir, '-m',
+        'Noissue - Releasing {} {} for {}'.format(
+            extension_name, version,
+            ', '.join([p.capitalize() for p in platforms]))])
+
+
+def release_tag(base_dir, tag_name, extension_name):
+    """Create a tag, along with a commit message for that tag."""
+    subprocess.check_call([
+        'hg', 'tag', '-R', base_dir, '-f', tag_name,
+        '-m', 'Noissue - Adding release tag for {} {}'.format(
+            extension_name, tag_name)])
+
+
+def run(baseDir, platforms, version, keyFile, downloads_repo):
+    if not can_safely_release(baseDir, downloads_repo):
+        print('Aborting release.')
+        return 1
+
+    target_platforms = sorted(platforms)
+    release_identifier = '-'.join([version] + [p for p in target_platforms])
+
+    release_possible, reason = release_combination_is_possible(
+        version, platforms, baseDir)
+
+    if not release_possible:
+        logging.error(reason)
+        return 2
+
+    downloads = []
+    # Read extension name from first provided platform
+    locale_config = read_locale_config(
+        baseDir, target_platforms[0],
+        readMetadata(baseDir, target_platforms[0]))
+    default_locale_path = os.path.join(locale_config['base_path'],
+                                       locale_config['default_locale'],
                                        'messages.json')
     with open(default_locale_path, 'r') as fp:
-        extensionName = json.load(fp)['name']['message']
+        extension_name = json.load(fp)['name']['message']
 
-    # Now commit the change and tag it
-    subprocess.check_call(['hg', 'commit', '-R', baseDir, '-m', 'Releasing %s %s' % (extensionName, version)])
-    tag_name = version
-    if type == 'edge':
-        tag_name = '{}-{}'.format(tag_name, type)
-    subprocess.check_call(['hg', 'tag', '-R', baseDir, '-f', tag_name])
+    for platform in target_platforms:
+        used_key_file = None
+        if platform == 'chrome':
+            # Currently, only chrome builds are provided by us as signed
+            # packages. Create an unsigned package in base_dir which should be
+            # uploaded to the Chrome Web Store
+            create_build(platform, baseDir, baseDir, version)
+            used_key_file = keyFile
 
-    # Create a release build
-    downloads = []
-    if type == 'chrome':
-        # Create both signed and unsigned Chrome builds (the latter for Chrome Web Store).
-        buildPath = os.path.join(downloadsRepo, getDefaultFileName(metadata, version, 'crx'))
-        packager.createBuild(baseDir, type=type, outFile=buildPath, releaseBuild=True, keyFile=keyFile)
-        downloads.append(buildPath)
+        downloads.append(
+            create_build(platform, baseDir, downloads_repo, version,
+                         used_key_file)
+        )
 
-        buildPathUnsigned = os.path.join(baseDir, getDefaultFileName(metadata, version, 'zip'))
-        packager.createBuild(baseDir, type=type, outFile=buildPathUnsigned, releaseBuild=True, keyFile=None)
-    elif type == 'edge':
-        # We only offer the Edge extension for use through the Windows Store
-        buildPath = os.path.join(downloadsRepo, getDefaultFileName(metadata, version, 'appx'))
-        packager.createBuild(baseDir, type=type, outFile=buildPath, releaseBuild=True)
-        downloads.append(buildPath)
+    # Only create one commit, one tag and one source archive for all
+    # platforms
+    archive_path = os.path.join(
+        downloads_repo,
+        'adblockplus-{}-source.tgz'.format(release_identifier),
+    )
+    create_sourcearchive(baseDir, archive_path)
+    downloads.append(archive_path)
 
-    # Create source archive
-    archivePath = os.path.splitext(buildPath)[0] + '-source.tgz'
-    create_sourcearchive(baseDir, archivePath)
-    downloads.append(archivePath)
+    release_commit(baseDir, extension_name, version, target_platforms)
+    release_tag(baseDir, release_identifier, extension_name)
 
     # Now add the downloads and commit
-    subprocess.check_call(['hg', 'add', '-R', downloadsRepo] + downloads)
-    subprocess.check_call(['hg', 'commit', '-R', downloadsRepo, '-m', 'Releasing %s %s' % (extensionName, version)])
+    subprocess.check_call(['hg', 'add', '-R', downloads_repo] + downloads)
+    release_commit(downloads_repo, extension_name, version, target_platforms)
 
     # Push all changes
     subprocess.check_call(['hg', 'push', '-R', baseDir])
-    subprocess.check_call(['hg', 'push', '-R', downloadsRepo])
+    subprocess.check_call(['hg', 'push', '-R', downloads_repo])