Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: releaseAutomation.py

Issue 29611593: Issue 5996 - Release consistent versions across WebExtensions (Closed) Base URL: https://codereview.adblockplus.org/29609559/
Left Patch Set: Created Nov. 17, 2017, 3:09 p.m.
Right Patch Set: Addressing comments Created Nov. 28, 2017, 2:17 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « packagerChrome.py ('k') | tox.ini » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # This Source Code Form is subject to the terms of the Mozilla Public 1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this 2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
5 from __future__ import print_function 5 from __future__ import print_function
6 6
7 import os 7 import os
8 import operator
8 import re 9 import re
9 import codecs 10 import codecs
10 import logging 11 import logging
11 import subprocess 12 import subprocess
12 import sys 13 import sys
13 import tarfile 14 import tarfile
14 import json 15 import json
15 16
16 from packager import readMetadata, getDefaultFileName, get_extension 17 from packager import readMetadata, getDefaultFileName, get_extension
17 from localeTools import read_locale_config 18 from localeTools import read_locale_config
18 19
19 WEBEXT_PLATFORMS = {'chrome', 'gecko'} 20 SOURCE_ARCHIVE = 'adblockplus-{}-source.tgz'
20 WEBEXT_SOURCE_ARCHIVE = 'adblockpluswebext-{}-source.tgz'
21 21
22 22
23 def get_dependencies(prefix, repos): 23 def get_dependencies(prefix, repos):
24 from ensure_dependencies import read_deps, safe_join 24 from ensure_dependencies import read_deps, safe_join
25 repo = repos[prefix] 25 repo = repos[prefix]
26 deps = read_deps(repo) 26 deps = read_deps(repo)
27 if deps: 27 if deps:
28 for subpath in deps: 28 for subpath in deps:
29 if subpath.startswith('_'): 29 if subpath.startswith('_'):
30 continue 30 continue
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after
112 """Run repository-checks in order to bail out early if necessary""" 112 """Run repository-checks in order to bail out early if necessary"""
113 if repo_has_uncommitted(): 113 if repo_has_uncommitted():
114 return False 114 return False
115 if repo_has_incoming(*repo_paths): 115 if repo_has_incoming(*repo_paths):
116 return False 116 return False
117 if repo_has_outgoing(): 117 if repo_has_outgoing():
118 return continue_with_outgoing() 118 return continue_with_outgoing()
119 return True 119 return True
120 120
121 121
122 def previously_released(version, downloads_repo): 122 def compare_versions(a, b):
123 regex = re.compile(r'^' + WEBEXT_SOURCE_ARCHIVE.format('(.*)'), 123 """Compare two version numbers."""
124 re.MULTILINE) 124 a_digits = [int(v) for v in a.split('.')]
125 125 b_digits = [int(v) for v in b.split('.')]
126 existing = regex.findall(os.linesep.join(os.listdir(downloads_repo))) 126
127 return version in existing 127 def safe_get(items, index):
128 return items[index] if index < len(items) else 0
129
130 for i in range(max(len(a_digits), len(b_digits))):
131 result = safe_get(a_digits, i) - safe_get(b_digits, i)
132 if result != 0:
133 return result
134 return 0
135
136
137 def release_combination_is_possible(version, platforms, base_dir):
138 """Determine whether a release for the given parameters is possible.
139
140 Examine existing tags in order to find either higher or matching versions.
141 The release is impossible if a) a higher version for a requested platform
142 exists, or if b) a matching version exists and the requested set of
143 platforms differs from what was already released.
144 """
145 def higher_tag_version(tag, version, platforms):
146 return (compare_versions(tag[0], version) > 0 and
147 set(tag[1:]).intersection(platforms))
148
149 def incomplete_platforms_for_version(tag, version, platforms):
150 intersection = set(tag[1:]).intersection(platforms)
151 return (compare_versions(tag[0], version) == 0 and
152 intersection and set(platforms) != set(tag[1:]))
153
154 # only consider tags of the form "1.2[.x ...]-platform[-platform ...]
155 platform_tags = re.compile(r'^(\d+(?:(?:\.\d+)*)(?:-\w+)+)$', re.MULTILINE)
156 tags = [
157 c for c in [
158 t.split('-') for t in
159 platform_tags.findall(subprocess.check_output(
160 ['hg', 'tags', '-R', base_dir, '-q']))
161 ] if compare_versions(c[0], version) >= 0
162 ]
163
164 for tag in tags:
165 if higher_tag_version(tag, version, platforms):
166 reason = ('The higher version {} has already been released for '
167 'the platforms {}.').format(tag[0], ', '.join(platforms))
168 return False, reason, None
169
170 if incomplete_platforms_for_version(tag, version, platforms):
171 reason = ('You have to re-release version {} for exactly all '
172 'of: {}').format(version, ', '.join(tag[1:]))
173 return False, reason, None
174
175 return (True, None,
176 any(compare_versions(tag[0], version) == 0 for tag in tags))
128 177
129 178
130 def update_metadata(metadata, version): 179 def update_metadata(metadata, version):
131 # Replace version number in metadata file "manually", ConfigParser will 180 """Replace version number in metadata file "manually".
132 # mess up the order of lines. 181
182 The ConfigParser would mess up the order of lines.
183 """
133 with open(metadata.option_source('general', 'version'), 'r+b') as fp: 184 with open(metadata.option_source('general', 'version'), 'r+b') as fp:
134 rawMetadata = fp.read() 185 rawMetadata = fp.read()
135 rawMetadata = re.sub( 186 rawMetadata = re.sub(
136 r'^(\s*version\s*=\s*).*', r'\g<1>%s' % version, 187 r'^(\s*version\s*=\s*).*', r'\g<1>%s' % version,
137 rawMetadata, flags=re.I | re.M 188 rawMetadata, flags=re.I | re.M
138 ) 189 )
139 190
140 fp.seek(0) 191 fp.seek(0)
141 fp.write(rawMetadata) 192 fp.write(rawMetadata)
142 fp.truncate() 193 fp.truncate()
143 194
144 195
145 def create_build(platform, base_dir, target_path, version, key_file=None): 196 def create_build(platform, base_dir, target_path, version, key_file=None):
146 if platform in WEBEXT_PLATFORMS: 197 """Create a build for the target platform and version."""
198 if platform == 'edge':
199 import buildtools.packagerEdge as packager
200 else:
147 import buildtools.packagerChrome as packager 201 import buildtools.packagerChrome as packager
148 else: 202
149 import buildtools.packagerEdge as packager
150 metadata = readMetadata(base_dir, platform) 203 metadata = readMetadata(base_dir, platform)
151 update_metadata(metadata, version) 204 update_metadata(metadata, version)
152 205
153 build_path = os.path.join( 206 build_path = os.path.join(
154 target_path, 207 target_path,
155 getDefaultFileName(metadata, version, 208 getDefaultFileName(metadata, version,
156 get_extension(platform, key_file is not None)) 209 get_extension(platform, key_file is not None))
157 ) 210 )
158 211
159 packager.createBuild(base_dir, type=platform, outFile=build_path, 212 packager.createBuild(base_dir, type=platform, outFile=build_path,
160 releaseBuild=True, keyFile=key_file) 213 releaseBuild=True, keyFile=key_file)
161 214
162 return build_path 215 return build_path
163 216
164 217
165 def release_commit(base_dir, extension_name, version, platforms): 218 def release_commit(base_dir, extension_name, version, platforms):
219 """Create a release commit with a representative message."""
220 subprocess.check_output([
221 'hg', 'commit', '-R', base_dir, '-m',
222 'Noissue - Releasing {} {} for {}'.format(
223 extension_name, version,
224 ', '.join([p.capitalize() for p in platforms]))],
225 stderr=subprocess.STDOUT)
226
227
228 def release_tag(base_dir, tag_name, extension_name):
229 """Create a tag, along with a commit message for that tag."""
166 subprocess.check_call([ 230 subprocess.check_call([
167 'hg', 'commit', '-R', base_dir, '-m', 231 'hg', 'tag', '-R', base_dir, '-f', tag_name,
168 'Releasing {} {} for {}'.format(extension_name, version, 232 '-m', 'Noissue - Adding release tag for {} {}'.format(
169 ', '.join(platforms))]) 233 extension_name, tag_name)])
170
171
172 def release_tag(base_dir, tag_name):
173 subprocess.check_call(['hg', 'tag', '-R', base_dir, '-f', tag_name])
174 234
175 235
176 def run(baseDir, platforms, version, keyFile, downloads_repo): 236 def run(baseDir, platforms, version, keyFile, downloads_repo):
177 if not can_safely_release(baseDir, downloads_repo): 237 if not can_safely_release(baseDir, downloads_repo):
178 print('Aborting release.') 238 print('Aborting release.')
179 return 1 239 return 1
180 240
181 if (WEBEXT_PLATFORMS.intersection(platforms) and 241 target_platforms = sorted(platforms)
182 previously_released(version, downloads_repo)): 242 release_identifier = '-'.join([version] + [p for p in target_platforms])
183 logging.error(('A release of version {} for WebExtension platforms ' 243
184 'was already done. Aborting.').format(version)) 244 release_possible, reason, re_release = release_combination_is_possible(
245 version, platforms, baseDir)
246
247 if not release_possible:
248 logging.error(reason)
185 return 2 249 return 2
186 250
187 downloads = [] 251 downloads = []
188 # Read extension name from first provided platform 252 # Read extension name from first provided platform
189 locale_config = read_locale_config(baseDir, platforms[0], 253 locale_config = read_locale_config(
190 readMetadata(baseDir, platforms[0])) 254 baseDir, target_platforms[0],
255 readMetadata(baseDir, target_platforms[0]))
191 default_locale_path = os.path.join(locale_config['base_path'], 256 default_locale_path = os.path.join(locale_config['base_path'],
192 locale_config['default_locale'], 257 locale_config['default_locale'],
193 'messages.json') 258 'messages.json')
194 with open(default_locale_path, 'r') as fp: 259 with open(default_locale_path, 'r') as fp:
195 extension_name = json.load(fp)['name']['message'] 260 extension_name = json.load(fp)['name']['message']
196 261
197 webext_builds = WEBEXT_PLATFORMS.intersection(platforms) 262 for platform in target_platforms:
198
199 for platform in webext_builds:
200 used_key_file = None 263 used_key_file = None
201 if platform == 'chrome': 264 if platform == 'chrome':
202 # Currently, only chrome builds are provided by us as signed 265 # Currently, only chrome builds are provided by us as signed
203 # packages. Create an unsigned package in base_dir which should be 266 # packages. Create an unsigned package in base_dir which should be
204 # uploaded to the Chrome Web Store 267 # uploaded to the Chrome Web Store
205 create_build(platform, baseDir, baseDir, version) 268 create_build(platform, baseDir, baseDir, version)
206 used_key_file = keyFile 269 used_key_file = keyFile
207 270
208 downloads.append( 271 downloads.append(
209 create_build(platform, baseDir, downloads_repo, version, 272 create_build(platform, baseDir, downloads_repo, version,
210 used_key_file) 273 used_key_file)
211 ) 274 )
212 275
213 if webext_builds: 276 # Only create one commit, one tag and one source archive for all
214 # Only create one commit, one tag and one source archive for all 277 # platforms
215 # WebExtension platforms 278 archive_path = os.path.join(
216 archive_path = os.path.join(downloads_repo, 279 downloads_repo,
217 WEBEXT_SOURCE_ARCHIVE.format(version)) 280 'adblockplus-{}-source.tgz'.format(release_identifier),
218 create_sourcearchive(baseDir, archive_path) 281 )
219 downloads.append(archive_path) 282 create_sourcearchive(baseDir, archive_path)
220 283 downloads.append(archive_path)
221 release_commit(baseDir, extension_name, version, webext_builds) 284 try:
222 release_tag(baseDir, version) 285 release_commit(baseDir, extension_name, version, target_platforms)
223 286 except subprocess.CalledProcessError as e:
224 for platform in set(platforms).difference(WEBEXT_PLATFORMS): 287 if not (re_release and 'nothing changed' in e.output):
225 # Create build for platforms not yet using WebExtension 288 raise
226 downloads.append( 289
227 create_build(platform, baseDir, downloads_repo, version, 290 release_tag(baseDir, release_identifier, extension_name)
228 keyFile)
229 )
230
231 archivePath = os.path.splitext(downloads[-1])[0] + '-source.tgz'
232 create_sourcearchive(baseDir, archivePath)
233 downloads.append(archivePath)
234
235 release_commit(baseDir, extension_name, version, {platform})
236 release_tag(baseDir, '{}-{}'.format(version, platform))
237 291
238 # Now add the downloads and commit 292 # Now add the downloads and commit
239 subprocess.check_call(['hg', 'add', '-R', downloads_repo] + downloads) 293 subprocess.check_call(['hg', 'add', '-R', downloads_repo] + downloads)
240 subprocess.check_call([ 294 release_commit(downloads_repo, extension_name, version, target_platforms)
241 'hg', 'commit', '-R', downloads_repo, '-m',
242 'Releasing {} {} for {}'.format(extension_name, version,
243 ', '.join(platforms))])
244 295
245 # Push all changes 296 # Push all changes
246 subprocess.check_call(['hg', 'push', '-R', baseDir]) 297 subprocess.check_call(['hg', 'push', '-R', baseDir])
247 subprocess.check_call(['hg', 'push', '-R', downloads_repo]) 298 subprocess.check_call(['hg', 'push', '-R', downloads_repo])
LEFTRIGHT

Powered by Google App Engine
This is Rietveld