Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/createNightlies.py

Issue 29345242: Noissue - Adapt quotes for compliance with our coding style in sitescripts (Closed)
Patch Set: Fixed raw string Created May 30, 2016, 8:47 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-2016 Eyeo GmbH 2 # Copyright (C) 2006-2016 Eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after
90 """ 90 """
91 retrieve changes between the current and previous ("first") revision 91 retrieve changes between the current and previous ("first") revision
92 """ 92 """
93 93
94 command = ['hg', 'log', '-R', self.config.repository, '-r', 'tip:0', 94 command = ['hg', 'log', '-R', self.config.repository, '-r', 'tip:0',
95 '-b', 'default', '-l', '50', '--encoding', 'utf-8', 95 '-b', 'default', '-l', '50', '--encoding', 'utf-8',
96 '--template', '{date|isodate}\\0{author|person}\\0{rev}\\0{de sc}\\0\\0', 96 '--template', '{date|isodate}\\0{author|person}\\0{rev}\\0{de sc}\\0\\0',
97 '--config', 'defaults.log='] 97 '--config', 'defaults.log=']
98 result = subprocess.check_output(command).decode('utf-8') 98 result = subprocess.check_output(command).decode('utf-8')
99 99
100 for change in result.split('\0\0'): 100 for change in result.split('\x00\x00'):
101 if change: 101 if change:
102 date, author, revision, description = change.split('\0') 102 date, author, revision, description = change.split('\x00')
103 yield {'date': date, 'author': author, 'revision': revision, 'de scription': description} 103 yield {'date': date, 'author': author, 'revision': revision, 'de scription': description}
104 104
105 def copyRepository(self): 105 def copyRepository(self):
106 ''' 106 """
107 Create a repository copy in a temporary directory 107 Create a repository copy in a temporary directory
108 ''' 108 """
109 # We cannot use hg archive here due to 109 # We cannot use hg archive here due to
110 # http://bz.selenic.com/show_bug.cgi?id=3747, have to clone properly :-( 110 # http://bz.selenic.com/show_bug.cgi?id=3747, have to clone properly :-(
111 self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName) 111 self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName)
112 command = ['hg', 'clone', '-q', self.config.repository, '-u', 'default', self.tempdir] 112 command = ['hg', 'clone', '-q', self.config.repository, '-u', 'default', self.tempdir]
113 subprocess.check_call(command) 113 subprocess.check_call(command)
114 114
115 # Make sure to process the dependencies file if it is present 115 # Make sure to process the dependencies file if it is present
116 import logging 116 import logging
117 logging.disable(logging.WARNING) 117 logging.disable(logging.WARNING)
118 try: 118 try:
119 from buildtools.ensure_dependencies import resolve_deps 119 from buildtools.ensure_dependencies import resolve_deps
120 resolve_deps(self.tempdir, self_update=False, 120 resolve_deps(self.tempdir, self_update=False,
121 overrideroots={"hg": os.path.abspath( 121 overrideroots={'hg': os.path.abspath(
122 os.path.join(self.config.repository, os.pardir) 122 os.path.join(self.config.repository, os.pardir)
123 )}, 123 )},
124 skipdependencies={"buildtools"}) 124 skipdependencies={'buildtools'})
125 finally: 125 finally:
126 logging.disable(logging.NOTSET) 126 logging.disable(logging.NOTSET)
127 127
128 def writeChangelog(self, changes): 128 def writeChangelog(self, changes):
129 """ 129 """
130 write the changelog file into the cloned repository 130 write the changelog file into the cloned repository
131 """ 131 """
132 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 132 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
133 if not os.path.exists(baseDir): 133 if not os.path.exists(baseDir):
134 os.makedirs(baseDir) 134 os.makedirs(baseDir)
135 changelogFile = "%s-%s.changelog.xhtml" % (self.basename, self.version) 135 changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version)
136 changelogPath = os.path.join(baseDir, changelogFile) 136 changelogPath = os.path.join(baseDir, changelogFile)
137 self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.base name + '/' + changelogFile) 137 self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.base name + '/' + changelogFile)
138 138
139 template = get_template(get_config().get('extensions', 'changelogTemplat e')) 139 template = get_template(get_config().get('extensions', 'changelogTemplat e'))
140 template.stream({'changes': changes}).dump(changelogPath, encoding='utf- 8') 140 template.stream({'changes': changes}).dump(changelogPath, encoding='utf- 8')
141 141
142 linkPath = os.path.join(baseDir, '00latest.changelog.xhtml') 142 linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
143 if hasattr(os, 'symlink'): 143 if hasattr(os, 'symlink'):
144 if os.path.exists(linkPath): 144 if os.path.exists(linkPath):
145 os.remove(linkPath) 145 os.remove(linkPath)
146 os.symlink(os.path.basename(changelogPath), linkPath) 146 os.symlink(os.path.basename(changelogPath), linkPath)
147 else: 147 else:
148 shutil.copyfile(changelogPath, linkPath) 148 shutil.copyfile(changelogPath, linkPath)
149 149
150 def readGeckoMetadata(self): 150 def readGeckoMetadata(self):
151 """ 151 """
152 read Gecko-specific metadata file from a cloned repository 152 read Gecko-specific metadata file from a cloned repository
153 and parse id, version, basename and the compat section 153 and parse id, version, basename and the compat section
154 out of the file 154 out of the file
155 """ 155 """
156 import buildtools.packagerGecko as packager 156 import buildtools.packagerGecko as packager
157 metadata = packager.readMetadata(self.tempdir, self.config.type) 157 metadata = packager.readMetadata(self.tempdir, self.config.type)
158 self.extensionID = metadata.get("general", "id") 158 self.extensionID = metadata.get('general', 'id')
159 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s elf.revision) 159 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s elf.revision)
160 self.basename = metadata.get("general", "basename") 160 self.basename = metadata.get('general', 'basename')
161 self.compat = [] 161 self.compat = []
162 for key, value in packager.KNOWN_APPS.iteritems(): 162 for key, value in packager.KNOWN_APPS.iteritems():
163 if metadata.has_option('compat', key): 163 if metadata.has_option('compat', key):
164 minVersion, maxVersion = metadata.get('compat', key).split('/') 164 minVersion, maxVersion = metadata.get('compat', key).split('/')
165 self.compat.append({'id': value, 'minVersion': minVersion, 'maxV ersion': maxVersion}) 165 self.compat.append({'id': value, 'minVersion': minVersion, 'maxV ersion': maxVersion})
166 166
167 def readAndroidMetadata(self): 167 def readAndroidMetadata(self):
168 """ 168 """
169 Read Android-specific metadata from AndroidManifest.xml file. 169 Read Android-specific metadata from AndroidManifest.xml file.
170 """ 170 """
171 manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), ' r') 171 manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), ' r')
172 manifest = parseXml(manifestFile) 172 manifest = parseXml(manifestFile)
173 manifestFile.close() 173 manifestFile.close()
174 174
175 root = manifest.documentElement 175 root = manifest.documentElement
176 self.version = root.attributes["android:versionName"].value 176 self.version = root.attributes['android:versionName'].value
177 while self.version.count('.') < 2: 177 while self.version.count('.') < 2:
178 self.version += '.0' 178 self.version += '.0'
179 self.version = '%s.%s' % (self.version, self.revision) 179 self.version = '%s.%s' % (self.version, self.revision)
180 180
181 usesSdk = manifest.getElementsByTagName('uses-sdk')[0] 181 usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
182 self.minSdkVersion = usesSdk.attributes["android:minSdkVersion"].value 182 self.minSdkVersion = usesSdk.attributes['android:minSdkVersion'].value
183 self.basename = os.path.basename(self.config.repository) 183 self.basename = os.path.basename(self.config.repository)
184 184
185 def readChromeMetadata(self): 185 def readChromeMetadata(self):
186 """ 186 """
187 Read Chrome-specific metadata from metadata file. This will also 187 Read Chrome-specific metadata from metadata file. This will also
188 calculate extension ID from the private key. 188 calculate extension ID from the private key.
189 """ 189 """
190 190
191 # Calculate extension ID from public key 191 # Calculate extension ID from public key
192 # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-f rom-your-private-key-233) 192 # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-f rom-your-private-key-233)
193 import buildtools.packagerChrome as packager 193 import buildtools.packagerChrome as packager
194 publicKey = packager.getPublicKey(self.config.keyFile) 194 publicKey = packager.getPublicKey(self.config.keyFile)
195 hash = hashlib.sha256() 195 hash = hashlib.sha256()
196 hash.update(publicKey) 196 hash.update(publicKey)
197 self.extensionID = hash.hexdigest()[0:32] 197 self.extensionID = hash.hexdigest()[0:32]
198 self.extensionID = ''.join(map(lambda c: chr(97 + int(c, 16)), self.exte nsionID)) 198 self.extensionID = ''.join(map(lambda c: chr(97 + int(c, 16)), self.exte nsionID))
199 199
200 # Now read metadata file 200 # Now read metadata file
201 metadata = packager.readMetadata(self.tempdir, self.config.type) 201 metadata = packager.readMetadata(self.tempdir, self.config.type)
202 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s elf.revision) 202 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s elf.revision)
203 self.basename = metadata.get("general", "basename") 203 self.basename = metadata.get('general', 'basename')
204 204
205 self.compat = [] 205 self.compat = []
206 if metadata.has_section('compat') and metadata.has_option('compat', 'chr ome'): 206 if metadata.has_section('compat') and metadata.has_option('compat', 'chr ome'):
207 self.compat.append({'id': 'chrome', 'minVersion': metadata.get('comp at', 'chrome')}) 207 self.compat.append({'id': 'chrome', 'minVersion': metadata.get('comp at', 'chrome')})
208 208
209 def readSafariMetadata(self): 209 def readSafariMetadata(self):
210 import buildtools.packagerSafari as packager 210 import buildtools.packagerSafari as packager
211 metadata = packager.readMetadata(self.tempdir, self.config.type) 211 metadata = packager.readMetadata(self.tempdir, self.config.type)
212 certs = packager.get_certificates_and_key(self.config.keyFile)[0] 212 certs = packager.get_certificates_and_key(self.config.keyFile)[0]
213 213
214 self.certificateID = packager.get_developer_identifier(certs) 214 self.certificateID = packager.get_developer_identifier(certs)
215 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s elf.revision) 215 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s elf.revision)
216 self.shortVersion = metadata.get("general", "version") 216 self.shortVersion = metadata.get('general', 'version')
217 self.basename = metadata.get("general", "basename") 217 self.basename = metadata.get('general', 'basename')
218 self.updatedFromGallery = False 218 self.updatedFromGallery = False
219 219
220 def writeUpdateManifest(self): 220 def writeUpdateManifest(self):
221 """ 221 """
222 Writes update.rdf file for the current build 222 Writes update.rdf file for the current build
223 """ 223 """
224 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 224 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
225 if self.config.type == 'safari': 225 if self.config.type == 'safari':
226 manifestPath = os.path.join(baseDir, "updates.plist") 226 manifestPath = os.path.join(baseDir, 'updates.plist')
227 templateName = 'safariUpdateManifest' 227 templateName = 'safariUpdateManifest'
228 elif self.config.type == 'android': 228 elif self.config.type == 'android':
229 manifestPath = os.path.join(baseDir, "updates.xml") 229 manifestPath = os.path.join(baseDir, 'updates.xml')
230 templateName = 'androidUpdateManifest' 230 templateName = 'androidUpdateManifest'
231 else: 231 else:
232 return 232 return
233 233
234 if not os.path.exists(baseDir): 234 if not os.path.exists(baseDir):
235 os.makedirs(baseDir) 235 os.makedirs(baseDir)
236 236
237 # ABP for Android used to have its own update manifest format. We need t o 237 # ABP for Android used to have its own update manifest format. We need t o
238 # generate both that and the new one in the libadblockplus format as lon g 238 # generate both that and the new one in the libadblockplus format as lon g
239 # as a significant amount of users is on an old version. 239 # as a significant amount of users is on an old version.
240 if self.config.type == 'android': 240 if self.config.type == 'android':
241 newManifestPath = os.path.join(baseDir, "update.json") 241 newManifestPath = os.path.join(baseDir, 'update.json')
242 writeAndroidUpdateManifest(newManifestPath, [{ 242 writeAndroidUpdateManifest(newManifestPath, [{
243 'basename': self.basename, 243 'basename': self.basename,
244 'version': self.version, 244 'version': self.version,
245 'updateURL': self.updateURL 245 'updateURL': self.updateURL
246 }]) 246 }])
247 247
248 template = get_template(get_config().get('extensions', templateName)) 248 template = get_template(get_config().get('extensions', templateName))
249 template.stream({'extensions': [self]}).dump(manifestPath) 249 template.stream({'extensions': [self]}).dump(manifestPath)
250 250
251 def writeIEUpdateManifest(self, versions): 251 def writeIEUpdateManifest(self, versions):
252 """ 252 """
253 Writes update.json file for the latest IE build 253 Writes update.json file for the latest IE build
254 """ 254 """
255 if len(versions) == 0: 255 if len(versions) == 0:
256 return 256 return
257 257
258 version = versions[0] 258 version = versions[0]
259 packageName = self.basename + '-' + version + self.config.packageSuffix 259 packageName = self.basename + '-' + version + self.config.packageSuffix
260 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + ' /' + packageName + '?update') 260 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + ' /' + packageName + '?update')
261 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 261 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
262 manifestPath = os.path.join(baseDir, 'update.json') 262 manifestPath = os.path.join(baseDir, 'update.json')
263 263
264 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit e 264 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit e
265 doWrite(manifestPath, [{ 265 doWrite(manifestPath, [{
266 'basename': self.basename, 266 'basename': self.basename,
267 'version': version, 267 'version': version,
268 'updateURL': updateURL 268 'updateURL': updateURL
269 }]) 269 }])
270 270
271 for suffix in ["-x86.msi", "-x64.msi", "-gpo-x86.msi", "-gpo-x64.msi"]: 271 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']:
272 linkPath = os.path.join(baseDir, '00latest%s' % suffix) 272 linkPath = os.path.join(baseDir, '00latest%s' % suffix)
273 outputPath = os.path.join(baseDir, self.basename + '-' + version + s uffix) 273 outputPath = os.path.join(baseDir, self.basename + '-' + version + s uffix)
274 if hasattr(os, 'symlink'): 274 if hasattr(os, 'symlink'):
275 if os.path.exists(linkPath): 275 if os.path.exists(linkPath):
276 os.remove(linkPath) 276 os.remove(linkPath)
277 os.symlink(os.path.basename(outputPath), linkPath) 277 os.symlink(os.path.basename(outputPath), linkPath)
278 else: 278 else:
279 shutil.copyfile(outputPath, linkPath) 279 shutil.copyfile(outputPath, linkPath)
280 280
281 def build(self): 281 def build(self):
282 """ 282 """
283 run the build command in the tempdir 283 run the build command in the tempdir
284 """ 284 """
285 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 285 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
286 if not os.path.exists(baseDir): 286 if not os.path.exists(baseDir):
287 os.makedirs(baseDir) 287 os.makedirs(baseDir)
288 outputFile = "%s-%s%s" % (self.basename, self.version, self.config.packa geSuffix) 288 outputFile = '%s-%s%s' % (self.basename, self.version, self.config.packa geSuffix)
289 self.path = os.path.join(baseDir, outputFile) 289 self.path = os.path.join(baseDir, outputFile)
290 self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basenam e + '/' + outputFile + '?update') 290 self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basenam e + '/' + outputFile + '?update')
291 291
292 if self.config.type == 'android': 292 if self.config.type == 'android':
293 apkFile = open(self.path, 'wb') 293 apkFile = open(self.path, 'wb')
294 294
295 try: 295 try:
296 try: 296 try:
297 port = get_config().get('extensions', 'androidBuildPort') 297 port = get_config().get('extensions', 'androidBuildPort')
298 except ConfigParser.NoOptionError: 298 except ConfigParser.NoOptionError:
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
347 os.remove(changelogPath) 347 os.remove(changelogPath)
348 return versions 348 return versions
349 349
350 def updateIndex(self, versions): 350 def updateIndex(self, versions):
351 """ 351 """
352 Updates index page listing all existing versions 352 Updates index page listing all existing versions
353 """ 353 """
354 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 354 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
355 if not os.path.exists(baseDir): 355 if not os.path.exists(baseDir):
356 os.makedirs(baseDir) 356 os.makedirs(baseDir)
357 outputFile = "index.html" 357 outputFile = 'index.html'
358 outputPath = os.path.join(baseDir, outputFile) 358 outputPath = os.path.join(baseDir, outputFile)
359 359
360 links = [] 360 links = []
361 for version in versions: 361 for version in versions:
362 packageFile = self.basename + '-' + version + self.config.packageSuf fix 362 packageFile = self.basename + '-' + version + self.config.packageSuf fix
363 changelogFile = self.basename + '-' + version + '.changelog.xhtml' 363 changelogFile = self.basename + '-' + version + '.changelog.xhtml'
364 if not os.path.exists(os.path.join(baseDir, packageFile)): 364 if not os.path.exists(os.path.join(baseDir, packageFile)):
365 # Oops 365 # Oops
366 continue 366 continue
367 367
(...skipping 242 matching lines...) Expand 10 before | Expand all | Expand 10 after
610 # build all extensions specified in the configuration file 610 # build all extensions specified in the configuration file
611 # and generate changelogs and documentations for each: 611 # and generate changelogs and documentations for each:
612 data = None 612 data = None
613 for repo in Configuration.getRepositoryConfigurations(nightlyConfig): 613 for repo in Configuration.getRepositoryConfigurations(nightlyConfig):
614 build = None 614 build = None
615 try: 615 try:
616 build = NightlyBuild(repo) 616 build = NightlyBuild(repo)
617 if build.hasChanges(): 617 if build.hasChanges():
618 build.run() 618 build.run()
619 except Exception, ex: 619 except Exception, ex:
620 logging.error("The build for %s failed:", repo) 620 logging.error('The build for %s failed:', repo)
621 logging.exception(ex) 621 logging.exception(ex)
622 622
623 file = open(nightlyConfigFile, 'wb') 623 file = open(nightlyConfigFile, 'wb')
624 nightlyConfig.write(file) 624 nightlyConfig.write(file)
625 625
626 626
627 if __name__ == '__main__': 627 if __name__ == '__main__':
628 main() 628 main()
OLDNEW

Powered by Google App Engine
This is Rietveld