Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/createNightlies.py

Issue 8327353: Crawler backend (Closed)
Patch Set: Created Sept. 27, 2012, 6:22 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This Source Code is subject to the terms of the Mozilla Public License 3 # This Source Code is subject to the terms of the Mozilla Public License
4 # version 2.0 (the "License"). You can obtain a copy of the License at 4 # version 2.0 (the "License"). You can obtain a copy of the License at
5 # http://mozilla.org/MPL/2.0/. 5 # http://mozilla.org/MPL/2.0/.
6 6
7 """ 7 """
8 8
9 Nightly builds generation script 9 Nightly builds generation script
10 ================================ 10 ================================
11 11
12 This script generates nightly builds of extensions, together 12 This script generates nightly builds of extensions, together
13 with changelogs and documentation. 13 with changelogs and documentation.
14 14
15 """ 15 """
16 16
17 import sys, os, os.path, subprocess, ConfigParser, traceback, json, hashlib 17 import sys, os, os.path, subprocess, ConfigParser, traceback, json, hashlib
18 import tempfile, re, shutil, urlparse 18 import tempfile, re, shutil, urlparse, pipes
19 from datetime import datetime 19 from datetime import datetime
20 from xml.dom.minidom import parse as parseXml
20 from sitescripts.utils import get_config, setupStderr, get_template 21 from sitescripts.utils import get_config, setupStderr, get_template
21 from sitescripts.extensions.utils import compareVersions, Configuration 22 from sitescripts.extensions.utils import compareVersions, Configuration
22 import buildtools.packager as packager 23 import buildtools.packager as packager
23 24
24 MAX_BUILDS = 50 25 MAX_BUILDS = 50
25 26
26 27
27 class NightlyBuild(object): 28 class NightlyBuild(object):
28 """ 29 """
29 Performs the build process for an extension, 30 Performs the build process for an extension,
(...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after
110 metadata = packager.readMetadata(self.tempdir) 111 metadata = packager.readMetadata(self.tempdir)
111 self.extensionID = metadata.get("general", "id") 112 self.extensionID = metadata.get("general", "id")
112 self.version = '%s.%s' % (metadata.get("general", "version"), self.revision) 113 self.version = '%s.%s' % (metadata.get("general", "version"), self.revision)
113 self.basename = metadata.get("general", "basename") 114 self.basename = metadata.get("general", "basename")
114 self.compat = [] 115 self.compat = []
115 for key, value in packager.KNOWN_APPS.iteritems(): 116 for key, value in packager.KNOWN_APPS.iteritems():
116 if metadata.has_option('compat', key): 117 if metadata.has_option('compat', key):
117 minVersion, maxVersion = metadata.get('compat', key).split('/') 118 minVersion, maxVersion = metadata.get('compat', key).split('/')
118 self.compat.append({'id': value, 'minVersion': minVersion, 'maxVersion': maxVersion}) 119 self.compat.append({'id': value, 'minVersion': minVersion, 'maxVersion': maxVersion})
119 120
121 def readAndroidMetadata(self):
122 """
123 Read Android-specific metadata from AndroidManifest.xml file.
124 """
125 manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), 'r')
126 manifest = parseXml(manifestFile)
127 manifestFile.close()
128
129 root = manifest.documentElement
130 self.version = root.attributes["android:versionName"].value
131 while self.version.count('.') < 2:
132 self.version += '.0'
133 self.version = '%s.%s' % (self.version, self.revision)
134
135 usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
136 self.minSdkVersion = usesSdk.attributes["android:minSdkVersion"].value
137 self.basename = os.path.basename(self.config.repository)
138
120 def readChromeMetadata(self): 139 def readChromeMetadata(self):
121 """ 140 """
122 Read Chrome-specific metadata from manifest.json file. It will also 141 Read Chrome-specific metadata from manifest.json file. It will also
123 calculate extension ID from the private key. 142 calculate extension ID from the private key.
124 """ 143 """
125 144
126 # Calculate extension ID from public key 145 # Calculate extension ID from public key
127 # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-from- your-private-key-233) 146 # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-from- your-private-key-233)
128 sys.path.append(self.tempdir) 147 sys.path.append(self.tempdir)
129 build = __import__('build') 148 build = __import__('build')
(...skipping 20 matching lines...) Expand all
150 if 'minimum_chrome_version' in manifest: 169 if 'minimum_chrome_version' in manifest:
151 self.compat.append({'id': 'chrome', 'minVersion': manifest['minimum_chrome _version']}) 170 self.compat.append({'id': 'chrome', 'minVersion': manifest['minimum_chrome _version']})
152 171
153 def writeUpdateManifest(self): 172 def writeUpdateManifest(self):
154 """ 173 """
155 Writes update.rdf file for the current build 174 Writes update.rdf file for the current build
156 """ 175 """
157 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 176 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
158 if not os.path.exists(baseDir): 177 if not os.path.exists(baseDir):
159 os.makedirs(baseDir) 178 os.makedirs(baseDir)
160 if self.config.type != 'chrome': 179 if self.config.type == 'chrome':
180 manifestPath = os.path.join(baseDir, "updates.xml")
181 templateName = 'chromeUpdateManifest'
182 elif self.config.type == 'android':
183 manifestPath = os.path.join(baseDir, "updates.xml")
184 templateName = 'androidUpdateManifest'
185 else:
161 manifestPath = os.path.join(baseDir, "update.rdf") 186 manifestPath = os.path.join(baseDir, "update.rdf")
162 templateName = 'geckoUpdateManifest' 187 templateName = 'geckoUpdateManifest'
163 else:
164 manifestPath = os.path.join(baseDir, "updates.xml")
165 templateName = 'chromeUpdateManifest'
166 188
167 template = get_template(get_config().get('extensions', templateName)) 189 template = get_template(get_config().get('extensions', templateName))
168 template.stream({'extensions': [self]}).dump(manifestPath) 190 template.stream({'extensions': [self]}).dump(manifestPath)
169 191
170 def build(self): 192 def build(self):
171 """ 193 """
172 run the build command in the tempdir 194 run the build command in the tempdir
173 """ 195 """
174 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 196 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
175 if not os.path.exists(baseDir): 197 if not os.path.exists(baseDir):
176 os.makedirs(baseDir) 198 os.makedirs(baseDir)
177 outputFile = "%s-%s%s" % (self.basename, self.version, self.config.packageSu ffix) 199 outputFile = "%s-%s%s" % (self.basename, self.version, self.config.packageSu ffix)
178 outputPath = os.path.join(baseDir, outputFile) 200 outputPath = os.path.join(baseDir, outputFile)
179 self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + outputFile + '?update') 201 self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + outputFile + '?update')
180 202
181 if self.config.type != 'chrome': 203 if self.config.type == 'android':
182 packager.createBuild(self.tempdir, outFile=outputPath, buildNum=self.revis ion, keyFile=self.config.keyFile) 204 apkFile = open(outputPath, 'wb')
183 else: 205 try:
206 port = get_config().get('extensions', 'androidBuildPort')
207 except ConfigParser.NoOptionError:
208 port = '22'
209 buildCommand = ['ssh', '-p', port, get_config().get('extensions', 'android BuildHost')]
210 buildCommand += map(pipes.quote, ['/home/android/bin/makedebugbuild.py', ' --revision', self.revision, '--version', self.version, '--stdout'])
211 process = subprocess.Popen(buildCommand, stdout=apkFile, stderr=None)
212 status = process.wait()
213 apkFile.close()
214 if status:
215 # clear broken output if any
216 # exception will be raised later
217 if os.path.exists(outputPath):
218 os.remove(outputPath)
219 elif self.config.type == 'chrome':
184 buildCommand = ['python', os.path.join(self.tempdir, 'build.py'), '-k', se lf.config.keyFile, '-b', self.revision, outputPath] 220 buildCommand = ['python', os.path.join(self.tempdir, 'build.py'), '-k', se lf.config.keyFile, '-b', self.revision, outputPath]
185 if self.config.experimental: 221 if self.config.experimental:
186 buildCommand[-1:0] = ['--experimental'] 222 buildCommand[-1:0] = ['--experimental']
187 subprocess.Popen(buildCommand, stdout=subprocess.PIPE).communicate() 223 subprocess.Popen(buildCommand, stdout=subprocess.PIPE).communicate()
224 else:
225 packager.createBuild(self.tempdir, outFile=outputPath, buildNum=self.revis ion, keyFile=self.config.keyFile)
188 226
189 if not os.path.exists(outputPath): 227 if not os.path.exists(outputPath):
190 raise Exception("Build failed, output file hasn't been created") 228 raise Exception("Build failed, output file hasn't been created")
191 229
192 linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix) 230 linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
193 if hasattr(os, 'symlink'): 231 if hasattr(os, 'symlink'):
194 if os.path.exists(linkPath): 232 if os.path.exists(linkPath):
195 os.remove(linkPath) 233 os.remove(linkPath)
196 os.symlink(outputPath, linkPath) 234 os.symlink(outputPath, linkPath)
197 else: 235 else:
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after
268 try: 306 try:
269 if self.config.type == 'kmeleon': 307 if self.config.type == 'kmeleon':
270 # We cannot build K-Meleon builds, simply list the builds already in 308 # We cannot build K-Meleon builds, simply list the builds already in
271 # the directory. Basename has to be deduced from the repository name. 309 # the directory. Basename has to be deduced from the repository name.
272 self.basename = os.path.basename(self.config.repository) 310 self.basename = os.path.basename(self.config.repository)
273 else: 311 else:
274 # copy the repository into a temporary directory 312 # copy the repository into a temporary directory
275 self.copyRepository() 313 self.copyRepository()
276 314
277 # get meta data from the repository 315 # get meta data from the repository
278 if self.config.type != 'chrome': 316 if self.config.type == 'android':
317 self.readAndroidMetadata()
318 elif self.config.type == 'chrome':
319 self.readChromeMetadata()
320 else:
279 self.readMetadata() 321 self.readMetadata()
280 else:
281 self.readChromeMetadata()
282 322
283 # create development build 323 # create development build
284 self.build() 324 self.build()
285 325
286 # write out changelog 326 # write out changelog
287 self.writeChangelog(self.getChanges()) 327 self.writeChangelog(self.getChanges())
288 328
289 # write update.rdf file 329 # write update.rdf file
290 self.writeUpdateManifest() 330 self.writeUpdateManifest()
291 331
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after
329 except Exception, ex: 369 except Exception, ex:
330 print >>sys.stderr, "The build for %s failed:" % repo 370 print >>sys.stderr, "The build for %s failed:" % repo
331 traceback.print_exc() 371 traceback.print_exc()
332 372
333 file = open(nightlyConfigFile, 'wb') 373 file = open(nightlyConfigFile, 'wb')
334 nightlyConfig.write(file) 374 nightlyConfig.write(file)
335 375
336 376
337 if __name__ == '__main__': 377 if __name__ == '__main__':
338 main() 378 main()
OLDNEW

Powered by Google App Engine
This is Rietveld