Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/createNightlies.py

Issue 29345508: Issue 4098 - Get rid of special build setup for development builds (Closed)
Patch Set: Added spiderMonkeyPath config setting Created June 3, 2016, 2:11 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « .sitescripts.example ('k') | sitescripts/extensions/utils.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-2016 Eyeo GmbH 2 # Copyright (C) 2006-2016 Eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after
58 generating changelogs and documentation. 58 generating changelogs and documentation.
59 """ 59 """
60 60
61 def __init__(self, config): 61 def __init__(self, config):
62 """ 62 """
63 Creates a NightlyBuild instance; we are simply 63 Creates a NightlyBuild instance; we are simply
64 recording the configuration settings here. 64 recording the configuration settings here.
65 """ 65 """
66 self.config = config 66 self.config = config
67 self.revision = self.getCurrentRevision() 67 self.revision = self.getCurrentRevision()
68 if self.config.type == 'gecko':
69 self.revision += '-beta'
70 try: 68 try:
71 self.previousRevision = config.latestRevision 69 self.previousRevision = config.latestRevision
72 except: 70 except:
73 self.previousRevision = '0' 71 self.previousRevision = '0'
72 self.buildNum = None
74 self.tempdir = None 73 self.tempdir = None
75 self.outputFilename = None 74 self.outputFilename = None
76 self.changelogFilename = None 75 self.changelogFilename = None
77 76
78 def hasChanges(self): 77 def hasChanges(self):
79 return self.revision != self.previousRevision 78 return self.revision != self.previousRevision
80 79
81 def getCurrentRevision(self): 80 def getCurrentRevision(self):
82 """ 81 """
83 retrieves the current revision number from the repository 82 retrieves the current revision ID from the repository
84 """ 83 """
85 command = ['hg', 'log', '-R', self.config.repository, '-r', 'default', 84 command = [
86 '--template', '{rev}', '--config', 'defaults.log='] 85 'hg', 'id', '-i', '-r', 'default', '--config', 'defaults.id=',
87 return subprocess.check_output(command) 86 self.config.repository
87 ]
88 return subprocess.check_output(command).strip()
89
90 def getCurrentBuild(self):
91 """
92 calculates the (typically numerical) build ID for the current build
93 """
94 command = ['hg', 'id', '-n', '--config', 'defaults.id=', self.tempdir]
95 build = subprocess.check_output(command).strip()
96 if self.config.type == 'gecko':
97 build += '-beta'
98 return build
88 99
89 def getChanges(self): 100 def getChanges(self):
90 """ 101 """
91 retrieve changes between the current and previous ("first") revision 102 retrieve changes between the current and previous ("first") revision
92 """ 103 """
93 104
94 command = ['hg', 'log', '-R', self.config.repository, '-r', 'tip:0', 105 command = ['hg', 'log', '-R', self.tempdir, '-r', 'tip:0',
95 '-b', 'default', '-l', '50', '--encoding', 'utf-8', 106 '-b', 'default', '-l', '50', '--encoding', 'utf-8',
96 '--template', '{date|isodate}\\0{author|person}\\0{rev}\\0{de sc}\\0\\0', 107 '--template', '{date|isodate}\\0{author|person}\\0{rev}\\0{de sc}\\0\\0',
97 '--config', 'defaults.log='] 108 '--config', 'defaults.log=']
98 result = subprocess.check_output(command).decode('utf-8') 109 result = subprocess.check_output(command).decode('utf-8')
99 110
100 for change in result.split('\x00\x00'): 111 for change in result.split('\x00\x00'):
101 if change: 112 if change:
102 date, author, revision, description = change.split('\x00') 113 date, author, revision, description = change.split('\x00')
103 yield {'date': date, 'author': author, 'revision': revision, 'de scription': description} 114 yield {'date': date, 'author': author, 'revision': revision, 'de scription': description}
104 115
105 def copyRepository(self): 116 def copyRepository(self):
106 """ 117 """
107 Create a repository copy in a temporary directory 118 Create a repository copy in a temporary directory
108 """ 119 """
109 # We cannot use hg archive here due to
110 # http://bz.selenic.com/show_bug.cgi?id=3747, have to clone properly :-(
111 self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName) 120 self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName)
112 command = ['hg', 'clone', '-q', self.config.repository, '-u', 'default', self.tempdir] 121 command = ['hg', 'clone', '-q', self.config.repository, '-u', 'default', self.tempdir]
113 subprocess.check_call(command) 122 subprocess.check_call(command)
114 123
115 # Make sure to process the dependencies file if it is present 124 # Make sure to run ensure_dependencies.py if present
116 import logging 125 command = [
117 logging.disable(logging.WARNING) 126 sys.executable,
118 try: 127 os.path.join(self.tempdir, 'ensure_dependencies.py'), '-q'
119 from buildtools.ensure_dependencies import resolve_deps 128 ]
120 resolve_deps(self.tempdir, self_update=False, 129 if os.path.isfile(command[0]):
121 overrideroots={'hg': os.path.abspath( 130 subprocess.check_call(command)
122 os.path.join(self.config.repository, os.pardir)
123 )},
124 skipdependencies={'buildtools'})
125 finally:
126 logging.disable(logging.NOTSET)
127 131
128 def writeChangelog(self, changes): 132 def writeChangelog(self, changes):
129 """ 133 """
130 write the changelog file into the cloned repository 134 write the changelog file into the cloned repository
131 """ 135 """
132 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 136 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
133 if not os.path.exists(baseDir): 137 if not os.path.exists(baseDir):
134 os.makedirs(baseDir) 138 os.makedirs(baseDir)
135 changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version) 139 changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version)
136 changelogPath = os.path.join(baseDir, changelogFile) 140 changelogPath = os.path.join(baseDir, changelogFile)
(...skipping 12 matching lines...) Expand all
149 153
150 def readGeckoMetadata(self): 154 def readGeckoMetadata(self):
151 """ 155 """
152 read Gecko-specific metadata file from a cloned repository 156 read Gecko-specific metadata file from a cloned repository
153 and parse id, version, basename and the compat section 157 and parse id, version, basename and the compat section
154 out of the file 158 out of the file
155 """ 159 """
156 import buildtools.packagerGecko as packager 160 import buildtools.packagerGecko as packager
157 metadata = packager.readMetadata(self.tempdir, self.config.type) 161 metadata = packager.readMetadata(self.tempdir, self.config.type)
158 self.extensionID = metadata.get('general', 'id') 162 self.extensionID = metadata.get('general', 'id')
159 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s elf.revision) 163 self.version = packager.getBuildVersion(self.tempdir, metadata, False,
164 self.buildNum)
160 self.basename = metadata.get('general', 'basename') 165 self.basename = metadata.get('general', 'basename')
161 self.compat = [] 166 self.compat = []
162 for key, value in packager.KNOWN_APPS.iteritems(): 167 for key, value in packager.KNOWN_APPS.iteritems():
163 if metadata.has_option('compat', key): 168 if metadata.has_option('compat', key):
164 minVersion, maxVersion = metadata.get('compat', key).split('/') 169 minVersion, maxVersion = metadata.get('compat', key).split('/')
165 self.compat.append({'id': value, 'minVersion': minVersion, 'maxV ersion': maxVersion}) 170 self.compat.append({'id': value, 'minVersion': minVersion, 'maxV ersion': maxVersion})
166 171
167 def readAndroidMetadata(self): 172 def readAndroidMetadata(self):
168 """ 173 """
169 Read Android-specific metadata from AndroidManifest.xml file. 174 Read Android-specific metadata from AndroidManifest.xml file.
170 """ 175 """
171 manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), ' r') 176 manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), ' r')
172 manifest = parseXml(manifestFile) 177 manifest = parseXml(manifestFile)
173 manifestFile.close() 178 manifestFile.close()
174 179
175 root = manifest.documentElement 180 root = manifest.documentElement
176 self.version = root.attributes['android:versionName'].value 181 self.version = root.attributes['android:versionName'].value
177 while self.version.count('.') < 2: 182 while self.version.count('.') < 2:
178 self.version += '.0' 183 self.version += '.0'
179 self.version = '%s.%s' % (self.version, self.revision) 184 self.version = '%s.%s' % (self.version, self.buildNum)
180 185
181 usesSdk = manifest.getElementsByTagName('uses-sdk')[0] 186 usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
182 self.minSdkVersion = usesSdk.attributes['android:minSdkVersion'].value 187 self.minSdkVersion = usesSdk.attributes['android:minSdkVersion'].value
183 self.basename = os.path.basename(self.config.repository) 188 self.basename = os.path.basename(self.config.repository)
184 189
185 def readChromeMetadata(self): 190 def readChromeMetadata(self):
186 """ 191 """
187 Read Chrome-specific metadata from metadata file. This will also 192 Read Chrome-specific metadata from metadata file. This will also
188 calculate extension ID from the private key. 193 calculate extension ID from the private key.
189 """ 194 """
190 195
191 # Calculate extension ID from public key 196 # Calculate extension ID from public key
192 # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-f rom-your-private-key-233) 197 # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-f rom-your-private-key-233)
193 import buildtools.packagerChrome as packager 198 import buildtools.packagerChrome as packager
194 publicKey = packager.getPublicKey(self.config.keyFile) 199 publicKey = packager.getPublicKey(self.config.keyFile)
195 hash = hashlib.sha256() 200 hash = hashlib.sha256()
196 hash.update(publicKey) 201 hash.update(publicKey)
197 self.extensionID = hash.hexdigest()[0:32] 202 self.extensionID = hash.hexdigest()[0:32]
198 self.extensionID = ''.join(map(lambda c: chr(97 + int(c, 16)), self.exte nsionID)) 203 self.extensionID = ''.join(map(lambda c: chr(97 + int(c, 16)), self.exte nsionID))
199 204
200 # Now read metadata file 205 # Now read metadata file
201 metadata = packager.readMetadata(self.tempdir, self.config.type) 206 metadata = packager.readMetadata(self.tempdir, self.config.type)
202 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s elf.revision) 207 self.version = packager.getBuildVersion(self.tempdir, metadata, False,
208 self.buildNum)
203 self.basename = metadata.get('general', 'basename') 209 self.basename = metadata.get('general', 'basename')
204 210
205 self.compat = [] 211 self.compat = []
206 if metadata.has_section('compat') and metadata.has_option('compat', 'chr ome'): 212 if metadata.has_section('compat') and metadata.has_option('compat', 'chr ome'):
207 self.compat.append({'id': 'chrome', 'minVersion': metadata.get('comp at', 'chrome')}) 213 self.compat.append({'id': 'chrome', 'minVersion': metadata.get('comp at', 'chrome')})
208 214
209 def readSafariMetadata(self): 215 def readSafariMetadata(self):
210 import buildtools.packagerSafari as packager 216 import buildtools.packagerSafari as packager
211 metadata = packager.readMetadata(self.tempdir, self.config.type) 217 metadata = packager.readMetadata(self.tempdir, self.config.type)
212 certs = packager.get_certificates_and_key(self.config.keyFile)[0] 218 certs = packager.get_certificates_and_key(self.config.keyFile)[0]
213 219
214 self.certificateID = packager.get_developer_identifier(certs) 220 self.certificateID = packager.get_developer_identifier(certs)
215 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s elf.revision) 221 self.version = packager.getBuildVersion(self.tempdir, metadata, False,
222 self.buildNum)
216 self.shortVersion = metadata.get('general', 'version') 223 self.shortVersion = metadata.get('general', 'version')
217 self.basename = metadata.get('general', 'basename') 224 self.basename = metadata.get('general', 'basename')
218 self.updatedFromGallery = False 225 self.updatedFromGallery = False
219 226
220 def writeUpdateManifest(self): 227 def writeUpdateManifest(self):
221 """ 228 """
222 Writes update.rdf file for the current build 229 Writes update.rdf file for the current build
223 """ 230 """
224 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) 231 baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
225 if self.config.type == 'safari': 232 if self.config.type == 'safari':
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after
291 298
292 if self.config.type == 'android': 299 if self.config.type == 'android':
293 apkFile = open(self.path, 'wb') 300 apkFile = open(self.path, 'wb')
294 301
295 try: 302 try:
296 try: 303 try:
297 port = get_config().get('extensions', 'androidBuildPort') 304 port = get_config().get('extensions', 'androidBuildPort')
298 except ConfigParser.NoOptionError: 305 except ConfigParser.NoOptionError:
299 port = '22' 306 port = '22'
300 buildCommand = ['ssh', '-p', port, get_config().get('extensions' , 'androidBuildHost')] 307 buildCommand = ['ssh', '-p', port, get_config().get('extensions' , 'androidBuildHost')]
301 buildCommand.extend(map(pipes.quote, ['/home/android/bin/makedeb ugbuild.py', '--revision', self.revision, '--version', self.version, '--stdout'] )) 308 buildCommand.extend(map(pipes.quote, [
309 '/home/android/bin/makedebugbuild.py', '--revision',
310 self.buildNum, '--version', self.version, '--stdout'
311 ]))
302 subprocess.check_call(buildCommand, stdout=apkFile, close_fds=Tr ue) 312 subprocess.check_call(buildCommand, stdout=apkFile, close_fds=Tr ue)
303 except: 313 except:
304 # clear broken output if any 314 # clear broken output if any
305 if os.path.exists(self.path): 315 if os.path.exists(self.path):
306 os.remove(self.path) 316 os.remove(self.path)
307 raise 317 raise
308 elif self.config.type == 'chrome':
309 import buildtools.packagerChrome as packager
310 packager.createBuild(self.tempdir, type=self.config.type, outFile=se lf.path, buildNum=self.revision, keyFile=self.config.keyFile)
311 elif self.config.type == 'safari':
312 import buildtools.packagerSafari as packager
313 packager.createBuild(self.tempdir, type=self.config.type, outFile=se lf.path, buildNum=self.revision, keyFile=self.config.keyFile)
314 else: 318 else:
315 import buildtools.packagerGecko as packager 319 env = os.environ
316 packager.createBuild(self.tempdir, outFile=self.path, buildNum=self. revision, keyFile=self.config.keyFile) 320 spiderMonkeyBinary = self.config.spiderMonkeyBinary
321 if spiderMonkeyBinary:
322 env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)
323
324 buildCommand = [
325 os.path.join(self.tempdir, 'build.py'), '-t', self.config.type,
326 'build', '-b', self.buildNum, '-k', self.config.keyFile,
327 self.path
328 ]
329 subprocess.check_call(buildCommand, env=env)
317 330
318 if not os.path.exists(self.path): 331 if not os.path.exists(self.path):
319 raise Exception("Build failed, output file hasn't been created") 332 raise Exception("Build failed, output file hasn't been created")
320 333
321 linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffi x) 334 linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffi x)
322 if hasattr(os, 'symlink'): 335 if hasattr(os, 'symlink'):
323 if os.path.exists(linkPath): 336 if os.path.exists(linkPath):
324 os.remove(linkPath) 337 os.remove(linkPath)
325 os.symlink(os.path.basename(self.path), linkPath) 338 os.symlink(os.path.basename(self.path), linkPath)
326 else: 339 else:
(...skipping 221 matching lines...) Expand 10 before | Expand all | Expand 10 after
548 Run the nightly build process for one extension 561 Run the nightly build process for one extension
549 """ 562 """
550 try: 563 try:
551 if self.config.type == 'ie': 564 if self.config.type == 'ie':
552 # We cannot build IE builds, simply list the builds already in 565 # We cannot build IE builds, simply list the builds already in
553 # the directory. Basename has to be deduced from the repository name. 566 # the directory. Basename has to be deduced from the repository name.
554 self.basename = os.path.basename(self.config.repository) 567 self.basename = os.path.basename(self.config.repository)
555 else: 568 else:
556 # copy the repository into a temporary directory 569 # copy the repository into a temporary directory
557 self.copyRepository() 570 self.copyRepository()
571 self.buildNum = self.getCurrentBuild()
558 572
559 # get meta data from the repository 573 # get meta data from the repository
560 if self.config.type == 'android': 574 if self.config.type == 'android':
561 self.readAndroidMetadata() 575 self.readAndroidMetadata()
562 elif self.config.type == 'chrome': 576 elif self.config.type == 'chrome':
563 self.readChromeMetadata() 577 self.readChromeMetadata()
564 elif self.config.type == 'safari': 578 elif self.config.type == 'safari':
565 self.readSafariMetadata() 579 self.readSafariMetadata()
566 else: 580 else:
567 self.readGeckoMetadata() 581 self.readGeckoMetadata()
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after
619 except Exception, ex: 633 except Exception, ex:
620 logging.error('The build for %s failed:', repo) 634 logging.error('The build for %s failed:', repo)
621 logging.exception(ex) 635 logging.exception(ex)
622 636
623 file = open(nightlyConfigFile, 'wb') 637 file = open(nightlyConfigFile, 'wb')
624 nightlyConfig.write(file) 638 nightlyConfig.write(file)
625 639
626 640
627 if __name__ == '__main__': 641 if __name__ == '__main__':
628 main() 642 main()
OLDNEW
« no previous file with comments | « .sitescripts.example ('k') | sitescripts/extensions/utils.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld