OLD | NEW |
1 # This file is part of the Adblock Plus web scripts, | 1 # This file is part of the Adblock Plus web scripts, |
2 # Copyright (C) 2006-2016 Eyeo GmbH | 2 # Copyright (C) 2006-2016 Eyeo GmbH |
3 # | 3 # |
4 # Adblock Plus is free software: you can redistribute it and/or modify | 4 # Adblock Plus is free software: you can redistribute it and/or modify |
5 # it under the terms of the GNU General Public License version 3 as | 5 # it under the terms of the GNU General Public License version 3 as |
6 # published by the Free Software Foundation. | 6 # published by the Free Software Foundation. |
7 # | 7 # |
8 # Adblock Plus is distributed in the hope that it will be useful, | 8 # Adblock Plus is distributed in the hope that it will be useful, |
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
90 """ | 90 """ |
91 retrieve changes between the current and previous ("first") revision | 91 retrieve changes between the current and previous ("first") revision |
92 """ | 92 """ |
93 | 93 |
94 command = ['hg', 'log', '-R', self.config.repository, '-r', 'tip:0', | 94 command = ['hg', 'log', '-R', self.config.repository, '-r', 'tip:0', |
95 '-b', 'default', '-l', '50', '--encoding', 'utf-8', | 95 '-b', 'default', '-l', '50', '--encoding', 'utf-8', |
96 '--template', '{date|isodate}\\0{author|person}\\0{rev}\\0{de
sc}\\0\\0', | 96 '--template', '{date|isodate}\\0{author|person}\\0{rev}\\0{de
sc}\\0\\0', |
97 '--config', 'defaults.log='] | 97 '--config', 'defaults.log='] |
98 result = subprocess.check_output(command).decode('utf-8') | 98 result = subprocess.check_output(command).decode('utf-8') |
99 | 99 |
100 for change in result.split('\0\0'): | 100 for change in result.split('\x00\x00'): |
101 if change: | 101 if change: |
102 date, author, revision, description = change.split('\0') | 102 date, author, revision, description = change.split('\x00') |
103 yield {'date': date, 'author': author, 'revision': revision, 'de
scription': description} | 103 yield {'date': date, 'author': author, 'revision': revision, 'de
scription': description} |
104 | 104 |
105 def copyRepository(self): | 105 def copyRepository(self): |
106 ''' | 106 """ |
107 Create a repository copy in a temporary directory | 107 Create a repository copy in a temporary directory |
108 ''' | 108 """ |
109 # We cannot use hg archive here due to | 109 # We cannot use hg archive here due to |
110 # http://bz.selenic.com/show_bug.cgi?id=3747, have to clone properly :-( | 110 # http://bz.selenic.com/show_bug.cgi?id=3747, have to clone properly :-( |
111 self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName) | 111 self.tempdir = tempfile.mkdtemp(prefix=self.config.repositoryName) |
112 command = ['hg', 'clone', '-q', self.config.repository, '-u', 'default',
self.tempdir] | 112 command = ['hg', 'clone', '-q', self.config.repository, '-u', 'default',
self.tempdir] |
113 subprocess.check_call(command) | 113 subprocess.check_call(command) |
114 | 114 |
115 # Make sure to process the dependencies file if it is present | 115 # Make sure to process the dependencies file if it is present |
116 import logging | 116 import logging |
117 logging.disable(logging.WARNING) | 117 logging.disable(logging.WARNING) |
118 try: | 118 try: |
119 from buildtools.ensure_dependencies import resolve_deps | 119 from buildtools.ensure_dependencies import resolve_deps |
120 resolve_deps(self.tempdir, self_update=False, | 120 resolve_deps(self.tempdir, self_update=False, |
121 overrideroots={"hg": os.path.abspath( | 121 overrideroots={'hg': os.path.abspath( |
122 os.path.join(self.config.repository, os.pardir) | 122 os.path.join(self.config.repository, os.pardir) |
123 )}, | 123 )}, |
124 skipdependencies={"buildtools"}) | 124 skipdependencies={'buildtools'}) |
125 finally: | 125 finally: |
126 logging.disable(logging.NOTSET) | 126 logging.disable(logging.NOTSET) |
127 | 127 |
128 def writeChangelog(self, changes): | 128 def writeChangelog(self, changes): |
129 """ | 129 """ |
130 write the changelog file into the cloned repository | 130 write the changelog file into the cloned repository |
131 """ | 131 """ |
132 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) | 132 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) |
133 if not os.path.exists(baseDir): | 133 if not os.path.exists(baseDir): |
134 os.makedirs(baseDir) | 134 os.makedirs(baseDir) |
135 changelogFile = "%s-%s.changelog.xhtml" % (self.basename, self.version) | 135 changelogFile = '%s-%s.changelog.xhtml' % (self.basename, self.version) |
136 changelogPath = os.path.join(baseDir, changelogFile) | 136 changelogPath = os.path.join(baseDir, changelogFile) |
137 self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.base
name + '/' + changelogFile) | 137 self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.base
name + '/' + changelogFile) |
138 | 138 |
139 template = get_template(get_config().get('extensions', 'changelogTemplat
e')) | 139 template = get_template(get_config().get('extensions', 'changelogTemplat
e')) |
140 template.stream({'changes': changes}).dump(changelogPath, encoding='utf-
8') | 140 template.stream({'changes': changes}).dump(changelogPath, encoding='utf-
8') |
141 | 141 |
142 linkPath = os.path.join(baseDir, '00latest.changelog.xhtml') | 142 linkPath = os.path.join(baseDir, '00latest.changelog.xhtml') |
143 if hasattr(os, 'symlink'): | 143 if hasattr(os, 'symlink'): |
144 if os.path.exists(linkPath): | 144 if os.path.exists(linkPath): |
145 os.remove(linkPath) | 145 os.remove(linkPath) |
146 os.symlink(os.path.basename(changelogPath), linkPath) | 146 os.symlink(os.path.basename(changelogPath), linkPath) |
147 else: | 147 else: |
148 shutil.copyfile(changelogPath, linkPath) | 148 shutil.copyfile(changelogPath, linkPath) |
149 | 149 |
150 def readGeckoMetadata(self): | 150 def readGeckoMetadata(self): |
151 """ | 151 """ |
152 read Gecko-specific metadata file from a cloned repository | 152 read Gecko-specific metadata file from a cloned repository |
153 and parse id, version, basename and the compat section | 153 and parse id, version, basename and the compat section |
154 out of the file | 154 out of the file |
155 """ | 155 """ |
156 import buildtools.packagerGecko as packager | 156 import buildtools.packagerGecko as packager |
157 metadata = packager.readMetadata(self.tempdir, self.config.type) | 157 metadata = packager.readMetadata(self.tempdir, self.config.type) |
158 self.extensionID = metadata.get("general", "id") | 158 self.extensionID = metadata.get('general', 'id') |
159 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s
elf.revision) | 159 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s
elf.revision) |
160 self.basename = metadata.get("general", "basename") | 160 self.basename = metadata.get('general', 'basename') |
161 self.compat = [] | 161 self.compat = [] |
162 for key, value in packager.KNOWN_APPS.iteritems(): | 162 for key, value in packager.KNOWN_APPS.iteritems(): |
163 if metadata.has_option('compat', key): | 163 if metadata.has_option('compat', key): |
164 minVersion, maxVersion = metadata.get('compat', key).split('/') | 164 minVersion, maxVersion = metadata.get('compat', key).split('/') |
165 self.compat.append({'id': value, 'minVersion': minVersion, 'maxV
ersion': maxVersion}) | 165 self.compat.append({'id': value, 'minVersion': minVersion, 'maxV
ersion': maxVersion}) |
166 | 166 |
167 def readAndroidMetadata(self): | 167 def readAndroidMetadata(self): |
168 """ | 168 """ |
169 Read Android-specific metadata from AndroidManifest.xml file. | 169 Read Android-specific metadata from AndroidManifest.xml file. |
170 """ | 170 """ |
171 manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), '
r') | 171 manifestFile = open(os.path.join(self.tempdir, 'AndroidManifest.xml'), '
r') |
172 manifest = parseXml(manifestFile) | 172 manifest = parseXml(manifestFile) |
173 manifestFile.close() | 173 manifestFile.close() |
174 | 174 |
175 root = manifest.documentElement | 175 root = manifest.documentElement |
176 self.version = root.attributes["android:versionName"].value | 176 self.version = root.attributes['android:versionName'].value |
177 while self.version.count('.') < 2: | 177 while self.version.count('.') < 2: |
178 self.version += '.0' | 178 self.version += '.0' |
179 self.version = '%s.%s' % (self.version, self.revision) | 179 self.version = '%s.%s' % (self.version, self.revision) |
180 | 180 |
181 usesSdk = manifest.getElementsByTagName('uses-sdk')[0] | 181 usesSdk = manifest.getElementsByTagName('uses-sdk')[0] |
182 self.minSdkVersion = usesSdk.attributes["android:minSdkVersion"].value | 182 self.minSdkVersion = usesSdk.attributes['android:minSdkVersion'].value |
183 self.basename = os.path.basename(self.config.repository) | 183 self.basename = os.path.basename(self.config.repository) |
184 | 184 |
185 def readChromeMetadata(self): | 185 def readChromeMetadata(self): |
186 """ | 186 """ |
187 Read Chrome-specific metadata from metadata file. This will also | 187 Read Chrome-specific metadata from metadata file. This will also |
188 calculate extension ID from the private key. | 188 calculate extension ID from the private key. |
189 """ | 189 """ |
190 | 190 |
191 # Calculate extension ID from public key | 191 # Calculate extension ID from public key |
192 # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-f
rom-your-private-key-233) | 192 # (see http://supercollider.dk/2010/01/calculating-chrome-extension-id-f
rom-your-private-key-233) |
193 import buildtools.packagerChrome as packager | 193 import buildtools.packagerChrome as packager |
194 publicKey = packager.getPublicKey(self.config.keyFile) | 194 publicKey = packager.getPublicKey(self.config.keyFile) |
195 hash = hashlib.sha256() | 195 hash = hashlib.sha256() |
196 hash.update(publicKey) | 196 hash.update(publicKey) |
197 self.extensionID = hash.hexdigest()[0:32] | 197 self.extensionID = hash.hexdigest()[0:32] |
198 self.extensionID = ''.join(map(lambda c: chr(97 + int(c, 16)), self.exte
nsionID)) | 198 self.extensionID = ''.join(map(lambda c: chr(97 + int(c, 16)), self.exte
nsionID)) |
199 | 199 |
200 # Now read metadata file | 200 # Now read metadata file |
201 metadata = packager.readMetadata(self.tempdir, self.config.type) | 201 metadata = packager.readMetadata(self.tempdir, self.config.type) |
202 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s
elf.revision) | 202 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s
elf.revision) |
203 self.basename = metadata.get("general", "basename") | 203 self.basename = metadata.get('general', 'basename') |
204 | 204 |
205 self.compat = [] | 205 self.compat = [] |
206 if metadata.has_section('compat') and metadata.has_option('compat', 'chr
ome'): | 206 if metadata.has_section('compat') and metadata.has_option('compat', 'chr
ome'): |
207 self.compat.append({'id': 'chrome', 'minVersion': metadata.get('comp
at', 'chrome')}) | 207 self.compat.append({'id': 'chrome', 'minVersion': metadata.get('comp
at', 'chrome')}) |
208 | 208 |
209 def readSafariMetadata(self): | 209 def readSafariMetadata(self): |
210 import buildtools.packagerSafari as packager | 210 import buildtools.packagerSafari as packager |
211 metadata = packager.readMetadata(self.tempdir, self.config.type) | 211 metadata = packager.readMetadata(self.tempdir, self.config.type) |
212 certs = packager.get_certificates_and_key(self.config.keyFile)[0] | 212 certs = packager.get_certificates_and_key(self.config.keyFile)[0] |
213 | 213 |
214 self.certificateID = packager.get_developer_identifier(certs) | 214 self.certificateID = packager.get_developer_identifier(certs) |
215 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s
elf.revision) | 215 self.version = packager.getBuildVersion(self.tempdir, metadata, False, s
elf.revision) |
216 self.shortVersion = metadata.get("general", "version") | 216 self.shortVersion = metadata.get('general', 'version') |
217 self.basename = metadata.get("general", "basename") | 217 self.basename = metadata.get('general', 'basename') |
218 self.updatedFromGallery = False | 218 self.updatedFromGallery = False |
219 | 219 |
220 def writeUpdateManifest(self): | 220 def writeUpdateManifest(self): |
221 """ | 221 """ |
222 Writes update.rdf file for the current build | 222 Writes update.rdf file for the current build |
223 """ | 223 """ |
224 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) | 224 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) |
225 if self.config.type == 'safari': | 225 if self.config.type == 'safari': |
226 manifestPath = os.path.join(baseDir, "updates.plist") | 226 manifestPath = os.path.join(baseDir, 'updates.plist') |
227 templateName = 'safariUpdateManifest' | 227 templateName = 'safariUpdateManifest' |
228 elif self.config.type == 'android': | 228 elif self.config.type == 'android': |
229 manifestPath = os.path.join(baseDir, "updates.xml") | 229 manifestPath = os.path.join(baseDir, 'updates.xml') |
230 templateName = 'androidUpdateManifest' | 230 templateName = 'androidUpdateManifest' |
231 else: | 231 else: |
232 return | 232 return |
233 | 233 |
234 if not os.path.exists(baseDir): | 234 if not os.path.exists(baseDir): |
235 os.makedirs(baseDir) | 235 os.makedirs(baseDir) |
236 | 236 |
237 # ABP for Android used to have its own update manifest format. We need t
o | 237 # ABP for Android used to have its own update manifest format. We need t
o |
238 # generate both that and the new one in the libadblockplus format as lon
g | 238 # generate both that and the new one in the libadblockplus format as lon
g |
239 # as a significant amount of users is on an old version. | 239 # as a significant amount of users is on an old version. |
240 if self.config.type == 'android': | 240 if self.config.type == 'android': |
241 newManifestPath = os.path.join(baseDir, "update.json") | 241 newManifestPath = os.path.join(baseDir, 'update.json') |
242 writeAndroidUpdateManifest(newManifestPath, [{ | 242 writeAndroidUpdateManifest(newManifestPath, [{ |
243 'basename': self.basename, | 243 'basename': self.basename, |
244 'version': self.version, | 244 'version': self.version, |
245 'updateURL': self.updateURL | 245 'updateURL': self.updateURL |
246 }]) | 246 }]) |
247 | 247 |
248 template = get_template(get_config().get('extensions', templateName)) | 248 template = get_template(get_config().get('extensions', templateName)) |
249 template.stream({'extensions': [self]}).dump(manifestPath) | 249 template.stream({'extensions': [self]}).dump(manifestPath) |
250 | 250 |
251 def writeIEUpdateManifest(self, versions): | 251 def writeIEUpdateManifest(self, versions): |
252 """ | 252 """ |
253 Writes update.json file for the latest IE build | 253 Writes update.json file for the latest IE build |
254 """ | 254 """ |
255 if len(versions) == 0: | 255 if len(versions) == 0: |
256 return | 256 return |
257 | 257 |
258 version = versions[0] | 258 version = versions[0] |
259 packageName = self.basename + '-' + version + self.config.packageSuffix | 259 packageName = self.basename + '-' + version + self.config.packageSuffix |
260 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '
/' + packageName + '?update') | 260 updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '
/' + packageName + '?update') |
261 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) | 261 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) |
262 manifestPath = os.path.join(baseDir, 'update.json') | 262 manifestPath = os.path.join(baseDir, 'update.json') |
263 | 263 |
264 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit
e | 264 from sitescripts.extensions.utils import writeIEUpdateManifest as doWrit
e |
265 doWrite(manifestPath, [{ | 265 doWrite(manifestPath, [{ |
266 'basename': self.basename, | 266 'basename': self.basename, |
267 'version': version, | 267 'version': version, |
268 'updateURL': updateURL | 268 'updateURL': updateURL |
269 }]) | 269 }]) |
270 | 270 |
271 for suffix in ["-x86.msi", "-x64.msi", "-gpo-x86.msi", "-gpo-x64.msi"]: | 271 for suffix in ['-x86.msi', '-x64.msi', '-gpo-x86.msi', '-gpo-x64.msi']: |
272 linkPath = os.path.join(baseDir, '00latest%s' % suffix) | 272 linkPath = os.path.join(baseDir, '00latest%s' % suffix) |
273 outputPath = os.path.join(baseDir, self.basename + '-' + version + s
uffix) | 273 outputPath = os.path.join(baseDir, self.basename + '-' + version + s
uffix) |
274 if hasattr(os, 'symlink'): | 274 if hasattr(os, 'symlink'): |
275 if os.path.exists(linkPath): | 275 if os.path.exists(linkPath): |
276 os.remove(linkPath) | 276 os.remove(linkPath) |
277 os.symlink(os.path.basename(outputPath), linkPath) | 277 os.symlink(os.path.basename(outputPath), linkPath) |
278 else: | 278 else: |
279 shutil.copyfile(outputPath, linkPath) | 279 shutil.copyfile(outputPath, linkPath) |
280 | 280 |
281 def build(self): | 281 def build(self): |
282 """ | 282 """ |
283 run the build command in the tempdir | 283 run the build command in the tempdir |
284 """ | 284 """ |
285 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) | 285 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) |
286 if not os.path.exists(baseDir): | 286 if not os.path.exists(baseDir): |
287 os.makedirs(baseDir) | 287 os.makedirs(baseDir) |
288 outputFile = "%s-%s%s" % (self.basename, self.version, self.config.packa
geSuffix) | 288 outputFile = '%s-%s%s' % (self.basename, self.version, self.config.packa
geSuffix) |
289 self.path = os.path.join(baseDir, outputFile) | 289 self.path = os.path.join(baseDir, outputFile) |
290 self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basenam
e + '/' + outputFile + '?update') | 290 self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basenam
e + '/' + outputFile + '?update') |
291 | 291 |
292 if self.config.type == 'android': | 292 if self.config.type == 'android': |
293 apkFile = open(self.path, 'wb') | 293 apkFile = open(self.path, 'wb') |
294 | 294 |
295 try: | 295 try: |
296 try: | 296 try: |
297 port = get_config().get('extensions', 'androidBuildPort') | 297 port = get_config().get('extensions', 'androidBuildPort') |
298 except ConfigParser.NoOptionError: | 298 except ConfigParser.NoOptionError: |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
347 os.remove(changelogPath) | 347 os.remove(changelogPath) |
348 return versions | 348 return versions |
349 | 349 |
350 def updateIndex(self, versions): | 350 def updateIndex(self, versions): |
351 """ | 351 """ |
352 Updates index page listing all existing versions | 352 Updates index page listing all existing versions |
353 """ | 353 """ |
354 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) | 354 baseDir = os.path.join(self.config.nightliesDirectory, self.basename) |
355 if not os.path.exists(baseDir): | 355 if not os.path.exists(baseDir): |
356 os.makedirs(baseDir) | 356 os.makedirs(baseDir) |
357 outputFile = "index.html" | 357 outputFile = 'index.html' |
358 outputPath = os.path.join(baseDir, outputFile) | 358 outputPath = os.path.join(baseDir, outputFile) |
359 | 359 |
360 links = [] | 360 links = [] |
361 for version in versions: | 361 for version in versions: |
362 packageFile = self.basename + '-' + version + self.config.packageSuf
fix | 362 packageFile = self.basename + '-' + version + self.config.packageSuf
fix |
363 changelogFile = self.basename + '-' + version + '.changelog.xhtml' | 363 changelogFile = self.basename + '-' + version + '.changelog.xhtml' |
364 if not os.path.exists(os.path.join(baseDir, packageFile)): | 364 if not os.path.exists(os.path.join(baseDir, packageFile)): |
365 # Oops | 365 # Oops |
366 continue | 366 continue |
367 | 367 |
(...skipping 242 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
610 # build all extensions specified in the configuration file | 610 # build all extensions specified in the configuration file |
611 # and generate changelogs and documentations for each: | 611 # and generate changelogs and documentations for each: |
612 data = None | 612 data = None |
613 for repo in Configuration.getRepositoryConfigurations(nightlyConfig): | 613 for repo in Configuration.getRepositoryConfigurations(nightlyConfig): |
614 build = None | 614 build = None |
615 try: | 615 try: |
616 build = NightlyBuild(repo) | 616 build = NightlyBuild(repo) |
617 if build.hasChanges(): | 617 if build.hasChanges(): |
618 build.run() | 618 build.run() |
619 except Exception, ex: | 619 except Exception, ex: |
620 logging.error("The build for %s failed:", repo) | 620 logging.error('The build for %s failed:', repo) |
621 logging.exception(ex) | 621 logging.exception(ex) |
622 | 622 |
623 file = open(nightlyConfigFile, 'wb') | 623 file = open(nightlyConfigFile, 'wb') |
624 nightlyConfig.write(file) | 624 nightlyConfig.write(file) |
625 | 625 |
626 | 626 |
627 if __name__ == '__main__': | 627 if __name__ == '__main__': |
628 main() | 628 main() |
OLD | NEW |