Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/updateDownloadLinks.py

Issue 6291923287408640: Issue 1093 - Separate update manifest generation (Closed)
Patch Set: Address comments Created July 22, 2014, 11:59 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2014 Eyeo GmbH 4 # Copyright (C) 2006-2014 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 """ 18 """
19 Update the list of extenstions 19 Update the list of extenstions
20 ============================== 20 ==============================
21 21
22 This script generates a list of extensions and saves these with download links 22 This script generates a list of extensions and saves these with download links
23 and version information 23 and version information
24 """ 24 """
25 25
26 import sys, os, re, urllib, urllib2, urlparse, subprocess, time
27 import xml.dom.minidom as dom
28 from ConfigParser import SafeConfigParser 26 from ConfigParser import SafeConfigParser
29 from StringIO import StringIO 27 from sitescripts.utils import get_config
30 from sitescripts.utils import get_config, get_template 28 from sitescripts.extensions.utils import Configuration, getDownloadLinks
31 from sitescripts.extensions.utils import compareVersions, Configuration, getSafa riCertificateID
32 from sitescripts.extensions.android import get_min_sdk_version
33 from sitescripts.extensions.pad import PadFile 29 from sitescripts.extensions.pad import PadFile
34 from buildtools.packagerGecko import KNOWN_APPS
35
36 def urlencode(value):
37 return urllib.quote(value.encode('utf-8'), '')
38
39 def urlopen(url, attempts=3):
40 """
41 Tries to open a particular URL, retries on failure.
42 """
43 for i in range(attempts):
44 try:
45 return urllib.urlopen(url)
46 except IOError, e:
47 error = e
48 time.sleep(5)
49 raise error
50
51 def getMozillaDownloadLink(galleryID):
52 """
53 gets download link for a Gecko add-on from the Mozilla Addons site
54 """
55 url = 'https://services.addons.mozilla.org/en-US/firefox/api/1/addon/%s' % url encode(galleryID)
56 contents = urlopen(url).read()
57 document = dom.parseString(contents)
58 linkTags = document.getElementsByTagName('install')
59 linkTag = linkTags[0] if len(linkTags) > 0 else None
60 versionTags = document.getElementsByTagName('version')
61 versionTag = versionTags[0] if len(versionTags) > 0 else None
62 if linkTag and versionTag and linkTag.firstChild and versionTag.firstChild:
63 return (linkTag.firstChild.data, versionTag.firstChild.data)
64 else:
65 return (None, None)
66
67 def getGoogleDownloadLink(galleryID):
68 """
69 gets download link for a Chrome add-on from the Chrome Gallery site
70 """
71 galleryID = urlencode(galleryID)
72
73 url = 'https://clients2.google.com/service/update2/crx?x=%s' % urlencode('id=% s&uc' % galleryID)
74 document = dom.parse(urlopen(url))
75 updateTags = document.getElementsByTagName('updatecheck')
76 version = updateTags and updateTags[0].getAttribute('version')
77
78 if not version:
79 return (None, None)
80
81 request = urllib2.Request('https://chrome.google.com/webstore/detail/_/' + gal leryID)
82 request.get_method = lambda : 'HEAD'
83 url = urllib2.urlopen(request).geturl()
84
85 return (url, version)
86
87 def getOperaDownloadLink(galleryID):
88 """
89 gets download link for an Opera add-on from the Opera Addons site
90 """
91 galleryID = urlencode(galleryID)
92
93 request = urllib2.Request('https://addons.opera.com/extensions/download/%s/' % galleryID)
94 request.get_method = lambda : 'HEAD'
95 response = urllib2.urlopen(request)
96
97 content_disposition = response.info().getheader('Content-Disposition')
98 if content_disposition:
99 match = re.search(r'filename=\S+-([\d.]+)-\d+\.crx$', content_disposition)
100 if match:
101 return ('https://addons.opera.com/extensions/details/%s/' % galleryID , ma tch.group(1))
102
103 return (None, None)
104
105 def getLocalLink(repo):
106 """
107 gets the link for the newest download of an add-on in the local downloads
108 repository
109 """
110 highestURL = None
111 highestVersion = None
112
113 for filename, version in repo.getDownloads():
114 if not highestVersion or compareVersions(version, highestVersion) > 0:
115 highestURL = urlparse.urljoin(repo.downloadsURL, filename)
116 highestVersion = version
117
118 return (highestURL, highestVersion)
119
120 def getDownloadLink(repo):
121 """
122 gets the download link to the most current version of an extension
123 """
124 # you can't easily install extensions from third-party sources on Chrome
125 # and Opera. So always get the link for the version on the Web Store.
126 if repo.galleryID:
127 if repo.type == "chrome":
128 return getGoogleDownloadLink(repo.galleryID)
129 if repo.type == "opera":
130 return getOperaDownloadLink(repo.galleryID)
131
132 (localURL, localVersion) = getLocalLink(repo)
133
134 # get a link to Firefox Add-Ons, if the latest version has been published ther e
135 if repo.type == 'gecko' and repo.galleryID:
136 (galleryURL, galleryVersion) = getMozillaDownloadLink(repo.galleryID)
137 if not localVersion or (galleryVersion and
138 compareVersions(galleryVersion, localVersion) >= 0):
139 return (galleryURL, galleryVersion)
140
141 return (localURL, localVersion)
142
143 def getQRCode(text):
144 try:
145 import qrcode
146 import base64
147 import Image # required by qrcode but not formally a dependency
148 except:
149 return None
150
151 data = StringIO()
152 qrcode.make(text, box_size=5).save(data, 'png')
153 return 'data:image/png;base64,' + base64.b64encode(data.getvalue())
154
155 def getDownloadLinks(result):
156 """
157 gets the download links for all extensions and puts them into the config
158 object
159 """
160 for repo in Configuration.getRepositoryConfigurations():
161 (downloadURL, version) = getDownloadLink(repo)
162 if downloadURL == None:
163 continue
164 if not result.has_section(repo.repositoryName):
165 result.add_section(repo.repositoryName)
166 result.set(repo.repositoryName, "downloadURL", downloadURL)
167 result.set(repo.repositoryName, "version", version)
168
169 qrcode = getQRCode(downloadURL)
170 if qrcode != None:
171 result.set(repo.repositoryName, "qrcode", qrcode)
172
173 def readMetadata(repo, version):
174 """
175 reads extension ID and compatibility information from metadata file in the
176 extension's repository
177 """
178 if repo.type == 'android':
179 command = ['hg', '-R', repo.repository, 'id', '-r', version, '-n']
180 result = subprocess.check_output(command)
181 revision = re.sub(r'\D', '', result)
182
183 return {
184 'revision': revision,
185 'minSdkVersion': get_min_sdk_version(repo, version),
186 }
187 elif repo.type == 'safari':
188 metadata = repo.readMetadata(version)
189 return {
190 'certificateID': getSafariCertificateID(repo.keyFile),
191 'version': version,
192 'shortVersion': version,
193 'basename': metadata.get('general', 'basename'),
194 }
195 elif repo.type == 'gecko':
196 metadata = repo.readMetadata(version)
197 result = {
198 'extensionID': metadata.get('general', 'id'),
199 'version': version,
200 'compat': []
201 }
202 for key, value in KNOWN_APPS.iteritems():
203 if metadata.has_option('compat', key):
204 minVersion, maxVersion = metadata.get('compat', key).split('/')
205 result['compat'].append({'id': value, 'minVersion': minVersion, 'maxVers ion': maxVersion})
206 return result
207 else:
208 raise Exception('unknown repository type %r' % repo.type)
209
210 def writeUpdateManifest(links):
211 """
212 writes an update manifest for all Gecko extensions and Android apps
213 """
214
215 extensions = {'gecko': [], 'android': [], 'safari': []}
216 for repo in Configuration.getRepositoryConfigurations():
217 if repo.type not in extensions or not links.has_section(repo.repositoryName) :
218 continue
219 data = readMetadata(repo, links.get(repo.repositoryName, 'version'))
220 data['updateURL'] = links.get(repo.repositoryName, 'downloadURL')
221 if data['updateURL'].startswith(repo.downloadsURL):
222 data['updateURL'] += "?update"
223 extensions[repo.type].append(data)
224
225 if len(extensions['android']) > 1:
226 print >>sys.stderr, 'Warning: more than one Android app defined, update mani fest only works for one'
227
228 for repoType in extensions.iterkeys():
229 manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoT ype)
230 template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
231 template.stream({'extensions': extensions[repoType]}).dump(manifestPath)
232 30
233 def writePadFile(links): 31 def writePadFile(links):
234 for repo in Configuration.getRepositoryConfigurations(): 32 for repo in Configuration.getRepositoryConfigurations():
235 if repo.pad and links.has_section(repo.repositoryName): 33 if repo.pad and links.has_section(repo.repositoryName):
236 PadFile.forRepository(repo, links.get(repo.repositoryName, 'version'), 34 PadFile.forRepository(repo, links.get(repo.repositoryName, 'version'),
237 links.get(repo.repositoryName, 'downloadURL')) .write() 35 links.get(repo.repositoryName, 'downloadURL')) .write()
238 36
239 def updateLinks(): 37 def updateLinks():
240 """ 38 """
241 writes the current extension download links to a file 39 writes the current extension download links to a file
242 """ 40 """
243 41
244 # Now get download links and save them to file 42 # Now get download links and save them to file
245 result = SafeConfigParser() 43 result = SafeConfigParser()
246 getDownloadLinks(result) 44 getDownloadLinks(result)
247 file = open(get_config().get('extensions', 'downloadLinksFile'), 'wb') 45 file = open(get_config().get('extensions', 'downloadLinksFile'), 'wb')
248 result.write(file) 46 result.write(file)
249 file.close() 47 file.close()
250 48
251 writeUpdateManifest(result)
252 writePadFile(result) 49 writePadFile(result)
253 50
254 if __name__ == "__main__": 51 if __name__ == "__main__":
255 updateLinks() 52 updateLinks()
OLDNEW

Powered by Google App Engine
This is Rietveld