Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: packagerGecko.py

Issue 29562599: Issue 5751 - Removing legacy gecko support (Closed)
Patch Set: Created Oct. 2, 2017, 10:33 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 import os
6 import sys
7 import re
8 import hashlib
9 import base64
10 import urllib
11 import json
12 import io
13 from ConfigParser import SafeConfigParser
14 from StringIO import StringIO
15 import xml.dom.minidom as minidom
16 import buildtools.localeTools as localeTools
17
18 import packager
19 from packager import readMetadata, getMetadataPath, getDefaultFileName, getBuild Version, getTemplate, Files
20
21 KNOWN_APPS = {
22 'conkeror': '{a79fe89b-6662-4ff4-8e88-09950ad4dfde}',
23 'emusic': 'dlm@emusic.com',
24 'fennec': '{a23983c0-fd0e-11dc-95ff-0800200c9a66}',
25 'fennec2': '{aa3c5121-dab2-40e2-81ca-7ea25febc110}',
26 'firefox': '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}',
27 'midbrowser': '{aa5ca914-c309-495d-91cf-3141bbb04115}',
28 'prism': 'prism@developer.mozilla.org',
29 'seamonkey': '{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}',
30 'songbird': 'songbird@songbirdnest.com',
31 'thunderbird': '{3550f703-e582-4d05-9a08-453d09bdfdc6}',
32 'toolkit': 'toolkit@mozilla.org',
33 'adblockbrowser': '{55aba3ac-94d3-41a8-9e25-5c21fe874539}',
34 }
35
36 defaultLocale = 'en-US'
37
38
39 def getChromeDir(baseDir):
40 return os.path.join(baseDir, 'chrome')
41
42
43 def getLocalesDir(baseDir):
44 return os.path.join(getChromeDir(baseDir), 'locale')
45
46
47 def getChromeSubdirs(baseDir, locales):
48 result = {}
49 chromeDir = getChromeDir(baseDir)
50 for subdir in ('content', 'skin'):
51 result[subdir] = os.path.join(chromeDir, subdir)
52 for locale in locales:
53 result['locale/%s' % locale] = os.path.join(chromeDir, 'locale', locale)
54 return result
55
56
57 def getPackageFiles(params):
58 result = {
59 'chrome', 'components', 'modules', 'lib', 'resources', 'webextension',
60 'chrome.manifest', 'icon.png', 'icon64.png'
61 }
62
63 baseDir = params['baseDir']
64 for file in os.listdir(baseDir):
65 if file.endswith('.js') or file.endswith('.xml'):
66 result.add(file)
67 return result
68
69
70 def getIgnoredFiles(params):
71 return {'.incomplete', 'meta.properties'}
72
73
74 def archive_path(path, baseDir):
75 return '/'.join(os.path.split(os.path.relpath(path, baseDir)))
76
77
78 def isValidLocale(localesDir, dir, includeIncomplete=False):
79 if re.search(r'[^\w\-]', dir):
80 return False
81 curLocaleDir = os.path.join(localesDir, dir)
82 if not os.path.isdir(curLocaleDir):
83 return False
84 if len(os.listdir(curLocaleDir)) == 0:
85 return False
86 if not includeIncomplete and os.path.exists(os.path.join(localesDir, dir, '. incomplete')):
87 return False
88 return True
89
90
91 def getLocales(baseDir, includeIncomplete=False):
92 global defaultLocale
93 localesDir = getLocalesDir(baseDir)
94 locales = filter(lambda dir: isValidLocale(localesDir, dir, includeIncomplet e), os.listdir(localesDir))
95 locales.sort(key=lambda x: '!' if x == defaultLocale else x)
96 return locales
97
98
99 def processFile(path, data, params):
100 if path.endswith('.manifest') and data.find('{{LOCALE}}') >= 0:
101 localesRegExp = re.compile(r'^(.*?){{LOCALE}}(.*?){{LOCALE}}(.*)$', re.M )
102 replacement = '\n'.join(map(lambda locale: r'\1%s\2%s\3' % (locale, loca le), params['locales']))
103 data = re.sub(localesRegExp, replacement, data)
104
105 return data
106
107
108 def readLocaleMetadata(baseDir, locales):
109 result = {}
110
111 # Make sure we always have fallback data even if the default locale isn't pa rt
112 # of the build
113 locales = list(locales)
114 if not defaultLocale in locales:
115 locales.append(defaultLocale)
116
117 for locale in locales:
118 data = SafeConfigParser()
119 data.optionxform = str
120 try:
121 result[locale] = localeTools.readFile(os.path.join(getLocalesDir(bas eDir), locale, 'meta.properties'))
122 except:
123 result[locale] = {}
124 return result
125
126
127 def getContributors(metadata):
128 main = []
129 additional = set()
130 if metadata.has_section('contributors'):
131 options = metadata.options('contributors')
132 options.sort()
133 for option in options:
134 value = metadata.get('contributors', option)
135 if re.search(r'\D', option):
136 match = re.search(r'^\s*(\S+)\s+//([^/\s]+)/@(\S+)\s*$', value)
137 if not match:
138 print >>sys.stderr, 'Warning: unrecognized contributor locat ion "%s"\n' % value
139 continue
140 baseDir = os.path.dirname(metadata.option_source('contributors', option))
141 parts = match.group(1).split('/')
142 dom = minidom.parse(os.path.join(baseDir, *parts))
143 tags = dom.getElementsByTagName(match.group(2))
144 for tag in tags:
145 if tag.hasAttribute(match.group(3)):
146 for name in re.split(r'\s*,\s*', tag.getAttribute(match. group(3))):
147 additional.add(name)
148 else:
149 main.append(value)
150 return main + sorted(additional, key=unicode.lower)
151
152
153 def initTranslators(localeMetadata):
154 for locale in localeMetadata.itervalues():
155 if 'translator' in locale:
156 locale['translators'] = sorted(map(lambda t: t.strip(), locale['tran slator'].split(',')), key=unicode.lower)
157 else:
158 locale['translators'] = []
159
160
161 def createManifest(params):
162 global KNOWN_APPS, defaultLocale
163 template = getTemplate('install.rdf.tmpl', autoEscape=True)
164 templateData = dict(params)
165 templateData['localeMetadata'] = readLocaleMetadata(params['baseDir'], param s['locales'])
166 initTranslators(templateData['localeMetadata'])
167 templateData['KNOWN_APPS'] = KNOWN_APPS
168 templateData['defaultLocale'] = defaultLocale
169 return template.render(templateData).encode('utf-8')
170
171
172 def importLocales(params, files):
173 SECTION = 'import_locales'
174 if not params['metadata'].has_section(SECTION):
175 return
176
177 import localeTools
178
179 for locale in params['locales']:
180 for item in params['metadata'].items(SECTION):
181 path, keys = item
182 parts = [locale if p == '*' else p for p in path.split('/')]
183 source = os.path.join(os.path.dirname(item.source), *parts)
184 if not os.path.exists(source):
185 continue
186
187 with io.open(source, 'r', encoding='utf-8') as handle:
188 data = json.load(handle)
189
190 target_name = os.path.splitext(os.path.basename(source))[0] + '.prop erties'
191 target = archive_path(os.path.join(getLocalesDir(params['baseDir']), locale, target_name), params['baseDir'])
192
193 files[target] = ''
194 for key, value in sorted(data.items()):
195 message = value['message']
196 files[target] += localeTools.generateStringEntry(key, message, t arget).encode('utf-8')
197
198
199 def fixupLocales(params, files):
200 global defaultLocale
201
202 # Read in default locale data, it might not be included in package files
203 defaultLocaleDir = os.path.join(getLocalesDir(params['baseDir']), defaultLoc ale)
204 reference_files = Files(getPackageFiles(params), getIgnoredFiles(params))
205 reference_files.read(defaultLocaleDir, archive_path(defaultLocaleDir, params ['baseDir']))
206 reference_params = dict(params)
207 reference_params['locales'] = [defaultLocale]
208 importLocales(reference_params, reference_files)
209
210 reference = {}
211 for path, data in reference_files.iteritems():
212 filename = path.split('/')[-1]
213 data = localeTools.parseString(data.decode('utf-8'), filename)
214 if data:
215 reference[filename] = data
216
217 for locale in params['locales']:
218 for file in reference.iterkeys():
219 path = 'chrome/locale/%s/%s' % (locale, file)
220 if path in files:
221 data = localeTools.parseString(files[path].decode('utf-8'), path )
222 for key, value in reference[file].iteritems():
223 if not key in data:
224 files[path] += localeTools.generateStringEntry(key, valu e, path).encode('utf-8')
225 else:
226 files[path] = reference[file]['_origData'].encode('utf-8')
227
228
229 def processJSONFiles(params, files):
230 prefix = 'lib/'
231 for name, content in files.iteritems():
232 if name.startswith(prefix) and name.endswith('.json'):
233 params['jsonRequires'][name[len(prefix):]] = json.loads(content)
234 for name in params['jsonRequires'].iterkeys():
235 del files[prefix + name]
236
237
238 def addMissingFiles(params, files):
239 templateData = {
240 'hasChrome': False,
241 'hasChromeRequires': False,
242 'hasShutdownHandlers': False,
243 'chromeWindows': [],
244 'requires': set(),
245 'jsonRequires': params['jsonRequires'],
246 'metadata': params['metadata'],
247 'hasWebExtension': params['hasWebExtension'],
248 'multicompartment': params['multicompartment'],
249 'applications': dict((v, k) for k, v in KNOWN_APPS.iteritems()),
250 }
251
252 def checkScript(name):
253 content = files[name]
254 for match in re.finditer(r'(?:^|\s)require\(\s*"([\w\-]+)"\s*\)', conten t):
255 templateData['requires'].add(match.group(1))
256 if name.startswith('chrome/content/'):
257 templateData['hasChromeRequires'] = True
258 if not '/' in name or name.startswith('lib/'):
259 if re.search(r'(?:^|\s)onShutdown\.', content):
260 templateData['hasShutdownHandlers'] = True
261
262 for name, content in files.iteritems():
263 if name == 'chrome.manifest':
264 templateData['hasChrome'] = True
265 elif name.endswith('.js'):
266 checkScript(name)
267 elif name.endswith('.xul'):
268 match = re.search(r'<(?:window|dialog)\s[^>]*\bwindowtype="([^">]+)" ', content)
269 if match:
270 templateData['chromeWindows'].append(match.group(1))
271
272 while True:
273 missing = []
274 for module in templateData['requires']:
275 moduleFile = 'lib/' + module + '.js'
276 if not moduleFile in files:
277 import buildtools
278 path = os.path.join(buildtools.__path__[0], moduleFile)
279 if os.path.exists(path):
280 missing.append((path, moduleFile))
281 if not len(missing):
282 break
283 for path, moduleFile in missing:
284 files.read(path, moduleFile)
285 checkScript(moduleFile)
286
287 template = getTemplate('bootstrap.js.tmpl')
288 files['bootstrap.js'] = template.render(templateData).encode('utf-8')
289
290
291 def createBuild(baseDir, type='gecko', outFile=None, locales=None, buildNum=None , releaseBuild=False, multicompartment=False):
292 if locales == None:
293 locales = getLocales(baseDir)
294 elif locales == 'all':
295 locales = getLocales(baseDir, True)
296
297 metadata = readMetadata(baseDir, type)
298 version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum)
299
300 if outFile == None:
301 outFile = getDefaultFileName(metadata, version, 'xpi')
302
303 contributors = getContributors(metadata)
304
305 params = {
306 'baseDir': baseDir,
307 'locales': locales,
308 'releaseBuild': releaseBuild,
309 'version': version.encode('utf-8'),
310 'metadata': metadata,
311 'contributors': contributors,
312 'multicompartment': multicompartment,
313 'hasWebExtension': os.path.isdir(os.path.join(baseDir, 'webextension')),
314 'jsonRequires': {},
315 }
316
317 mapped = metadata.items('mapping') if metadata.has_section('mapping') else [ ]
318 skip = [opt for opt, _ in mapped] + ['chrome']
319 files = Files(getPackageFiles(params), getIgnoredFiles(params),
320 process=lambda path, data: processFile(path, data, params))
321 files['install.rdf'] = createManifest(params)
322 files.readMappedFiles(mapped)
323 files.read(baseDir, skip=skip)
324 for name, path in getChromeSubdirs(baseDir, params['locales']).iteritems():
325 if os.path.isdir(path):
326 files.read(path, 'chrome/%s' % name, skip=skip)
327 importLocales(params, files)
328 fixupLocales(params, files)
329 processJSONFiles(params, files)
330 if not 'bootstrap.js' in files:
331 addMissingFiles(params, files)
332 if metadata.has_section('preprocess'):
333 files.preprocess([f for f, _ in metadata.items('preprocess')])
334 files.zip(outFile, sortKey=lambda x: '!' if x == 'META-INF/zigbert.rsa' else x)
335
336
337 def autoInstall(baseDir, type, host, port, multicompartment=False):
338 fileBuffer = StringIO()
339 createBuild(baseDir, type=type, outFile=fileBuffer, multicompartment=multico mpartment)
340 urllib.urlopen('http://%s:%s/' % (host, port), data=fileBuffer.getvalue())
OLDNEW

Powered by Google App Engine
This is Rietveld