packagerGecko.py - Issue 29562599: Issue 5751 - Removing legacy gecko support

Side by Side Diff: packagerGecko.py

Issue 29562599: Issue 5751 - Removing legacy gecko support (Closed)

Patch Set: Created Oct. 2, 2017, 10:33 a.m.

Left:
Right:

Use n/p to move between diff chunks; N/P to move between comments.

Jump to:

View unified diff | Download patch

OLD	NEW
	(Empty)
1 # This Source Code Form is subject to the terms of the Mozilla Public

2 # License, v. 2.0. If a copy of the MPL was not distributed with this

3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

4

5 import os

6 import sys

7 import re

8 import hashlib

9 import base64

10 import urllib

11 import json

12 import io

13 from ConfigParser import SafeConfigParser

14 from StringIO import StringIO

15 import xml.dom.minidom as minidom

16 import buildtools.localeTools as localeTools

17

18 import packager

19 from packager import readMetadata, getMetadataPath, getDefaultFileName, getBuild Version, getTemplate, Files

20

21 KNOWN_APPS = {

22 'conkeror': '{a79fe89b-6662-4ff4-8e88-09950ad4dfde}',

23 'emusic': 'dlm@emusic.com',

24 'fennec': '{a23983c0-fd0e-11dc-95ff-0800200c9a66}',

25 'fennec2': '{aa3c5121-dab2-40e2-81ca-7ea25febc110}',

26 'firefox': '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}',

27 'midbrowser': '{aa5ca914-c309-495d-91cf-3141bbb04115}',

28 'prism': 'prism@developer.mozilla.org',

29 'seamonkey': '{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}',

30 'songbird': 'songbird@songbirdnest.com',

31 'thunderbird': '{3550f703-e582-4d05-9a08-453d09bdfdc6}',

32 'toolkit': 'toolkit@mozilla.org',

33 'adblockbrowser': '{55aba3ac-94d3-41a8-9e25-5c21fe874539}',

34 }

35

36 defaultLocale = 'en-US'

37

38

39 def getChromeDir(baseDir):

40 return os.path.join(baseDir, 'chrome')

41

42

43 def getLocalesDir(baseDir):

44 return os.path.join(getChromeDir(baseDir), 'locale')

45

46

47 def getChromeSubdirs(baseDir, locales):

48 result = {}

49 chromeDir = getChromeDir(baseDir)

50 for subdir in ('content', 'skin'):

51 result[subdir] = os.path.join(chromeDir, subdir)

52 for locale in locales:

53 result['locale/%s' % locale] = os.path.join(chromeDir, 'locale', locale)

54 return result

55

56

57 def getPackageFiles(params):

58 result = {

59 'chrome', 'components', 'modules', 'lib', 'resources', 'webextension',

60 'chrome.manifest', 'icon.png', 'icon64.png'

61 }

62

63 baseDir = params['baseDir']

64 for file in os.listdir(baseDir):

65 if file.endswith('.js') or file.endswith('.xml'):

66 result.add(file)

67 return result

68

69

70 def getIgnoredFiles(params):

71 return {'.incomplete', 'meta.properties'}

72

73

74 def archive_path(path, baseDir):

75 return '/'.join(os.path.split(os.path.relpath(path, baseDir)))

76

77

78 def isValidLocale(localesDir, dir, includeIncomplete=False):

79 if re.search(r'[^\w\-]', dir):

80 return False

81 curLocaleDir = os.path.join(localesDir, dir)

82 if not os.path.isdir(curLocaleDir):

83 return False

84 if len(os.listdir(curLocaleDir)) == 0:

85 return False

86 if not includeIncomplete and os.path.exists(os.path.join(localesDir, dir, '. incomplete')):

87 return False

88 return True

89

90

91 def getLocales(baseDir, includeIncomplete=False):

92 global defaultLocale

93 localesDir = getLocalesDir(baseDir)

94 locales = filter(lambda dir: isValidLocale(localesDir, dir, includeIncomplet e), os.listdir(localesDir))

95 locales.sort(key=lambda x: '!' if x == defaultLocale else x)

96 return locales

97

98

99 def processFile(path, data, params):

100 if path.endswith('.manifest') and data.find('{{LOCALE}}') >= 0:

101 localesRegExp = re.compile(r'^(.?){{LOCALE}}(.?){{LOCALE}}(.*)$', re.M )

102 replacement = '\n'.join(map(lambda locale: r'\1%s\2%s\3' % (locale, loca le), params['locales']))

103 data = re.sub(localesRegExp, replacement, data)

104

105 return data

106

107

108 def readLocaleMetadata(baseDir, locales):

109 result = {}

110

111 # Make sure we always have fallback data even if the default locale isn't pa rt

112 # of the build

113 locales = list(locales)

114 if not defaultLocale in locales:

115 locales.append(defaultLocale)

116

117 for locale in locales:

118 data = SafeConfigParser()

119 data.optionxform = str

120 try:

121 result[locale] = localeTools.readFile(os.path.join(getLocalesDir(bas eDir), locale, 'meta.properties'))

122 except:

123 result[locale] = {}

124 return result

125

126

127 def getContributors(metadata):

128 main = []

129 additional = set()

130 if metadata.has_section('contributors'):

131 options = metadata.options('contributors')

132 options.sort()

133 for option in options:

134 value = metadata.get('contributors', option)

135 if re.search(r'\D', option):

136 match = re.search(r'^\s(\S+)\s+//([^/\s]+)/@(\S+)\s$', value)

137 if not match:

138 print >>sys.stderr, 'Warning: unrecognized contributor locat ion "%s"\n' % value

139 continue

140 baseDir = os.path.dirname(metadata.option_source('contributors', option))

141 parts = match.group(1).split('/')

142 dom = minidom.parse(os.path.join(baseDir, *parts))

143 tags = dom.getElementsByTagName(match.group(2))

144 for tag in tags:

145 if tag.hasAttribute(match.group(3)):

146 for name in re.split(r'\s,\s', tag.getAttribute(match. group(3))):

147 additional.add(name)

148 else:

149 main.append(value)

150 return main + sorted(additional, key=unicode.lower)

151

152

153 def initTranslators(localeMetadata):

154 for locale in localeMetadata.itervalues():

155 if 'translator' in locale:

156 locale['translators'] = sorted(map(lambda t: t.strip(), locale['tran slator'].split(',')), key=unicode.lower)

157 else:

158 locale['translators'] = []

159

160

161 def createManifest(params):

162 global KNOWN_APPS, defaultLocale

163 template = getTemplate('install.rdf.tmpl', autoEscape=True)

164 templateData = dict(params)

165 templateData['localeMetadata'] = readLocaleMetadata(params['baseDir'], param s['locales'])

166 initTranslators(templateData['localeMetadata'])

167 templateData['KNOWN_APPS'] = KNOWN_APPS

168 templateData['defaultLocale'] = defaultLocale

169 return template.render(templateData).encode('utf-8')

170

171

172 def importLocales(params, files):

173 SECTION = 'import_locales'

174 if not params['metadata'].has_section(SECTION):

175 return

176

177 import localeTools

178

179 for locale in params['locales']:

180 for item in params['metadata'].items(SECTION):

181 path, keys = item

182 parts = [locale if p == '*' else p for p in path.split('/')]

183 source = os.path.join(os.path.dirname(item.source), *parts)

184 if not os.path.exists(source):

185 continue

186

187 with io.open(source, 'r', encoding='utf-8') as handle:

188 data = json.load(handle)

189

190 target_name = os.path.splitext(os.path.basename(source))[0] + '.prop erties'

191 target = archive_path(os.path.join(getLocalesDir(params['baseDir']), locale, target_name), params['baseDir'])

192

193 files[target] = ''

194 for key, value in sorted(data.items()):

195 message = value['message']

196 files[target] += localeTools.generateStringEntry(key, message, t arget).encode('utf-8')

197

198

199 def fixupLocales(params, files):

200 global defaultLocale

201

202 # Read in default locale data, it might not be included in package files

203 defaultLocaleDir = os.path.join(getLocalesDir(params['baseDir']), defaultLoc ale)

204 reference_files = Files(getPackageFiles(params), getIgnoredFiles(params))

205 reference_files.read(defaultLocaleDir, archive_path(defaultLocaleDir, params ['baseDir']))

206 reference_params = dict(params)

207 reference_params['locales'] = [defaultLocale]

208 importLocales(reference_params, reference_files)

209

210 reference = {}

211 for path, data in reference_files.iteritems():

212 filename = path.split('/')[-1]

213 data = localeTools.parseString(data.decode('utf-8'), filename)

214 if data:

215 reference[filename] = data

216

217 for locale in params['locales']:

218 for file in reference.iterkeys():

219 path = 'chrome/locale/%s/%s' % (locale, file)

220 if path in files:

221 data = localeTools.parseString(files[path].decode('utf-8'), path )

222 for key, value in reference[file].iteritems():

223 if not key in data:

224 files[path] += localeTools.generateStringEntry(key, valu e, path).encode('utf-8')

225 else:

226 files[path] = reference[file]['_origData'].encode('utf-8')

227

228

229 def processJSONFiles(params, files):

230 prefix = 'lib/'

231 for name, content in files.iteritems():

232 if name.startswith(prefix) and name.endswith('.json'):

233 params['jsonRequires'][name[len(prefix):]] = json.loads(content)

234 for name in params['jsonRequires'].iterkeys():

235 del files[prefix + name]

236

237

238 def addMissingFiles(params, files):

239 templateData = {

240 'hasChrome': False,

241 'hasChromeRequires': False,

242 'hasShutdownHandlers': False,

243 'chromeWindows': [],

244 'requires': set(),

245 'jsonRequires': params['jsonRequires'],

246 'metadata': params['metadata'],

247 'hasWebExtension': params['hasWebExtension'],

248 'multicompartment': params['multicompartment'],

249 'applications': dict((v, k) for k, v in KNOWN_APPS.iteritems()),

250 }

251

252 def checkScript(name):

253 content = files[name]

254 for match in re.finditer(r'(?:^\|\s)require$\s"([\w\-]+)"\s$', conten t):

255 templateData['requires'].add(match.group(1))

256 if name.startswith('chrome/content/'):

257 templateData['hasChromeRequires'] = True

258 if not '/' in name or name.startswith('lib/'):

259 if re.search(r'(?:^\|\s)onShutdown\.', content):

260 templateData['hasShutdownHandlers'] = True

261

262 for name, content in files.iteritems():

263 if name == 'chrome.manifest':

264 templateData['hasChrome'] = True

265 elif name.endswith('.js'):

266 checkScript(name)

267 elif name.endswith('.xul'):

268 match = re.search(r'<(?:window\|dialog)\s[^>]*\bwindowtype="([^">]+)" ', content)

269 if match:

270 templateData['chromeWindows'].append(match.group(1))

271

272 while True:

273 missing = []

274 for module in templateData['requires']:

275 moduleFile = 'lib/' + module + '.js'

276 if not moduleFile in files:

277 import buildtools

278 path = os.path.join(buildtools.__path__[0], moduleFile)

279 if os.path.exists(path):

280 missing.append((path, moduleFile))

281 if not len(missing):

282 break

283 for path, moduleFile in missing:

284 files.read(path, moduleFile)

285 checkScript(moduleFile)

286

287 template = getTemplate('bootstrap.js.tmpl')

288 files['bootstrap.js'] = template.render(templateData).encode('utf-8')

289

290

291 def createBuild(baseDir, type='gecko', outFile=None, locales=None, buildNum=None , releaseBuild=False, multicompartment=False):

292 if locales == None:

293 locales = getLocales(baseDir)

294 elif locales == 'all':

295 locales = getLocales(baseDir, True)

296

297 metadata = readMetadata(baseDir, type)

298 version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum)

299

300 if outFile == None:

301 outFile = getDefaultFileName(metadata, version, 'xpi')

302

303 contributors = getContributors(metadata)

304

305 params = {

306 'baseDir': baseDir,

307 'locales': locales,

308 'releaseBuild': releaseBuild,

309 'version': version.encode('utf-8'),

310 'metadata': metadata,

311 'contributors': contributors,

312 'multicompartment': multicompartment,

313 'hasWebExtension': os.path.isdir(os.path.join(baseDir, 'webextension')),

314 'jsonRequires': {},

315 }

316

317 mapped = metadata.items('mapping') if metadata.has_section('mapping') else [ ]

318 skip = [opt for opt, _ in mapped] + ['chrome']

319 files = Files(getPackageFiles(params), getIgnoredFiles(params),

320 process=lambda path, data: processFile(path, data, params))

321 files['install.rdf'] = createManifest(params)

322 files.readMappedFiles(mapped)

323 files.read(baseDir, skip=skip)

324 for name, path in getChromeSubdirs(baseDir, params['locales']).iteritems():

325 if os.path.isdir(path):

326 files.read(path, 'chrome/%s' % name, skip=skip)

327 importLocales(params, files)

328 fixupLocales(params, files)

329 processJSONFiles(params, files)

330 if not 'bootstrap.js' in files:

331 addMissingFiles(params, files)

332 if metadata.has_section('preprocess'):

333 files.preprocess([f for f, _ in metadata.items('preprocess')])

334 files.zip(outFile, sortKey=lambda x: '!' if x == 'META-INF/zigbert.rsa' else x)

335

336

337 def autoInstall(baseDir, type, host, port, multicompartment=False):

338 fileBuffer = StringIO()

339 createBuild(baseDir, type=type, outFile=fileBuffer, multicompartment=multico mpartment)

340 urllib.urlopen('http://%s:%s/' % (host, port), data=fileBuffer.getvalue())

OLD	NEW

« localeTools.py ('K') | « packagerChrome.py ('k') | releaseAutomation.py » ('j') | releaseAutomation.py » ('J')