Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: packagerChrome.py

Issue 29501558: Issue 5383 - Add tests for the Chrome and Firefox packagers (Closed)
Left Patch Set: Simplifying parameters Created Sept. 21, 2017, 11:27 a.m.
Right Patch Set: Addressing Vasily's comments Created Oct. 22, 2017, 11:11 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « package.json ('k') | tests/README.md » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # This Source Code Form is subject to the terms of the Mozilla Public 1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this 2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
5 import errno 5 import errno
6 import glob
6 import io 7 import io
7 import json 8 import json
8 import os 9 import os
9 import re 10 import re
10 from StringIO import StringIO
11 import struct 11 import struct
12 import subprocess
12 import sys 13 import sys
13 import collections 14 import random
14 15
15 from packager import (readMetadata, getDefaultFileName, getBuildVersion, 16 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
16 getTemplate, Files) 17 getTemplate, Files)
17 18
18 defaultLocale = 'en_US' 19 defaultLocale = 'en_US'
19 20
20 21
21 def getIgnoredFiles(params): 22 def getIgnoredFiles(params):
22 return {'store.description'} 23 return {'store.description'}
23 24
(...skipping 20 matching lines...) Expand all
44 45
45 def makeIcons(files, filenames): 46 def makeIcons(files, filenames):
46 icons = {} 47 icons = {}
47 for filename in filenames: 48 for filename in filenames:
48 try: 49 try:
49 magic, width, height = struct.unpack_from('>8s8xii', 50 magic, width, height = struct.unpack_from('>8s8xii',
50 files[filename]) 51 files[filename])
51 except struct.error: 52 except struct.error:
52 magic = None 53 magic = None
53 if magic != '\x89PNG\r\n\x1a\n': 54 if magic != '\x89PNG\r\n\x1a\n':
54 raise TypeError(filename + ' is no valid PNG.') 55 raise Exception(filename + ' is no valid PNG.')
Sebastian Noack 2017/09/21 19:09:03 The definition of TypeError doesn't seem to apply
tlucas 2017/09/22 09:02:18 Done.
55 if(width != height): 56 if(width != height):
56 print >>sys.stderr, 'Warning: %s size is %ix%i, icon should be squar e' % (filename, width, height) 57 print >>sys.stderr, 'Warning: %s size is %ix%i, icon should be squar e' % (filename, width, height)
57 icons[width] = filename 58 icons[width] = filename
58 return icons 59 return icons
59 60
60 61
61 def createScriptPage(params, template_name, script_option): 62 def createScriptPage(params, template_name, script_option):
62 template = getTemplate(template_name, autoEscape=True) 63 template = getTemplate(template_name, autoEscape=True)
63 return template.render( 64 return template.render(
64 basename=params['metadata'].get('general', 'basename'), 65 basename=params['metadata'].get('general', 'basename'),
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
134 # Normalize JSON structure 135 # Normalize JSON structure
135 licenseComment = re.compile(r'/\*.*?\*/', re.S) 136 licenseComment = re.compile(r'/\*.*?\*/', re.S)
136 data = json.loads(re.sub(licenseComment, '', manifest, 1)) 137 data = json.loads(re.sub(licenseComment, '', manifest, 1))
137 if '_dummy' in data: 138 if '_dummy' in data:
138 del data['_dummy'] 139 del data['_dummy']
139 manifest = json.dumps(data, sort_keys=True, indent=2) 140 manifest = json.dumps(data, sort_keys=True, indent=2)
140 141
141 return manifest.encode('utf-8') 142 return manifest.encode('utf-8')
142 143
143 144
144 def convertJS(params, files):
145 output_files = collections.OrderedDict()
146 args = {}
147
148 for item in params['metadata'].items('convert_js'):
149 name, value = item
150 filename, arg = re.search(r'^(.*?)(?:\[(.*)\])?$', name).groups()
151 if arg is None:
152 output_files[filename] = (value.split(), item.source)
153 else:
154 args.setdefault(filename, {})[arg] = value
155
156 template = getTemplate('modules.js.tmpl')
157
158 for filename, (input_files, origin) in output_files.iteritems():
159 if '/' in filename and not files.isIncluded(filename):
160 continue
161
162 current_args = args.get(filename, {})
163 current_args['autoload'] = [module for module in
164 current_args.get('autoload', '').split(',')
165 if module != '']
166
167 base_dir = os.path.dirname(origin)
168 modules = []
169
170 for input_filename in input_files:
171 module_name = os.path.splitext(os.path.basename(input_filename))[0]
172 prefix = os.path.basename(os.path.dirname(input_filename))
173 if prefix != 'lib':
174 module_name = '{}_{}'.format(prefix, module_name)
175 with open(os.path.join(base_dir, input_filename), 'r') as file:
176 modules.append((module_name, file.read().decode('utf-8')))
177 files.pop(input_filename, None)
178
179 files[filename] = template.render(
180 args=current_args,
181 basename=params['metadata'].get('general', 'basename'),
182 modules=modules,
183 type=params['type'],
184 version=params['metadata'].get('general', 'version')
185 ).encode('utf-8')
186
187
188 def toJson(data): 145 def toJson(data):
189 return json.dumps( 146 return json.dumps(
190 data, ensure_ascii=False, sort_keys=True, 147 data, ensure_ascii=False, sort_keys=True,
191 indent=2, separators=(',', ': ') 148 indent=2, separators=(',', ': ')
192 ).encode('utf-8') + '\n' 149 ).encode('utf-8') + '\n'
193 150
194 151
195 def import_string_webext(data, key, source): 152 def create_bundles(params, files):
196 """Import a single translation from the source dictionary into data""" 153 base_extension_path = params['baseDir']
197 data[key] = source 154 info_templates = {
198 155 'chrome': 'chromeInfo.js.tmpl',
199 156 'edge': 'edgeInfo.js.tmpl',
200 def import_string_gecko(data, key, value): 157 'gecko': 'geckoInfo.js.tmpl'
201 """Import Gecko-style locales into data. 158 }
202 159
203 Only sets {'message': value} in the data-dictionary, after stripping 160 # Historically we didn't use relative paths when requiring modules, so in
204 undesired Gecko-style access keys. 161 # order for webpack to know where to find them we need to pass in a list of
205 """ 162 # resolve paths. Going forward we should always use relative paths, once we
206 match = re.search(r'^(.*?)\s*\(&.\)$', value) 163 # do that consistently this can be removed. See issues 5760, 5761 and 5762.
207 if match: 164 resolve_paths = [os.path.join(base_extension_path, dir, 'lib')
208 value = match.group(1) 165 for dir in ['', 'adblockpluscore', 'adblockplusui']]
209 else: 166
210 index = value.find('&') 167 info_template = getTemplate(info_templates[params['type']])
211 if index >= 0: 168 info_module = info_template.render(
212 value = value[0:index] + value[index + 1:] 169 basename=params['metadata'].get('general', 'basename'),
213 170 version=params['metadata'].get('general', 'version')
214 data[key] = {'message': value} 171 ).encode('utf-8')
172
173 configuration = {
174 'bundles': [],
175 'extension_path': base_extension_path,
176 'info_module': info_module,
177 'resolve_paths': resolve_paths,
178 }
179
180 for item in params['metadata'].items('bundles'):
181 name, value = item
182 base_item_path = os.path.dirname(item.source)
183
184 bundle_file = os.path.relpath(os.path.join(base_item_path, name),
185 base_extension_path)
186 entry_files = [os.path.join(base_item_path, module_path)
187 for module_path in value.split()]
188 configuration['bundles'].append({
189 'bundle_name': bundle_file,
190 'entry_points': entry_files,
191 })
192
193 cmd = ['node', os.path.join(os.path.dirname(__file__), 'webpack_runner.js')]
194 process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
195 stdin=subprocess.PIPE)
196 output = process.communicate(input=toJson(configuration))[0]
197 if process.returncode != 0:
198 raise subprocess.CalledProcessError(process.returncode, cmd=cmd)
199 output = json.loads(output)
200
201 # Clear the mapping for any files included in a bundle, to avoid them being
202 # duplicated in the build.
203 for to_ignore in output['included']:
204 files.pop(to_ignore, None)
205
206 for bundle in output['files']:
207 files[bundle] = output['files'][bundle].encode('utf-8')
215 208
216 209
217 def import_locales(params, files): 210 def import_locales(params, files):
218 import localeTools 211 for item in params['metadata'].items('import_locales'):
219 212 filename, keys = item
220 # FIXME: localeTools doesn't use real Chrome locales, it uses dash as 213 for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
221 # separator instead. 214 *filename.split('/'))):
222 convert_locale_code = lambda code: code.replace('-', '_') 215 locale = sourceFile.split(os.path.sep)[-2]
223 216 targetFile = os.path.join('_locales', locale, 'messages.json')
224 # We need to map Chrome locales to Gecko locales. Start by mapping Chrome 217 data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
225 # locales to themselves, merely with the dash as separator.
226 locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocal es}
227
228 # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
229 for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems ():
230 locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale
231
232 # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
233 reverse_mapping = {v: k for k, v in locale_mapping.iteritems()}
234 for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems() :
235 if crowdin_locale in reverse_mapping:
236 locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale
237
238 for target, source in locale_mapping.iteritems():
239 targetFile = '_locales/%s/messages.json' % target
240 if not targetFile in files:
241 continue
242
243 for item in params['metadata'].items('import_locales'):
244 fileName, keys = item
245 parts = map(lambda n: source if n == '*' else n, fileName.split('/') )
246 sourceFile = os.path.join(os.path.dirname(item.source), *parts)
247 incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incom plete')
248 if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker ):
249 continue
250
251 data = json.loads(files[targetFile].decode('utf-8'))
252 218
253 try: 219 try:
254 # The WebExtensions (.json) and Gecko format provide 220 with io.open(sourceFile, 'r', encoding='utf-8') as handle:
255 # translations differently and/or provide additional 221 sourceData = json.load(handle)
256 # information like e.g. "placeholders". We want to adhere to
257 # that and preserve the addtional info.
258 if sourceFile.endswith('.json'):
259 with io.open(sourceFile, 'r', encoding='utf-8') as handle:
260 sourceData = json.load(handle)
261 import_string = import_string_webext
262 else:
263 sourceData = localeTools.readFile(sourceFile)
264 import_string = import_string_gecko
265 222
266 # Resolve wildcard imports 223 # Resolve wildcard imports
267 if keys == '*' or keys == '=*': 224 if keys == '*':
268 importList = sourceData.keys() 225 importList = sourceData.keys()
269 importList = filter(lambda k: not k.startswith('_'), importL ist) 226 importList = filter(lambda k: not k.startswith('_'), importL ist)
270 if keys == '=*':
271 importList = map(lambda k: '=' + k, importList)
272 keys = ' '.join(importList) 227 keys = ' '.join(importList)
273 228
274 for stringID in keys.split(): 229 for stringID in keys.split():
275 noMangling = False
276 if stringID.startswith('='):
277 stringID = stringID[1:]
278 noMangling = True
279
280 if stringID in sourceData: 230 if stringID in sourceData:
281 if noMangling: 231 if stringID in data:
282 key = re.sub(r'\W', '_', stringID) 232 print ('Warning: locale string {} defined multiple'
283 else: 233 ' times').format(stringID)
284 key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub( r'\W', '_', stringID) 234
285 if key in data: 235 data[stringID] = sourceData[stringID]
286 print 'Warning: locale string %s defined multiple ti mes' % key
287
288 import_string(data, key, sourceData[stringID])
289 except Exception as e: 236 except Exception as e:
290 print 'Warning: error importing locale data from %s: %s' % (sour ceFile, e) 237 print 'Warning: error importing locale data from %s: %s' % (sour ceFile, e)
291 238
292 files[targetFile] = toJson(data) 239 files[targetFile] = toJson(data)
293 240
294 241
295 def truncate(text, length_limit): 242 def truncate(text, length_limit):
296 if len(text) <= length_limit: 243 if len(text) <= length_limit:
297 return text 244 return text
298 return text[:length_limit - 1].rstrip() + u'\u2026' 245 return text[:length_limit - 1].rstrip() + u'\u2026'
299 246
300 247
301 def fixTranslationsForCWS(files): 248 def fix_translations_for_chrome(files):
302 # Chrome Web Store requires messages used in manifest.json to be present in
303 # all languages. It also enforces length limits for extension names and
304 # descriptions.
305 defaults = {} 249 defaults = {}
306 data = json.loads(files['_locales/%s/messages.json' % defaultLocale]) 250 data = json.loads(files['_locales/%s/messages.json' % defaultLocale])
307 for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']): 251 for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
308 name = match.group(1) 252 name = match.group(1)
309 defaults[name] = data[name] 253 defaults[name] = data[name]
310 254
311 limits = {} 255 limits = {}
312 manifest = json.loads(files['manifest.json']) 256 manifest = json.loads(files['manifest.json'])
313 for key, limit in (('name', 45), ('description', 132), ('short_name', 12)): 257 for key, limit in (('name', 45), ('description', 132), ('short_name', 12)):
314 match = re.search(r'__MSG_(\S+)__', manifest.get(key, '')) 258 match = re.search(r'__MSG_(\S+)__', manifest.get(key, ''))
315 if match: 259 if match:
316 limits[match.group(1)] = limit 260 limits[match.group(1)] = limit
317 261
318 for filename in files: 262 for path in list(files):
319 if not filename.startswith('_locales/') or not filename.endswith('/messa ges.json'): 263 match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
264 if not match:
320 continue 265 continue
321 266
322 data = json.loads(files[filename]) 267 # The Chrome Web Store requires messages used in manifest.json to
323 for name, info in defaults.iteritems(): 268 # be present in all languages, and enforces length limits on
324 data.setdefault(name, info) 269 # extension name and description.
325 for name, limit in limits.iteritems(): 270 is_latam, is_mexican, filename = match.groups()
326 if name in data: 271 if filename == 'messages.json':
327 data[name]['message'] = truncate(data[name]['message'], limit) 272 data = json.loads(files[path])
328 files[filename] = toJson(data) 273 for name, info in defaults.iteritems():
274 data.setdefault(name, info)
275 for name, limit in limits.iteritems():
276 info = data.get(name)
277 if info:
278 info['message'] = truncate(info['message'], limit)
279 files[path] = toJson(data)
280
281 # Chrome combines Latin American dialects of Spanish into es-419.
282 if is_latam:
283 data = files.pop(path)
284 if is_mexican:
285 files['_locales/es_419/' + filename] = data
329 286
330 287
331 def signBinary(zipdata, keyFile): 288 def signBinary(zipdata, keyFile):
332 from Crypto.Hash import SHA 289 from Crypto.Hash import SHA
333 from Crypto.PublicKey import RSA 290 from Crypto.PublicKey import RSA
334 from Crypto.Signature import PKCS1_v1_5 291 from Crypto.Signature import PKCS1_v1_5
335 292
336 try: 293 try:
337 with open(keyFile, 'rb') as file: 294 with open(keyFile, 'rb') as file:
338 key = RSA.importKey(file.read()) 295 key = RSA.importKey(file.read())
(...skipping 18 matching lines...) Expand all
357 file = open(outputFile, 'wb') 314 file = open(outputFile, 'wb')
358 else: 315 else:
359 file = outputFile 316 file = outputFile
360 if pubkey != None and signature != None: 317 if pubkey != None and signature != None:
361 file.write(struct.pack('<4sIII', 'Cr24', 2, len(pubkey), len(signature)) ) 318 file.write(struct.pack('<4sIII', 'Cr24', 2, len(pubkey), len(signature)) )
362 file.write(pubkey) 319 file.write(pubkey)
363 file.write(signature) 320 file.write(signature)
364 file.write(zipdata) 321 file.write(zipdata)
365 322
366 323
324 def add_devenv_requirements(files, metadata, params):
325 files.read(
326 os.path.join(os.path.dirname(__file__), 'chromeDevenvPoller__.js'),
327 relpath='devenvPoller__.js',
328 )
329 files['devenvVersion__'] = str(random.random())
330
331 if metadata.has_option('general', 'testScripts'):
332 files['qunit/index.html'] = createScriptPage(
333 params, 'testIndex.html.tmpl', ('general', 'testScripts')
334 )
335
336
367 def createBuild(baseDir, type='chrome', outFile=None, buildNum=None, releaseBuil d=False, keyFile=None, devenv=False): 337 def createBuild(baseDir, type='chrome', outFile=None, buildNum=None, releaseBuil d=False, keyFile=None, devenv=False):
368 metadata = readMetadata(baseDir, type) 338 metadata = readMetadata(baseDir, type)
369 version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum) 339 version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum)
370 340
371 if outFile == None: 341 if outFile == None:
372 if type == 'gecko-webext': 342 if type == 'gecko':
373 file_extension = 'xpi' 343 file_extension = 'xpi'
374 else: 344 else:
375 file_extension = 'crx' if keyFile else 'zip' 345 file_extension = 'crx' if keyFile else 'zip'
376 outFile = getDefaultFileName(metadata, version, file_extension) 346 outFile = getDefaultFileName(metadata, version, file_extension)
377 347
378 params = { 348 params = {
379 'type': type, 349 'type': type,
380 'baseDir': baseDir, 350 'baseDir': baseDir,
381 'releaseBuild': releaseBuild, 351 'releaseBuild': releaseBuild,
382 'version': version, 352 'version': version,
383 'devenv': devenv, 353 'devenv': devenv,
384 'metadata': metadata, 354 'metadata': metadata,
385 } 355 }
386 356
387 mapped = metadata.items('mapping') if metadata.has_section('mapping') else [ ] 357 mapped = metadata.items('mapping') if metadata.has_section('mapping') else [ ]
388 files = Files(getPackageFiles(params), getIgnoredFiles(params), 358 files = Files(getPackageFiles(params), getIgnoredFiles(params),
389 process=lambda path, data: processFile(path, data, params)) 359 process=lambda path, data: processFile(path, data, params))
390 360
391 files.readMappedFiles(mapped) 361 files.readMappedFiles(mapped)
392 files.read(baseDir, skip=[opt for opt, _ in mapped]) 362 files.read(baseDir, skip=[opt for opt, _ in mapped])
393 363
394 if metadata.has_section('convert_js'): 364 if metadata.has_section('bundles'):
395 convertJS(params, files) 365 create_bundles(params, files)
396 366
397 if metadata.has_section('preprocess'): 367 if metadata.has_section('preprocess'):
398 files.preprocess( 368 files.preprocess(
399 [f for f, _ in metadata.items('preprocess')], 369 [f for f, _ in metadata.items('preprocess')],
400 {'needsExt': True} 370 {'needsExt': True}
401 ) 371 )
402 372
403 if metadata.has_section('import_locales'): 373 if metadata.has_section('import_locales'):
404 import_locales(params, files) 374 import_locales(params, files)
405 375
406 files['manifest.json'] = createManifest(params, files) 376 files['manifest.json'] = createManifest(params, files)
407 if type == 'chrome': 377 if type == 'chrome':
408 fixTranslationsForCWS(files) 378 fix_translations_for_chrome(files)
409 379
410 if devenv: 380 if devenv:
411 import buildtools 381 add_devenv_requirements(files, metadata, params)
412 import random
413 files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js '), relpath='devenvPoller__.js')
414 files['devenvVersion__'] = str(random.random())
415
416 if metadata.has_option('general', 'testScripts'):
417 files['qunit/index.html'] = createScriptPage(
418 params, 'testIndex.html.tmpl', ('general', 'testScripts')
419 )
420 382
421 zipdata = files.zipToString() 383 zipdata = files.zipToString()
422 signature = None 384 signature = None
423 pubkey = None 385 pubkey = None
424 if keyFile != None: 386 if keyFile != None:
425 signature = signBinary(zipdata, keyFile) 387 signature = signBinary(zipdata, keyFile)
426 pubkey = getPublicKey(keyFile) 388 pubkey = getPublicKey(keyFile)
427 writePackage(outFile, pubkey, signature, zipdata) 389 writePackage(outFile, pubkey, signature, zipdata)
LEFTRIGHT

Powered by Google App Engine
This is Rietveld