Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: packagerChrome.py

Issue 29549786: Issue 5535 - Replace our module system with webpack (Closed)
Left Patch Set: Fixed tox errors Created Oct. 6, 2017, 1:44 p.m.
Right Patch Set: Addressed final nits Created Oct. 10, 2017, 5:02 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « package-lock.json ('k') | packagerEdge.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # This Source Code Form is subject to the terms of the Mozilla Public 1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this 2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
5 import errno 5 import errno
6 import glob
6 import io 7 import io
7 import json 8 import json
8 import os 9 import os
9 import re 10 import re
10 from StringIO import StringIO 11 from StringIO import StringIO
11 import struct 12 import struct
12 import subprocess 13 import subprocess
13 import sys 14 import sys
14 15
15 from ensure_dependencies import read_deps
16 from packager import (readMetadata, getDefaultFileName, getBuildVersion, 16 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
17 getTemplate, Files) 17 getTemplate, Files)
18 18
19 defaultLocale = 'en_US' 19 defaultLocale = 'en_US'
20 20
21 21
22 def getIgnoredFiles(params): 22 def getIgnoredFiles(params):
23 return {'store.description'} 23 return {'store.description'}
24 24
25 25
(...skipping 135 matching lines...) Expand 10 before | Expand all | Expand 10 after
161 # do that consistently this can be removed. See issues 5760, 5761 and 5762. 161 # do that consistently this can be removed. See issues 5760, 5761 and 5762.
162 resolve_paths = [os.path.join(base_extension_path, dir, 'lib') 162 resolve_paths = [os.path.join(base_extension_path, dir, 'lib')
163 for dir in ['', 'adblockpluscore', 'adblockplusui']] 163 for dir in ['', 'adblockpluscore', 'adblockplusui']]
164 164
165 info_template = getTemplate(info_templates[params['type']]) 165 info_template = getTemplate(info_templates[params['type']])
166 info_module = info_template.render( 166 info_module = info_template.render(
167 basename=params['metadata'].get('general', 'basename'), 167 basename=params['metadata'].get('general', 'basename'),
168 version=params['metadata'].get('general', 'version') 168 version=params['metadata'].get('general', 'version')
169 ).encode('utf-8') 169 ).encode('utf-8')
170 170
171 boundary = '=============================WEBPACK-BOUNDARY'
172 configuration = { 171 configuration = {
173 'BOUNDARY': boundary, 172 'bundles': [],
174 'BUNDLES': [], 173 'extension_path': base_extension_path,
175 'EXTENSION_PATH': base_extension_path, 174 'info_module': info_module,
176 'INFO_MODULE': info_module, 175 'resolve_paths': resolve_paths,
177 'RESOLVE_PATHS': resolve_paths,
Wladimir Palant 2017/10/09 10:54:54 Nit: it's a JSON structure, why are properties all
kzar 2017/10/09 13:52:14 Done.
178 } 176 }
179 177
180 for item in params['metadata'].items('bundles'): 178 for item in params['metadata'].items('bundles'):
181 name, value = item 179 name, value = item
182 base_item_path = os.path.dirname(item.source) 180 base_item_path = os.path.dirname(item.source)
183 181
184 bundle_file = os.path.relpath(os.path.join(base_item_path, name), 182 bundle_file = os.path.relpath(os.path.join(base_item_path, name),
185 base_extension_path) 183 base_extension_path)
186 entry_files = [os.path.join(base_item_path, module_path) 184 entry_files = [os.path.join(base_item_path, module_path)
187 for module_path in value.split()] 185 for module_path in value.split()]
188 configuration['BUNDLES'].append({ 186 configuration['bundles'].append({
189 'BUNDLE_NAME': bundle_file, 187 'bundle_name': bundle_file,
190 'ENTRY_POINTS': entry_files, 188 'entry_points': entry_files,
191 }) 189 })
192 190
193 output = subprocess.check_output( 191 cmd = ['node', os.path.join(os.path.dirname(__file__), 'webpack_runner.js')]
194 ['node', 192 process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
195 os.path.join(os.path.dirname(__file__), 'webpack_runner.js'), 193 stdin=subprocess.PIPE)
196 toJson(configuration)] 194 output = process.communicate(input=toJson(configuration))[0]
Wladimir Palant 2017/10/09 10:54:55 Passing the configuration as parameter has an ugly
kzar 2017/10/09 13:52:15 Done, you're right the error output looks way nice
197 ).split(boundary) 195 if process.returncode != 0:
198 for i in range(0, len(output) - 1, 2): 196 raise subprocess.CalledProcessError(process.returncode, cmd=cmd)
199 files[output[i].strip()] = output[i + 1].strip() 197
200 198 bundles = json.loads(output)
201 199 for bundle in bundles:
202 def import_string_webext(data, key, source): 200 files[bundle] = bundles[bundle].encode('utf-8')
203 """Import a single translation from the source dictionary into data"""
204 data[key] = source
205
206
207 def import_string_gecko(data, key, value):
208 """Import Gecko-style locales into data.
209
210 Only sets {'message': value} in the data-dictionary, after stripping
211 undesired Gecko-style access keys.
212 """
213 match = re.search(r'^(.*?)\s*\(&.\)$', value)
214 if match:
215 value = match.group(1)
216 else:
217 index = value.find('&')
218 if index >= 0:
219 value = value[0:index] + value[index + 1:]
220
221 data[key] = {'message': value}
222 201
223 202
224 def import_locales(params, files): 203 def import_locales(params, files):
225 import localeTools 204 for item in params['metadata'].items('import_locales'):
226 205 filename, keys = item
227 # FIXME: localeTools doesn't use real Chrome locales, it uses dash as 206 for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
228 # separator instead. 207 *filename.split('/'))):
229 convert_locale_code = lambda code: code.replace('-', '_') 208 locale = sourceFile.split(os.path.sep)[-2]
230 209 targetFile = os.path.join('_locales', locale, 'messages.json')
231 # We need to map Chrome locales to Gecko locales. Start by mapping Chrome 210 data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
232 # locales to themselves, merely with the dash as separator.
233 locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocal es}
234
235 # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
236 for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems ():
237 locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale
238
239 # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
240 reverse_mapping = {v: k for k, v in locale_mapping.iteritems()}
241 for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems() :
242 if crowdin_locale in reverse_mapping:
243 locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale
244
245 for target, source in locale_mapping.iteritems():
246 targetFile = '_locales/%s/messages.json' % target
247 if not targetFile in files:
248 continue
249
250 for item in params['metadata'].items('import_locales'):
251 fileName, keys = item
252 parts = map(lambda n: source if n == '*' else n, fileName.split('/') )
253 sourceFile = os.path.join(os.path.dirname(item.source), *parts)
254 incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incom plete')
255 if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker ):
256 continue
257
258 data = json.loads(files[targetFile].decode('utf-8'))
259 211
260 try: 212 try:
261 # The WebExtensions (.json) and Gecko format provide 213 with io.open(sourceFile, 'r', encoding='utf-8') as handle:
262 # translations differently and/or provide additional 214 sourceData = json.load(handle)
263 # information like e.g. "placeholders". We want to adhere to
264 # that and preserve the addtional info.
265 if sourceFile.endswith('.json'):
266 with io.open(sourceFile, 'r', encoding='utf-8') as handle:
267 sourceData = json.load(handle)
268 import_string = import_string_webext
269 else:
270 sourceData = localeTools.readFile(sourceFile)
271 import_string = import_string_gecko
272 215
273 # Resolve wildcard imports 216 # Resolve wildcard imports
274 if keys == '*' or keys == '=*': 217 if keys == '*':
275 importList = sourceData.keys() 218 importList = sourceData.keys()
276 importList = filter(lambda k: not k.startswith('_'), importL ist) 219 importList = filter(lambda k: not k.startswith('_'), importL ist)
277 if keys == '=*':
278 importList = map(lambda k: '=' + k, importList)
279 keys = ' '.join(importList) 220 keys = ' '.join(importList)
280 221
281 for stringID in keys.split(): 222 for stringID in keys.split():
282 noMangling = False
283 if stringID.startswith('='):
284 stringID = stringID[1:]
285 noMangling = True
286
287 if stringID in sourceData: 223 if stringID in sourceData:
288 if noMangling: 224 if stringID in data:
289 key = re.sub(r'\W', '_', stringID) 225 print ('Warning: locale string {} defined multiple'
290 else: 226 ' times').format(stringID)
291 key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub( r'\W', '_', stringID) 227
292 if key in data: 228 data[stringID] = sourceData[stringID]
293 print 'Warning: locale string %s defined multiple ti mes' % key
294
295 import_string(data, key, sourceData[stringID])
296 except Exception as e: 229 except Exception as e:
297 print 'Warning: error importing locale data from %s: %s' % (sour ceFile, e) 230 print 'Warning: error importing locale data from %s: %s' % (sour ceFile, e)
298 231
299 files[targetFile] = toJson(data) 232 files[targetFile] = toJson(data)
300 233
301 234
302 def truncate(text, length_limit): 235 def truncate(text, length_limit):
303 if len(text) <= length_limit: 236 if len(text) <= length_limit:
304 return text 237 return text
305 return text[:length_limit - 1].rstrip() + u'\u2026' 238 return text[:length_limit - 1].rstrip() + u'\u2026'
306 239
307 240
308 def fixTranslationsForCWS(files): 241 def fix_translations_for_chrome(files):
309 # Chrome Web Store requires messages used in manifest.json to be present in
310 # all languages. It also enforces length limits for extension names and
311 # descriptions.
312 defaults = {} 242 defaults = {}
313 data = json.loads(files['_locales/%s/messages.json' % defaultLocale]) 243 data = json.loads(files['_locales/%s/messages.json' % defaultLocale])
314 for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']): 244 for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
315 name = match.group(1) 245 name = match.group(1)
316 defaults[name] = data[name] 246 defaults[name] = data[name]
317 247
318 limits = {} 248 limits = {}
319 manifest = json.loads(files['manifest.json']) 249 manifest = json.loads(files['manifest.json'])
320 for key, limit in (('name', 45), ('description', 132), ('short_name', 12)): 250 for key, limit in (('name', 45), ('description', 132), ('short_name', 12)):
321 match = re.search(r'__MSG_(\S+)__', manifest.get(key, '')) 251 match = re.search(r'__MSG_(\S+)__', manifest.get(key, ''))
322 if match: 252 if match:
323 limits[match.group(1)] = limit 253 limits[match.group(1)] = limit
324 254
325 for filename in files: 255 for path in list(files):
326 if not filename.startswith('_locales/') or not filename.endswith('/messa ges.json'): 256 match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
257 if not match:
327 continue 258 continue
328 259
329 data = json.loads(files[filename]) 260 # The Chrome Web Store requires messages used in manifest.json to
330 for name, info in defaults.iteritems(): 261 # be present in all languages, and enforces length limits on
331 data.setdefault(name, info) 262 # extension name and description.
332 for name, limit in limits.iteritems(): 263 is_latam, is_mexican, filename = match.groups()
333 if name in data: 264 if filename == 'messages.json':
334 data[name]['message'] = truncate(data[name]['message'], limit) 265 data = json.loads(files[path])
335 files[filename] = toJson(data) 266 for name, info in defaults.iteritems():
267 data.setdefault(name, info)
268 for name, limit in limits.iteritems():
269 info = data.get(name)
270 if info:
271 info['message'] = truncate(info['message'], limit)
272 files[path] = toJson(data)
273
274 # Chrome combines Latin American dialects of Spanish into es-419.
275 if is_latam:
276 data = files.pop(path)
277 if is_mexican:
278 files['_locales/es_419/' + filename] = data
336 279
337 280
338 def signBinary(zipdata, keyFile): 281 def signBinary(zipdata, keyFile):
339 from Crypto.Hash import SHA 282 from Crypto.Hash import SHA
340 from Crypto.PublicKey import RSA 283 from Crypto.PublicKey import RSA
341 from Crypto.Signature import PKCS1_v1_5 284 from Crypto.Signature import PKCS1_v1_5
342 285
343 try: 286 try:
344 with open(keyFile, 'rb') as file: 287 with open(keyFile, 'rb') as file:
345 key = RSA.importKey(file.read()) 288 key = RSA.importKey(file.read())
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after
405 files.preprocess( 348 files.preprocess(
406 [f for f, _ in metadata.items('preprocess')], 349 [f for f, _ in metadata.items('preprocess')],
407 {'needsExt': True} 350 {'needsExt': True}
408 ) 351 )
409 352
410 if metadata.has_section('import_locales'): 353 if metadata.has_section('import_locales'):
411 import_locales(params, files) 354 import_locales(params, files)
412 355
413 files['manifest.json'] = createManifest(params, files) 356 files['manifest.json'] = createManifest(params, files)
414 if type == 'chrome': 357 if type == 'chrome':
415 fixTranslationsForCWS(files) 358 fix_translations_for_chrome(files)
416 359
417 if devenv: 360 if devenv:
418 import buildtools 361 import buildtools
419 import random 362 import random
420 files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js '), relpath='devenvPoller__.js') 363 files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js '), relpath='devenvPoller__.js')
421 files['devenvVersion__'] = str(random.random()) 364 files['devenvVersion__'] = str(random.random())
422 365
423 if metadata.has_option('general', 'testScripts'): 366 if metadata.has_option('general', 'testScripts'):
424 files['qunit/index.html'] = createScriptPage( 367 files['qunit/index.html'] = createScriptPage(
425 params, 'testIndex.html.tmpl', ('general', 'testScripts') 368 params, 'testIndex.html.tmpl', ('general', 'testScripts')
426 ) 369 )
427 370
428 zipdata = files.zipToString() 371 zipdata = files.zipToString()
429 signature = None 372 signature = None
430 pubkey = None 373 pubkey = None
431 if keyFile != None: 374 if keyFile != None:
432 signature = signBinary(zipdata, keyFile) 375 signature = signBinary(zipdata, keyFile)
433 pubkey = getPublicKey(keyFile) 376 pubkey = getPublicKey(keyFile)
434 writePackage(outFile, pubkey, signature, zipdata) 377 writePackage(outFile, pubkey, signature, zipdata)
LEFTRIGHT

Powered by Google App Engine
This is Rietveld