Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: packagerChrome.py

Issue 29549786: Issue 5535 - Replace our module system with webpack (Closed)
Left Patch Set: Improved resolve paths comment Created Sept. 24, 2017, 12:35 p.m.
Right Patch Set: Addressed final nits Created Oct. 10, 2017, 5:02 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « package-lock.json ('k') | packagerEdge.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # This Source Code Form is subject to the terms of the Mozilla Public 1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this 2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
5 import errno 5 import errno
6 import glob
6 import io 7 import io
7 import json 8 import json
8 import os 9 import os
9 import re 10 import re
10 import shutil
11 from StringIO import StringIO 11 from StringIO import StringIO
12 import struct 12 import struct
13 import subprocess 13 import subprocess
14 import sys 14 import sys
15 import tempfile 15
16
17 from ensure_dependencies import read_deps
18 from packager import (readMetadata, getDefaultFileName, getBuildVersion, 16 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
19 getTemplate, Files) 17 getTemplate, Files)
20 18
21 defaultLocale = 'en_US' 19 defaultLocale = 'en_US'
22 20
23 21
24 def getIgnoredFiles(params): 22 def getIgnoredFiles(params):
25 return {'store.description'} 23 return {'store.description'}
26 24
27 25
(...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after
135 # Normalize JSON structure 133 # Normalize JSON structure
136 licenseComment = re.compile(r'/\*.*?\*/', re.S) 134 licenseComment = re.compile(r'/\*.*?\*/', re.S)
137 data = json.loads(re.sub(licenseComment, '', manifest, 1)) 135 data = json.loads(re.sub(licenseComment, '', manifest, 1))
138 if '_dummy' in data: 136 if '_dummy' in data:
139 del data['_dummy'] 137 del data['_dummy']
140 manifest = json.dumps(data, sort_keys=True, indent=2) 138 manifest = json.dumps(data, sort_keys=True, indent=2)
141 139
142 return manifest.encode('utf-8') 140 return manifest.encode('utf-8')
143 141
144 142
143 def toJson(data):
144 return json.dumps(
145 data, ensure_ascii=False, sort_keys=True,
146 indent=2, separators=(',', ': ')
147 ).encode('utf-8') + '\n'
148
149
145 def create_bundles(params, files): 150 def create_bundles(params, files):
146 base_extension_path = params['baseDir'] 151 base_extension_path = params['baseDir']
147 info_templates = { 152 info_templates = {
148 'chrome': 'chromeInfo.js.tmpl', 153 'chrome': 'chromeInfo.js.tmpl',
149 'edge': 'edgeInfo.js.tmpl', 154 'edge': 'edgeInfo.js.tmpl',
150 'gecko-webext': 'geckoInfo.js.tmpl' 155 'gecko-webext': 'geckoInfo.js.tmpl'
151 } 156 }
152 info_module = None
153 157
154 # Historically we didn't use relative paths when requiring modules, so in 158 # Historically we didn't use relative paths when requiring modules, so in
155 # order for webpack to know where to find them we need to pass in a list of 159 # order for webpack to know where to find them we need to pass in a list of
156 # resolve paths. Going forward we should always use relative paths, once we 160 # resolve paths. Going forward we should always use relative paths, once we
157 # do that consistently this can be removed. See issues 5760, 5761 and 5762. 161 # do that consistently this can be removed. See issues 5760, 5761 and 5762.
158 resolve_paths = ' '.join( 162 resolve_paths = [os.path.join(base_extension_path, dir, 'lib')
159 [os.path.join(base_extension_path, dir, 'lib') 163 for dir in ['', 'adblockpluscore', 'adblockplusui']]
160 for dir in ['', 'adblockpluscore', 'adblockplusui']] 164
161 ) 165 info_template = getTemplate(info_templates[params['type']])
162 166 info_module = info_template.render(
163 temp_dir = tempfile.mkdtemp() 167 basename=params['metadata'].get('general', 'basename'),
164 try: 168 version=params['metadata'].get('general', 'version')
165 info_module = os.path.join(temp_dir, 'info.js') 169 ).encode('utf-8')
166 template = getTemplate(info_templates[params['type']]) 170
167 with open(info_module, 'w') as info_file: 171 configuration = {
168 info_file.write( 172 'bundles': [],
169 template.render( 173 'extension_path': base_extension_path,
170 basename=params['metadata'].get('general', 'basename'), 174 'info_module': info_module,
171 version=params['metadata'].get('general', 'version') 175 'resolve_paths': resolve_paths,
172 ).encode('utf-8') 176 }
173 ) 177
174 178 for item in params['metadata'].items('bundles'):
175 for item in params['metadata'].items('bundles'): 179 name, value = item
176 name, value = item 180 base_item_path = os.path.dirname(item.source)
177 base_item_path = os.path.dirname(item.source) 181
178 182 bundle_file = os.path.relpath(os.path.join(base_item_path, name),
179 bundle_file = os.path.relpath(os.path.join(base_item_path, name), 183 base_extension_path)
180 base_extension_path) 184 entry_files = [os.path.join(base_item_path, module_path)
181 entry_files = [ 185 for module_path in value.split()]
182 os.path.join( 186 configuration['bundles'].append({
183 '.', 187 'bundle_name': bundle_file,
184 os.path.relpath(os.path.join(base_item_path, module_path), 188 'entry_points': entry_files,
185 base_extension_path) 189 })
186 ) 190
187 for module_path in value.split() 191 cmd = ['node', os.path.join(os.path.dirname(__file__), 'webpack_runner.js')]
188 ] 192 process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
189 subprocess.check_call( 193 stdin=subprocess.PIPE)
190 ['npm', 'run-script', 'webpack', '--silent'], 194 output = process.communicate(input=toJson(configuration))[0]
191 cwd=os.path.dirname(__file__), 195 if process.returncode != 0:
192 env={ 196 raise subprocess.CalledProcessError(process.returncode, cmd=cmd)
193 'EXTENSION_PATH': base_extension_path, 197
194 'ENTRY_POINTS': ' '.join(entry_files), 198 bundles = json.loads(output)
195 'OUTPUT_PATH': temp_dir, 199 for bundle in bundles:
196 'BUNDLE_NAME': bundle_file, 200 files[bundle] = bundles[bundle].encode('utf-8')
197 'RESOLVE_PATHS': resolve_paths,
198 'INFO_PATH': info_module,
199 'PATH': os.environ['PATH']
200 }
201 )
202 for file_name in [bundle_file, bundle_file + '.map']:
203 with open(os.path.join(temp_dir, file_name), 'r') as f:
204 files[file_name] = f.read()
205 finally:
206 shutil.rmtree(temp_dir)
207
208
209 def toJson(data):
210 return json.dumps(
211 data, ensure_ascii=False, sort_keys=True,
212 indent=2, separators=(',', ': ')
213 ).encode('utf-8') + '\n'
214
215
216 def import_string_webext(data, key, source):
217 """Import a single translation from the source dictionary into data"""
218 data[key] = source
219
220
221 def import_string_gecko(data, key, value):
222 """Import Gecko-style locales into data.
223
224 Only sets {'message': value} in the data-dictionary, after stripping
225 undesired Gecko-style access keys.
226 """
227 match = re.search(r'^(.*?)\s*\(&.\)$', value)
228 if match:
229 value = match.group(1)
230 else:
231 index = value.find('&')
232 if index >= 0:
233 value = value[0:index] + value[index + 1:]
234
235 data[key] = {'message': value}
236 201
237 202
238 def import_locales(params, files): 203 def import_locales(params, files):
239 import localeTools 204 for item in params['metadata'].items('import_locales'):
240 205 filename, keys = item
241 # FIXME: localeTools doesn't use real Chrome locales, it uses dash as 206 for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
242 # separator instead. 207 *filename.split('/'))):
243 convert_locale_code = lambda code: code.replace('-', '_') 208 locale = sourceFile.split(os.path.sep)[-2]
244 209 targetFile = os.path.join('_locales', locale, 'messages.json')
245 # We need to map Chrome locales to Gecko locales. Start by mapping Chrome 210 data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
246 # locales to themselves, merely with the dash as separator.
247 locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocal es}
248
249 # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
250 for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems ():
251 locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale
252
253 # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
254 reverse_mapping = {v: k for k, v in locale_mapping.iteritems()}
255 for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems() :
256 if crowdin_locale in reverse_mapping:
257 locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale
258
259 for target, source in locale_mapping.iteritems():
260 targetFile = '_locales/%s/messages.json' % target
261 if not targetFile in files:
262 continue
263
264 for item in params['metadata'].items('import_locales'):
265 fileName, keys = item
266 parts = map(lambda n: source if n == '*' else n, fileName.split('/') )
267 sourceFile = os.path.join(os.path.dirname(item.source), *parts)
268 incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incom plete')
269 if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker ):
270 continue
271
272 data = json.loads(files[targetFile].decode('utf-8'))
273 211
274 try: 212 try:
275 # The WebExtensions (.json) and Gecko format provide 213 with io.open(sourceFile, 'r', encoding='utf-8') as handle:
276 # translations differently and/or provide additional 214 sourceData = json.load(handle)
277 # information like e.g. "placeholders". We want to adhere to
278 # that and preserve the addtional info.
279 if sourceFile.endswith('.json'):
280 with io.open(sourceFile, 'r', encoding='utf-8') as handle:
281 sourceData = json.load(handle)
282 import_string = import_string_webext
283 else:
284 sourceData = localeTools.readFile(sourceFile)
285 import_string = import_string_gecko
286 215
287 # Resolve wildcard imports 216 # Resolve wildcard imports
288 if keys == '*' or keys == '=*': 217 if keys == '*':
289 importList = sourceData.keys() 218 importList = sourceData.keys()
290 importList = filter(lambda k: not k.startswith('_'), importL ist) 219 importList = filter(lambda k: not k.startswith('_'), importL ist)
291 if keys == '=*':
292 importList = map(lambda k: '=' + k, importList)
293 keys = ' '.join(importList) 220 keys = ' '.join(importList)
294 221
295 for stringID in keys.split(): 222 for stringID in keys.split():
296 noMangling = False
297 if stringID.startswith('='):
298 stringID = stringID[1:]
299 noMangling = True
300
301 if stringID in sourceData: 223 if stringID in sourceData:
302 if noMangling: 224 if stringID in data:
303 key = re.sub(r'\W', '_', stringID) 225 print ('Warning: locale string {} defined multiple'
304 else: 226 ' times').format(stringID)
305 key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub( r'\W', '_', stringID) 227
306 if key in data: 228 data[stringID] = sourceData[stringID]
307 print 'Warning: locale string %s defined multiple ti mes' % key
308
309 import_string(data, key, sourceData[stringID])
310 except Exception as e: 229 except Exception as e:
311 print 'Warning: error importing locale data from %s: %s' % (sour ceFile, e) 230 print 'Warning: error importing locale data from %s: %s' % (sour ceFile, e)
312 231
313 files[targetFile] = toJson(data) 232 files[targetFile] = toJson(data)
314 233
315 234
316 def truncate(text, length_limit): 235 def truncate(text, length_limit):
317 if len(text) <= length_limit: 236 if len(text) <= length_limit:
318 return text 237 return text
319 return text[:length_limit - 1].rstrip() + u'\u2026' 238 return text[:length_limit - 1].rstrip() + u'\u2026'
320 239
321 240
322 def fixTranslationsForCWS(files): 241 def fix_translations_for_chrome(files):
323 # Chrome Web Store requires messages used in manifest.json to be present in
324 # all languages. It also enforces length limits for extension names and
325 # descriptions.
326 defaults = {} 242 defaults = {}
327 data = json.loads(files['_locales/%s/messages.json' % defaultLocale]) 243 data = json.loads(files['_locales/%s/messages.json' % defaultLocale])
328 for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']): 244 for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
329 name = match.group(1) 245 name = match.group(1)
330 defaults[name] = data[name] 246 defaults[name] = data[name]
331 247
332 limits = {} 248 limits = {}
333 manifest = json.loads(files['manifest.json']) 249 manifest = json.loads(files['manifest.json'])
334 for key, limit in (('name', 45), ('description', 132), ('short_name', 12)): 250 for key, limit in (('name', 45), ('description', 132), ('short_name', 12)):
335 match = re.search(r'__MSG_(\S+)__', manifest.get(key, '')) 251 match = re.search(r'__MSG_(\S+)__', manifest.get(key, ''))
336 if match: 252 if match:
337 limits[match.group(1)] = limit 253 limits[match.group(1)] = limit
338 254
339 for filename in files: 255 for path in list(files):
340 if not filename.startswith('_locales/') or not filename.endswith('/messa ges.json'): 256 match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
257 if not match:
341 continue 258 continue
342 259
343 data = json.loads(files[filename]) 260 # The Chrome Web Store requires messages used in manifest.json to
344 for name, info in defaults.iteritems(): 261 # be present in all languages, and enforces length limits on
345 data.setdefault(name, info) 262 # extension name and description.
346 for name, limit in limits.iteritems(): 263 is_latam, is_mexican, filename = match.groups()
347 if name in data: 264 if filename == 'messages.json':
348 data[name]['message'] = truncate(data[name]['message'], limit) 265 data = json.loads(files[path])
349 files[filename] = toJson(data) 266 for name, info in defaults.iteritems():
267 data.setdefault(name, info)
268 for name, limit in limits.iteritems():
269 info = data.get(name)
270 if info:
271 info['message'] = truncate(info['message'], limit)
272 files[path] = toJson(data)
273
274 # Chrome combines Latin American dialects of Spanish into es-419.
275 if is_latam:
276 data = files.pop(path)
277 if is_mexican:
278 files['_locales/es_419/' + filename] = data
350 279
351 280
352 def signBinary(zipdata, keyFile): 281 def signBinary(zipdata, keyFile):
353 from Crypto.Hash import SHA 282 from Crypto.Hash import SHA
354 from Crypto.PublicKey import RSA 283 from Crypto.PublicKey import RSA
355 from Crypto.Signature import PKCS1_v1_5 284 from Crypto.Signature import PKCS1_v1_5
356 285
357 try: 286 try:
358 with open(keyFile, 'rb') as file: 287 with open(keyFile, 'rb') as file:
359 key = RSA.importKey(file.read()) 288 key = RSA.importKey(file.read())
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after
419 files.preprocess( 348 files.preprocess(
420 [f for f, _ in metadata.items('preprocess')], 349 [f for f, _ in metadata.items('preprocess')],
421 {'needsExt': True} 350 {'needsExt': True}
422 ) 351 )
423 352
424 if metadata.has_section('import_locales'): 353 if metadata.has_section('import_locales'):
425 import_locales(params, files) 354 import_locales(params, files)
426 355
427 files['manifest.json'] = createManifest(params, files) 356 files['manifest.json'] = createManifest(params, files)
428 if type == 'chrome': 357 if type == 'chrome':
429 fixTranslationsForCWS(files) 358 fix_translations_for_chrome(files)
430 359
431 if devenv: 360 if devenv:
432 import buildtools 361 import buildtools
433 import random 362 import random
434 files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js '), relpath='devenvPoller__.js') 363 files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js '), relpath='devenvPoller__.js')
435 files['devenvVersion__'] = str(random.random()) 364 files['devenvVersion__'] = str(random.random())
436 365
437 if metadata.has_option('general', 'testScripts'): 366 if metadata.has_option('general', 'testScripts'):
438 files['qunit/index.html'] = createScriptPage( 367 files['qunit/index.html'] = createScriptPage(
439 params, 'testIndex.html.tmpl', ('general', 'testScripts') 368 params, 'testIndex.html.tmpl', ('general', 'testScripts')
440 ) 369 )
441 370
442 zipdata = files.zipToString() 371 zipdata = files.zipToString()
443 signature = None 372 signature = None
444 pubkey = None 373 pubkey = None
445 if keyFile != None: 374 if keyFile != None:
446 signature = signBinary(zipdata, keyFile) 375 signature = signBinary(zipdata, keyFile)
447 pubkey = getPublicKey(keyFile) 376 pubkey = getPublicKey(keyFile)
448 writePackage(outFile, pubkey, signature, zipdata) 377 writePackage(outFile, pubkey, signature, zipdata)
LEFTRIGHT

Powered by Google App Engine
This is Rietveld