Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code
Issues
All Issues

Delta Between Two Patch Sets: cms/converters.py

Issue 29327966: Issue 3084 - [cms] Show full tracebacks for exceptions passing template code (Closed)
Left Patch Set: Created Sept. 15, 2015, 5:37 p.m.
Right Patch Set: Unpack converter source before calling get_html() Created Sept. 17, 2015, 10 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « cms/bin/generate_static_pages.py ('k') | cms/sources.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH 4 # Copyright (C) 2006-2015 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after
105 def handle_entityref(self, name): 105 def handle_entityref(self, name):
106 self._append_text(self.unescape("&%s;" % name)) 106 self._append_text(self.unescape("&%s;" % name))
107 107
108 def handle_charref(self, name): 108 def handle_charref(self, name):
109 self._append_text(self.unescape("&#%s;" % name)) 109 self._append_text(self.unescape("&#%s;" % name))
110 110
111 class Converter: 111 class Converter:
112 whitelist = {"a", "em", "strong", "code", "span"} 112 whitelist = {"a", "em", "strong", "code", "span"}
113 missing_translations = 0 113 missing_translations = 0
114 total_translations = 0 114 total_translations = 0
115 removed_line = ''
116 115
117 def __init__(self, params, key="pagedata"): 116 def __init__(self, params, key="pagedata"):
118 self._params = params 117 self._params = params
119 self._key = key 118 self._key = key
120 self._attribute_parser = AttributeParser(self.whitelist) 119 self._attribute_parser = AttributeParser(self.whitelist)
121 self._seen_defaults = {} 120 self._seen_defaults = {}
122 121
123 # Read in any parameters specified at the beginning of the file 122 # Read in any parameters specified at the beginning of the file
124 lines = params[key].splitlines(True) 123 data, filename = params[key]
124 lines = data.splitlines(True)
125 for i, line in enumerate(lines): 125 for i, line in enumerate(lines):
Sebastian Noack 2015/09/15 17:48:36 In order to prevent line numbers from being off we
126 if not re.search(r"^\s*[\w\-]+\s*=", line): 126 if not re.search(r"^\s*[\w\-]+\s*=", line):
127 break 127 break
128 name, value = line.split("=", 1) 128 name, value = line.split("=", 1)
129 params[name.strip()] = value.strip() 129 params[name.strip()] = value.strip()
130 lines[i] = self.removed_line 130 lines[i] = "\n"
131 params[key] = "".join(lines) 131 params[key] = ("".join(lines), filename)
132 132
133 def localize_string(self, page, name, default, comment, localedata, escapes): 133 def localize_string(self, page, name, default, comment, localedata, escapes):
134 def escape(s): 134 def escape(s):
135 return re.sub(r".", 135 return re.sub(r".",
136 lambda match: escapes.get(match.group(0), match.group(0)), 136 lambda match: escapes.get(match.group(0), match.group(0)),
137 s, flags=re.S) 137 s, flags=re.S)
138 def re_escape(s): 138 def re_escape(s):
139 return re.escape(escape(s)) 139 return re.escape(escape(s))
140 140
141 # Handle duplicated strings 141 # Handle duplicated strings
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after
243 243
244 include_start_regex = '<' 244 include_start_regex = '<'
245 include_end_regex = '>' 245 include_end_regex = '>'
246 246
247 def resolve_includes(self, text): 247 def resolve_includes(self, text):
248 def resolve_include(match): 248 def resolve_include(match):
249 global converters 249 global converters
250 name = match.group(1) 250 name = match.group(1)
251 for format, converter_class in converters.iteritems(): 251 for format, converter_class in converters.iteritems():
252 if self._params["source"].has_include(name, format): 252 if self._params["source"].has_include(name, format):
253 self._params["include"] = name
Sebastian Noack 2015/09/15 17:48:36 There seems to be no way to get the filename for i
254 self._params["includedata"] = self._params["source"].read_include(name , format) 253 self._params["includedata"] = self._params["source"].read_include(name , format)
255 converter = converter_class(self._params, key="includedata") 254 converter = converter_class(self._params, key="includedata")
256 result = converter() 255 result = converter()
257 self.missing_translations += converter.missing_translations 256 self.missing_translations += converter.missing_translations
258 self.total_translations += converter.total_translations 257 self.total_translations += converter.total_translations
259 return result 258 return result
260 raise Exception("Failed to resolve include %s on page %s" % (name, self._p arams["page"])) 259 raise Exception("Failed to resolve include %s on page %s" % (name, self._p arams["page"]))
261 260
262 return re.sub( 261 return re.sub(
263 r'%s\?\s*include\s+([^\s<>"]+)\s*\?%s' % ( 262 r'%s\?\s*include\s+([^\s<>"]+)\s*\?%s' % (
264 self.include_start_regex, 263 self.include_start_regex,
265 self.include_end_regex 264 self.include_end_regex
266 ), 265 ),
267 resolve_include, 266 resolve_include,
268 text 267 text
269 ) 268 )
270 269
271 def __call__(self): 270 def __call__(self):
272 result = self.get_html(self._params[self._key]) 271 result = self.get_html(*self._params[self._key])
273 result = self.resolve_includes(result) 272 result = self.resolve_includes(result)
274 if self._key == "pagedata": 273 if self._key == "pagedata":
275 head = [] 274 head = []
276 def add_to_head(match): 275 def add_to_head(match):
277 head.append(match.group(1)) 276 head.append(match.group(1))
278 return "" 277 return ""
279 body = re.sub(r"<head>(.*?)</head>", add_to_head, result, flags=re.S) 278 body = re.sub(r"<head>(.*?)</head>", add_to_head, result, flags=re.S)
280 return "".join(head), body 279 return "".join(head), body
281 else: 280 else:
282 return result 281 return result
283 282
284 class RawConverter(Converter): 283 class RawConverter(Converter):
285 def get_html(self, source): 284 def get_html(self, source, filename):
286 result = self.insert_localized_strings(source, html_escapes) 285 result = self.insert_localized_strings(source, html_escapes)
287 result = self.process_links(result) 286 result = self.process_links(result)
288 return result 287 return result
289 288
290 class MarkdownConverter(Converter): 289 class MarkdownConverter(Converter):
291 include_start_regex = r'(?:%s|%s)' % ( 290 include_start_regex = r'(?:%s|%s)' % (
292 Converter.include_start_regex, 291 Converter.include_start_regex,
293 re.escape(jinja2.escape(Converter.include_start_regex)) 292 re.escape(jinja2.escape(Converter.include_start_regex))
294 ) 293 )
295 include_end_regex = r'(?:%s|%s)' % ( 294 include_end_regex = r'(?:%s|%s)' % (
296 Converter.include_end_regex, 295 Converter.include_end_regex,
297 re.escape(jinja2.escape(Converter.include_end_regex)) 296 re.escape(jinja2.escape(Converter.include_end_regex))
298 ) 297 )
299 298
300 def get_html(self, source): 299 def get_html(self, source, filename):
301 def remove_unnecessary_entities(match): 300 def remove_unnecessary_entities(match):
302 char = unichr(int(match.group(1))) 301 char = unichr(int(match.group(1)))
303 if char in html_escapes: 302 if char in html_escapes:
304 return match.group(0) 303 return match.group(0)
305 else: 304 else:
306 return char 305 return char
307 306
308 escapes = {} 307 escapes = {}
309 for char in markdown.Markdown.ESCAPED_CHARS: 308 for char in markdown.Markdown.ESCAPED_CHARS:
310 escapes[char] = "&#" + str(ord(char)) + ";" 309 escapes[char] = "&#" + str(ord(char)) + ";"
311 for key, value in html_escapes.iteritems(): 310 for key, value in html_escapes.iteritems():
312 escapes[key] = value 311 escapes[key] = value
313 312
314 md = markdown.Markdown(output="html5", extensions=["attr_list"]) 313 md = markdown.Markdown(output="html5", extensions=["attr_list"])
315 md.preprocessors["html_block"].markdown_in_raw = True 314 md.preprocessors["html_block"].markdown_in_raw = True
316 315
317 def to_html(s): 316 def to_html(s):
318 return re.sub(r'</?p>', '', md.convert(s)) 317 return re.sub(r'</?p>', '', md.convert(s))
319 318
320 result = self.insert_localized_strings(source, escapes, to_html) 319 result = self.insert_localized_strings(source, escapes, to_html)
321 result = md.convert(result) 320 result = md.convert(result)
322 result = re.sub(r"&#(\d+);", remove_unnecessary_entities, result) 321 result = re.sub(r"&#(\d+);", remove_unnecessary_entities, result)
323 result = self.process_links(result) 322 result = self.process_links(result)
324 return result 323 return result
325 324
325 class SourceTemplateLoader(jinja2.BaseLoader):
326 def __init__(self, source):
327 self.source = source
328
329 def get_source(self, environment, template):
330 try:
331 result = self.source.read_file(template + ".tmpl")
332 except Exception:
333 raise jinja2.TemplateNotFound(template)
334 return result + (None,)
335
326 class TemplateConverter(Converter): 336 class TemplateConverter(Converter):
327 removed_line = "{#\n#}"
328
329 def __init__(self, *args, **kwargs): 337 def __init__(self, *args, **kwargs):
330 Converter.__init__(self, *args, **kwargs) 338 Converter.__init__(self, *args, **kwargs)
331 339
332 filters = { 340 filters = {
333 "translate": self.translate, 341 "translate": self.translate,
334 "linkify": self.linkify, 342 "linkify": self.linkify,
335 "toclist": self.toclist, 343 "toclist": self.toclist,
336 } 344 }
337 345
338 globals = { 346 globals = {
339 "get_string": self.get_string, 347 "get_string": self.get_string,
340 "get_page_content": self.get_page_content, 348 "get_page_content": self.get_page_content,
341 } 349 }
342 350
343 for dirname, dictionary in [("filters", filters), ("globals", globals)]: 351 for dirname, dictionary in [("filters", filters), ("globals", globals)]:
344 for filename in self._params["source"].list_files(dirname): 352 for filename in self._params["source"].list_files(dirname):
345 root, ext = os.path.splitext(filename) 353 root, ext = os.path.splitext(filename)
346 if ext.lower() != ".py": 354 if ext.lower() != ".py":
347 continue 355 continue
348 356
349 path = "%s/%s" % (dirname, filename) 357 path = "%s/%s" % (dirname, filename)
358 namespace = self._params["source"].exec_file(path)
359
350 name = os.path.basename(root) 360 name = os.path.basename(root)
351 dictionary[name] = self._params["source"].import_symbol(path, name) 361 try:
352 362 dictionary[name] = namespace[name]
353 self._env = jinja2.Environment(loader=self._params["source"].get_template_lo ader(), autoescape=True) 363 except KeyError:
364 raise Exception("Expected symbol %r not found in %r" % (name, path))
365
366 self._env = jinja2.Environment(loader=SourceTemplateLoader(self._params["sou rce"]), autoescape=True)
354 self._env.filters.update(filters) 367 self._env.filters.update(filters)
355 self._env.globals.update(globals) 368 self._env.globals.update(globals)
356 369
357 def get_template_filename(self): 370 def get_html(self, source, filename):
358 source = self._params["source"]
Sebastian Noack 2015/09/15 17:48:36 And here is where the problematic part starts. I d
359 if hasattr(source, "get_path"):
360 if self._key == "pagedata":
361 return source.get_path(source.page_filename(self._params["page"], "tmpl" ))
362 if self._key == "includedata":
363 return source.get_path(source.include_filename(self._params["include"], "tmpl"))
364 if self._key == "templatedata":
365 return source.get_path(source.template_filename(self._params["template"] ))
366
367 def get_html(self, source):
368 env = self._env 371 env = self._env
369 code = env.compile(source, None, self.get_template_filename()) 372 code = env.compile(source, None, filename)
Sebastian Noack 2015/09/15 17:48:36 There are two ways to make the template aware of t
370 template = env.template_class.from_code(env, code, env.globals) 373 template = jinja2.Template.from_code(env, code, env.globals)
371 374
372 try: 375 try:
373 module = template.make_module(self._params) 376 module = template.make_module(self._params)
374 except Exception: 377 except Exception:
375 env.handle_exception() 378 env.handle_exception()
Sebastian Noack 2015/09/15 17:48:36 Note that this is necessary because as opposed to
376 379
377 for key, value in module.__dict__.iteritems(): 380 for key, value in module.__dict__.iteritems():
378 if not key.startswith("_"): 381 if not key.startswith("_"):
379 self._params[key] = value 382 self._params[key] = value
380 383
381 result = unicode(module) 384 result = unicode(module)
382 result = self.process_links(result) 385 result = self.process_links(result)
383 return result 386 return result
384 387
385 def translate(self, default, name, comment=None): 388 def translate(self, default, name, comment=None):
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
434 stack.pop() 437 stack.pop()
435 stack[-1]["subitems"].append(item) 438 stack[-1]["subitems"].append(item)
436 stack.append(item) 439 stack.append(item)
437 return structured 440 return structured
438 441
439 converters = { 442 converters = {
440 "html": RawConverter, 443 "html": RawConverter,
441 "md": MarkdownConverter, 444 "md": MarkdownConverter,
442 "tmpl": TemplateConverter, 445 "tmpl": TemplateConverter,
443 } 446 }
LEFTRIGHT
« cms/bin/generate_static_pages.py ('k') | cms/sources.py » ('j') | Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Toggle Comments ('s')

Powered by Google App Engine
This is Rietveld