Issue 3084 - [cms] Show full tracebacks for exceptions passing template code

cms/converters.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 94 matching lines @@
   def handle_entityref(self, name):
     self._append_text(self.unescape("&%s;" % name))
 
   def handle_charref(self, name):
     self._append_text(self.unescape("&#%s;" % name))
 
 class Converter:
   whitelist = {"a", "em", "strong", "code", "span"}
   missing_translations = 0
   total_translations = 0
+  removed_line = ''
 
   def __init__(self, params, key="pagedata"):
     self._params = params
     self._key = key
     self._attribute_parser = AttributeParser(self.whitelist)
     self._seen_defaults  = {}
 
     # Read in any parameters specified at the beginning of the file
     lines = params[key].splitlines(True)
-    while lines and re.search(r"^\s*[\w\-]+\s*=", lines[0]):
-      name, value = lines.pop(0).split("=", 1)
+    for i, line in enumerate(lines):

[Sebastian Noack, 2015/09/15 17:48:36]

In order to prevent line numbers from being off we replace lines setting
parameters with "{#\n#}" rather than removing them in jinja2 templates.

+      if not re.search(r"^\s*[\w\-]+\s*=", line):
+        break
+      name, value = line.split("=", 1)
       params[name.strip()] = value.strip()
+      lines[i] = self.removed_line
     params[key] = "".join(lines)
 
   def localize_string(self, page, name, default, comment, localedata, escapes):
     def escape(s):
       return re.sub(r".",
         lambda match: escapes.get(match.group(0), match.group(0)),
         s, flags=re.S)
     def re_escape(s):
       return re.escape(escape(s))
 
@@ skipping 102 matching lines @@
 
   include_start_regex = '<'
   include_end_regex = '>'
 
   def resolve_includes(self, text):
     def resolve_include(match):
       global converters
       name = match.group(1)
       for format, converter_class in converters.iteritems():
         if self._params["source"].has_include(name, format):
+          self._params["include"] = name

[Sebastian Noack, 2015/09/15 17:48:36]

There seems to be no way to get the filename for includes currently.

           self._params["includedata"] = self._params["source"].read_include(name, format)
           converter = converter_class(self._params, key="includedata")
           result = converter()
           self.missing_translations += converter.missing_translations
           self.total_translations += converter.total_translations
           return result
       raise Exception("Failed to resolve include %s on page %s" % (name, self._params["page"]))
 
     return re.sub(
       r'%s\?\s*include\s+([^\s<>"]+)\s*\?%s' % (
@@ skipping 53 matching lines @@
     def to_html(s):
       return re.sub(r'</?p>', '', md.convert(s))
 
     result = self.insert_localized_strings(source, escapes, to_html)
     result = md.convert(result)
     result = re.sub(r"&#(\d+);", remove_unnecessary_entities, result)
     result = self.process_links(result)
     return result
 
 class TemplateConverter(Converter):
-  class _SourceLoader(jinja2.BaseLoader):
-    def __init__(self, source):
-      self.source = source
-
-    def get_source(self, environment, template):
-      try:
-        return self.source.read_file(template + ".tmpl"), None, None
-      except Exception:
-        raise jinja2.TemplateNotFound(template)
+  removed_line = "{#\n#}"
 
   def __init__(self, *args, **kwargs):
     Converter.__init__(self, *args, **kwargs)
 
     filters = {
       "translate": self.translate,
       "linkify": self.linkify,
       "toclist": self.toclist,
     }
 
     globals = {
       "get_string": self.get_string,
       "get_page_content": self.get_page_content,
     }
 
     for dirname, dictionary in [("filters", filters), ("globals", globals)]:
       for filename in self._params["source"].list_files(dirname):
         root, ext = os.path.splitext(filename)
         if ext.lower() != ".py":
           continue
 
         path = "%s/%s" % (dirname, filename)
         name = os.path.basename(root)
         dictionary[name] = self._params["source"].import_symbol(path, name)
 
-    self._env = jinja2.Environment(loader=self._SourceLoader(self._params["source"]), autoescape=True)
+    self._env = jinja2.Environment(loader=self._params["source"].get_template_loader(), autoescape=True)
     self._env.filters.update(filters)
     self._env.globals.update(globals)
 
+  def get_template_filename(self):
+    source = self._params["source"]

[Sebastian Noack, 2015/09/15 17:48:36]

And here is where the problematic part starts. I don't really want to have that
logic.

+    if hasattr(source, "get_path"):
+      if self._key == "pagedata":
+        return source.get_path(source.page_filename(self._params["page"], "tmpl"))
+      if self._key == "includedata":
+        return source.get_path(source.include_filename(self._params["include"], "tmpl"))
+      if self._key == "templatedata":
+        return source.get_path(source.template_filename(self._params["template"]))
+
   def get_html(self, source):
-    template = self._env.from_string(source)
-    module = template.make_module(self._params)
+    env = self._env
+    code = env.compile(source, None, self.get_template_filename())

[Sebastian Noack, 2015/09/15 17:48:36]

There are two ways to make the template aware of the filename, either calling
compile() directly passing the filename, as we do here. Or using a loader that
is aware of the filename and using get_template().

+    template = env.template_class.from_code(env, code, env.globals)
+
+    try:
+      module = template.make_module(self._params)
+    except Exception:
+      env.handle_exception()

[Sebastian Noack, 2015/09/15 17:48:36]

Note that this is necessary because as opposed to make_module() doesn't call 
handle_exception(). Hence the line number refers to the line in the generated
Python code, not the jinja source code, without calling handle_exception().

+
     for key, value in module.__dict__.iteritems():
       if not key.startswith("_"):
         self._params[key] = value
 
     result = unicode(module)
     result = self.process_links(result)
     return result
 
   def translate(self, default, name, comment=None):
     return jinja2.Markup(self.localize_string(
@@ skipping 47 matching lines @@
         stack.pop()
       stack[-1]["subitems"].append(item)
       stack.append(item)
     return structured
 
 converters = {
   "html": RawConverter,
   "md": MarkdownConverter,
   "tmpl": TemplateConverter,
 }