Issue 2139 - Allow nested translations for tag attributes.

cms/converters.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 114 matching lines @@
       return re.sub(r".",
         lambda match: escapes.get(match.group(0), match.group(0)),
         s, flags=re.S)
     def re_escape(s):
       return re.escape(escape(s))
 
     # Extract tag attributes from default string
     default, saved_attributes, fixed_strings = self._attribute_parser.parse(default, self._params["page"])
 
     # Get translation
-    locale = self._params["locale"]
-    if locale != self._params["defaultlocale"] and name in localedata:
+    if self._params["locale"] != self._params["defaultlocale"] and name in localedata:
       result = localedata[name].strip()
     else:
       result = default
 
     # Insert fixed strings
     for i in range(len(fixed_strings)):
       result = re.sub(r"\{%d\}" % (i + 1), fixed_strings[i], result, 1)
 
     # Insert attributes
     result = escape(result)
     def stringify_attribute((name, value)):
-      value = self.insert_localized_strings(value, escapes)
-      if name == "href":
-        link_locale, link = self._params["source"].resolve_link(value, locale)
-        if link:
-          return 'href="%s" hreflang="%s"' % (escape(link), escape(link_locale))
-      return '%s="%s"' % (escape(name), escape(value))
+      return '%s="%s"' % (
+        escape(name),
+        escape(self.insert_localized_strings(value, escapes))
+      )
 
     for tag in self.whitelist:
       saved = saved_attributes.get(tag, [])
       for attrs in saved:
         attrs = map(stringify_attribute, attrs)
         result = re.sub(
           r"%s([^<>]*?)%s" % (re_escape("<%s>" % tag), re_escape("</%s>" % tag)),
-          r'<%s %s>\1</%s>' % (tag, " ".join(attrs), tag),
+          r'<%s%s>\1</%s>' % (tag, " " + " ".join(attrs) if attrs else "", tag),
           result, 1, flags=re.S
         )
       result = re.sub(
         r"%s([^<>]*?)%s" % (re_escape("<%s>" % tag), re_escape("</%s>" % tag)),
         r"<%s>\1</%s>" % (tag, tag),
         result, flags=re.S
       )
     return result
 
   def insert_localized_strings(self, text, escapes, to_html=lambda s: s):
-    # Find the positions for the top-level translatable strings
-    level = 0
-    start = None
-    positions = []
-    for m in re.finditer(r"(\{\{|\}\})", text):
-      if m.group(0) == "{{":
-        if level == 0:
-          start = m.start()
-        level += 1
-      else:
-        level -= 1
-        if level == 0:
-          positions.append((start, m.end()))
-
-    # Replace each string with it's translation
-    parts = []
-    last_end = 0
-    for start, end in positions:
-      # Append any text before this tag since the last one
-      parts.append(text[last_end:start])
-      # Append our localized string
-      name, comment, default = re.match(
-        r"\{\{\s*([\w\-]+)(?:\[(.*?)\])?\s+(.*)\}\}",

[Wladimir Palant, 2015/04/16 21:01:13]

Only one nesting level is actually allowed here. So instead of finding positions
above we can use a more complicated regexp:

  r"\{\{\s*"
  r"([\w\-]+)"      # String ID
  r"(?:\[(.*?)\])?" # Optional comment
  r"\s+"
  r"(?:[^\{]|\{(?!{)|\{\{.*\}\})*?"  # Translatable text (could contain other
translatable strings)
  r"\}\}"

Yes, you can split the regexp into multiple lines like that.

[kzar, 2015/04/21 11:28:57]

With a couple of tweaks I managed to get your regexp matching how we want:

r"\{\{\s*"
r"([\w\-]+)" # String ID
r"(?:\[(.*?)\])?" # Optional comment
r"\s+"
r"((?:[^\{]|\{(?!\{)|\{\{.*\}\})*?)" # Translatable text
r"\}\}"

Problem is when that regexp is used with the re.S (dot matches all) flag it runs
forever (or a long time at least) maxing out one of my CPU cores. Without the
flag it works fine, but I guess we want to support multi-line default
translation strings.

Any ideas what's happening?

[kzar, 2015/04/21 14:44:53]

OK I made another change to the regexp to hopefully avoid exponential
complexity. For the contents of nested tags I replaced the .* with
(?:[^\}]|\}(?!\}))*? .

As we know there can only be one level of nesting I ensure "}}" does not appear
inside the child tag. I think that avoids backticking crazily each time the
final "}}" of the regexp doesn't match.

The regexp is getting pretty huge though now, I worry it's less readable than my
old solution.

-        text[start:end],
-        flags=re.S
-      ).groups()
-      parts.append(self.localize_string(
-        name,
-        to_html(default).strip(),
-        self._params["localedata"],
-        escapes
-      ))
-      last_end = end
-    return "".join(parts) + text[last_end:]
+    def lookup_string(match):
+      name, comment, default = match.groups()
+      default = to_html(default).strip()
+
+      # Note: We currently ignore the comment, it is only relevant when
+      # generating the master translation.
+      return self.localize_string(name, default, self._params["localedata"], escapes)
+
+    return re.sub(
+      r"{{\s*"
+      r"([\w\-]+)" # String ID
+      r"(?:\[(.*?)\])?" # Optional comment
+      r"\s+"
+      r"((?:(?!{{).|" # Translatable text
+        r"{{(?:(?!}}).)*}}" # Nested translation
+      r")*?)"
+      r"}}",
+      lookup_string,
+      text,
+      flags=re.S
+    )
 
   def process_links(self, text):
     def process_link(match):
       pre, attr, url, post = match.groups()
       url = jinja2.Markup(url).unescape()
 
       locale, new_url = self._params["source"].resolve_link(url, self._params["locale"])
       if new_url != None:
         url = new_url
         if attr == "href":
@@ skipping 136 matching lines @@
     self._env = jinja2.Environment(loader=self._SourceLoader(self._params["source"]), autoescape=True)
     self._env.filters.update(filters)
     self._env.globals.update(globals)
 
   def get_html(self, source):
     template = self._env.from_string(source)
     module = template.make_module(self._params)
     for key, value in module.__dict__.iteritems():
       if not key.startswith("_"):
         self._params[key] = value
-    return unicode(module)
+
+    result = unicode(module)
+    result = self.process_links(result)
+    return result
 
   def translate(self, default, name, comment=None):
     # Note: We currently ignore the comment, it is only relevant when
     # generating the master translation.
     localedata = self._params["localedata"]
     return jinja2.Markup(self.localize_string(name, default, localedata, html_escapes))
 
   def get_string(self, name, page):
     localedata = self._params["source"].read_locale(self._params["locale"], page)
     default = localedata[name]
@@ skipping 35 matching lines @@
         stack.pop()
       stack[-1]["subitems"].append(item)
       stack.append(item)
     return structured
 
 converters = {
   "html": RawConverter,
   "md": MarkdownConverter,
   "tmpl": TemplateConverter,
 }