Issue 2139 - Allow nested translations for tag attributes.

cms/converters.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 127 matching lines @@
     else:
       result = default
 
     # Insert fixed strings
     for i in range(len(fixed_strings)):
       result = re.sub(r"\{%d\}" % (i + 1), fixed_strings[i], result, 1)
 
     # Insert attributes
     result = escape(result)
     def stringify_attribute((name, value)):
+      value = self.insert_localized_strings(value, escapes)
       if name == "href":
         link_locale, link = self._params["source"].resolve_link(value, locale)
         if link:
           return 'href="%s" hreflang="%s"' % (escape(link), escape(link_locale))
       return '%s="%s"' % (escape(name), escape(value))
 
     for tag in self.whitelist:
       saved = saved_attributes.get(tag, [])
       for attrs in saved:
         attrs = map(stringify_attribute, attrs)
         result = re.sub(
           r"%s([^<>]*?)%s" % (re_escape("<%s>" % tag), re_escape("</%s>" % tag)),
           r'<%s %s>\1</%s>' % (tag, " ".join(attrs), tag),
           result, 1, flags=re.S
         )
       result = re.sub(
         r"%s([^<>]*?)%s" % (re_escape("<%s>" % tag), re_escape("</%s>" % tag)),
         r"<%s>\1</%s>" % (tag, tag),
         result, flags=re.S
       )
     return result
 
   def insert_localized_strings(self, text, escapes, to_html=lambda s: s):
-    def lookup_string(match):
-      name, comment, default = match.groups()
-      default = to_html(default).strip()
+    # Find the positions for the top-level translatable strings
+    level = 0
+    start = None
+    positions = []
+    for m in re.finditer(r"(\{\{|\}\})", text):
+      if m.group(0) == "{{":
+        if level == 0:
+          start = m.start()
+        level += 1
+      else:
+        level -= 1
+        if level == 0:
+          positions.append((start, m.end()))
 
-      # Note: We currently ignore the comment, it is only relevant when
-      # generating the master translation.
-      return self.localize_string(name, default, self._params["localedata"], escapes)
-
-    return re.sub(
-      r"\{\{\s*([\w\-]+)(?:\[(.*?)\])?\s+(.*?)\}\}",
-      lookup_string,
-      text,
-      flags=re.S
-    )
+    # Replace each string with it's translation
+    parts = []
+    last_end = 0
+    for start, end in positions:
+      # Append any text before this tag since the last one
+      parts.append(text[last_end:start])
+      # Append our localized string
+      name, comment, default = re.match(
+        r"\{\{\s*([\w\-]+)(?:\[(.*?)\])?\s+(.*)\}\}",

[Wladimir Palant, 2015/04/16 21:01:13]

Only one nesting level is actually allowed here. So instead of finding positions
above we can use a more complicated regexp:

  r"\{\{\s*"
  r"([\w\-]+)"      # String ID
  r"(?:\[(.*?)\])?" # Optional comment
  r"\s+"
  r"(?:[^\{]|\{(?!{)|\{\{.*\}\})*?"  # Translatable text (could contain other
translatable strings)
  r"\}\}"

Yes, you can split the regexp into multiple lines like that.

[kzar, 2015/04/21 11:28:57]

With a couple of tweaks I managed to get your regexp matching how we want:

r"\{\{\s*"
r"([\w\-]+)" # String ID
r"(?:\[(.*?)\])?" # Optional comment
r"\s+"
r"((?:[^\{]|\{(?!\{)|\{\{.*\}\})*?)" # Translatable text
r"\}\}"

Problem is when that regexp is used with the re.S (dot matches all) flag it runs
forever (or a long time at least) maxing out one of my CPU cores. Without the
flag it works fine, but I guess we want to support multi-line default
translation strings.

Any ideas what's happening?

[kzar, 2015/04/21 14:44:53]

OK I made another change to the regexp to hopefully avoid exponential
complexity. For the contents of nested tags I replaced the .* with
(?:[^\}]|\}(?!\}))*? .

As we know there can only be one level of nesting I ensure "}}" does not appear
inside the child tag. I think that avoids backticking crazily each time the
final "}}" of the regexp doesn't match.

The regexp is getting pretty huge though now, I worry it's less readable than my
old solution.

+        text[start:end],
+        flags=re.S
+      ).groups()
+      parts.append(self.localize_string(
+        name,
+        to_html(default).strip(),
+        self._params["localedata"],
+        escapes
+      ))
+      last_end = end
+    return "".join(parts) + text[last_end:]
 
   def process_links(self, text):
     def process_link(match):
       pre, attr, url, post = match.groups()
       url = jinja2.Markup(url).unescape()
 
       locale, new_url = self._params["source"].resolve_link(url, self._params["locale"])
       if new_url != None:
         url = new_url
         if attr == "href":
@@ skipping 192 matching lines @@
         stack.pop()
       stack[-1]["subitems"].append(item)
       stack.append(item)
     return structured
 
 converters = {
   "html": RawConverter,
   "md": MarkdownConverter,
   "tmpl": TemplateConverter,
 }