Issue 2340 - Don`t generate pages if less than 30% have been translated

cms/converters.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 41 matching lines @@
     self._whitelist = whitelist
 
   def parse(self, text, pagename):
     self.reset()
     self._string = []
     self._fixed_strings = []
     self._inside_fixed = False
     self._attrs = {}
     self._pagename = pagename
 
+    # Force-escape ampersands, otherwise the parser will autocomplete bogus
+    # entities.
+    text = re.sub(r"&(?!\S+;)", "&", text)
+
     try:
       self.feed(text)
       return "".join(self._string), self._attrs, ["".join(s) for s in self._fixed_strings]
     finally:
       self._string = None
       self._attrs = None
       self._pagename = None
       self._inside_fixed = False
       self._fixed_strings = None
 
@@ skipping 27 matching lines @@
     # HTML escaping is applied when this string is inserted into the document.
     self._append_text(data)
 
   def handle_entityref(self, name):
     self._append_text(self.unescape("&%s;" % name))
 
   def handle_charref(self, name):
     self._append_text(self.unescape("&#%s;" % name))
 
 class Converter:
-  whitelist = set(["a", "em", "strong"])
+  whitelist = {"a", "em", "strong", "code", "span"}
   missing_translations = 0
   total_translations = 0
 
   def __init__(self, params, key="pagedata"):
     self._params = params
     self._key = key
     self._attribute_parser = AttributeParser(self.whitelist)
 
     # Read in any parameters specified at the beginning of the file
     lines = params[key].splitlines(True)
@@ skipping 18 matching lines @@
     if locale == self._params["defaultlocale"]:
       result = default
     elif name in localedata:
       result = localedata[name].strip()
     else:
       result = default
       self.missing_translations += 1
     self.total_translations += 1
 
     # Insert fixed strings
-    for i in range(len(fixed_strings)):
-      result = re.sub(r"\{%d\}" % (i + 1), fixed_strings[i], result, 1)
+    for i, fixed_string in enumerate(fixed_strings, 1):
+      result = result.replace("{%d}" % i, fixed_string)
 
     # Insert attributes
     result = escape(result)
     def stringify_attribute((name, value)):
-      if name == "href":
-        link_locale, link = self._params["source"].resolve_link(value, locale)
-        if link:
-          return 'href="%s" hreflang="%s"' % (escape(link), escape(link_locale))
-      return '%s="%s"' % (escape(name), escape(value))
+      return '%s="%s"' % (
+        escape(name),
+        escape(self.insert_localized_strings(value, {}))
+      )
 
     for tag in self.whitelist:
       saved = saved_attributes.get(tag, [])
       for attrs in saved:
         attrs = map(stringify_attribute, attrs)
         result = re.sub(
           r"%s([^<>]*?)%s" % (re_escape("<%s>" % tag), re_escape("</%s>" % tag)),
-          r'<%s %s>\1</%s>' % (tag, " ".join(attrs), tag),
+          r'<%s%s>\1</%s>' % (tag, " " + " ".join(attrs) if attrs else "", tag),
           result, 1, flags=re.S
         )
       result = re.sub(
         r"%s([^<>]*?)%s" % (re_escape("<%s>" % tag), re_escape("</%s>" % tag)),
         r"<%s>\1</%s>" % (tag, tag),
         result, flags=re.S
       )
     return result
 
   def insert_localized_strings(self, text, escapes, to_html=lambda s: s):
     def lookup_string(match):
       name, comment, default = match.groups()
       default = to_html(default).strip()
 
       # Note: We currently ignore the comment, it is only relevant when
       # generating the master translation.
       return self.localize_string(name, default, self._params["localedata"], escapes)
 
     return re.sub(
-      r"\{\{\s*([\w\-]+)(?:\[(.*?)\])?\s+(.*?)\}\}",
+      r"{{\s*"
+      r"([\w\-]+)" # String ID
+      r"(?:\[(.*?)\])?" # Optional comment
+      r"\s+"
+      r"((?:(?!{{).|" # Translatable text
+        r"{{(?:(?!}}).)*}}" # Nested translation
+      r")*?)"
+      r"}}",
       lookup_string,
       text,
       flags=re.S
     )
 
   def process_links(self, text):
     def process_link(match):
       pre, attr, url, post = match.groups()
       url = jinja2.Markup(url).unescape()
 
@@ skipping 108 matching lines @@
       "translate": self.translate,
       "linkify": self.linkify,
       "toclist": self.toclist,
     }
 
     globals = {
       "get_string": self.get_string,
       "get_page_content": self.get_page_content,
     }
 
+    self._module_refs = []
     for dirname, dictionary in [("filters", filters), ("globals", globals)]:
       for filename in self._params["source"].list_files(dirname):
         root, ext = os.path.splitext(filename)
         if ext.lower() != ".py":
           continue
 
         path = "%s/%s" % (dirname, filename)
         code = self._params["source"].read_file(path)
         module = imp.new_module(root.replace("/", "."))
         exec code in module.__dict__
 
         name = os.path.basename(root)
         if not hasattr(module, name):
           raise Exception("Expected symbol %s not found in %s file %s" % (name, dirname, filename))
         dictionary[name] = getattr(module, name)
 
         # HACK: The module we created here can be garbage collected because it
         # isn't added to sys.modules. If a function is called and its module is
         # gone it might cause weird errors (imports and module variables
-        # unavailable). We avoid this situation by explicitly referencing the
-        # module from the function so they can only be garbage collected
-        # together.
-        if callable(dictionary[name]):
-          dictionary[name].module_ref = module
+        # unavailable). We avoid this situation by keeping a reference.
+        self._module_refs.append(module)
 
     self._env = jinja2.Environment(loader=self._SourceLoader(self._params["source"]), autoescape=True)
     self._env.filters.update(filters)
     self._env.globals.update(globals)
 
   def get_html(self, source):
     template = self._env.from_string(source)
     module = template.make_module(self._params)
     for key, value in module.__dict__.iteritems():
       if not key.startswith("_"):
         self._params[key] = value
-    return unicode(module)
+
+    result = unicode(module)
+    result = self.process_links(result)
+    return result
 
   def translate(self, default, name, comment=None):
     # Note: We currently ignore the comment, it is only relevant when
     # generating the master translation.
     localedata = self._params["localedata"]
     return jinja2.Markup(self.localize_string(name, default, localedata, html_escapes))
 
   def get_string(self, name, page):
     localedata = self._params["source"].read_locale(self._params["locale"], page)
     default = localedata[name]
@@ skipping 35 matching lines @@
         stack.pop()
       stack[-1]["subitems"].append(item)
       stack.append(item)
     return structured
 
 converters = {
   "html": RawConverter,
   "md": MarkdownConverter,
   "tmpl": TemplateConverter,
 }