Issue 3956 - Convert domain whitelisting filters

lib/abp2blocklist.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-2016 Eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ skipping 48 matching lines @@
   let excluded = [];
   let rules = [];
 
   parseDomains(filter.domains, included, excluded);
 
   if (excluded.length == 0 && !(filter.selector in elemhideSelectorExceptions))
     return {matchDomains: included.map(matchDomain), selector: filter.selector};
 }
 
 /**
- * Convert the given filter "regexpSource" string into a regular expression,
- * handling the conversion of unicode inside hostnames to punycode.
- * (Also deciding if the regular expression can be safely converted to and
- *  matched as lower case or not.)
+ * Parse the given filter "regexpSource" string. Producing a regular expression,
+ * extracting the hostname (if any), deciding if the regular expression is safe
+ * to be converted + matched as lower case and noting if the source contains
+ * anything after the hostname.)
  *
  * @param   {string} text regexpSource property of a filter
- * @returns {object} An object containing a regular expression string and a bool
+ * @returns {object} An object containing a regular expression string, a bool
  *                   indicating if the filter can be safely matched as lower
- *                   case: {regexp: "...", canSafelyMatchAsLowercase: true/false}
+ *                   case, a hostname string (or undefined) and a bool
+ *                   indicating if the source only contains a hostname or not:
+ *                     {regexp: "...",
+ *                      canSafelyMatchAsLowercase: true/false,
+ *                      hostname: "...",
+ *                      justHostname: true/false}
  */
-function toRegExp(text)
+function parseFilterRegexpSource(text)
 {
-  let result = [];
+  let regexp = [];
   let lastIndex = text.length - 1;
+  let hostname;
   let hostnameStart = null;
   let hostnameFinished = false;
+  let justHostname = false;
   let canSafelyMatchAsLowercase = false;
 
   for (let i = 0; i < text.length; i++)
   {
     let c = text[i];
 
+    if (hostnameFinished)
+      justHostname = false;
+
     // If we're currently inside the hostname we have to be careful not to
     // escape any characters until after we have converted it to punycode.
     if (hostnameStart != null && !hostnameFinished)
     {
       let endingChar = (c == "*" || c == "^" ||
                         c == "?" || c == "/" || c == "|");
       if (!endingChar && i != lastIndex)
         continue;
 
-      let hostname = text.substring(hostnameStart, endingChar ? i : i + 1);
-      hostnameFinished = true;
-      result.push(escapeRegExp(punycode.toASCII(hostname)));
+      hostname = punycode.toASCII(
+        text.substring(hostnameStart, endingChar ? i : i + 1)
+      );
+      hostnameFinished = justHostname = true;
+      regexp.push(escapeRegExp(hostname));
       if (!endingChar)
         break;
     }
 
     switch (c)
     {
       case "*":
-        if (result.length > 0 && i < lastIndex && text[i + 1] != "*")
-          result.push(".*");
+        if (regexp.length > 0 && i < lastIndex && text[i + 1] != "*")
+          regexp.push(".*");
         break;
       case "^":
         if (i < lastIndex)
-          result.push(".");
+          regexp.push(".");
         break;
       case "|":
         if (i == 0)
         {
-          result.push("^");
+          regexp.push("^");
           break;
         }
         if (i == lastIndex)
         {
-          result.push("$");
+          regexp.push("$");
           break;
         }
         if (i == 1 && text[0] == "|")
         {
           hostnameStart = i + 1;
           canSafelyMatchAsLowercase = true;
-          result.push("https?://");
+          regexp.push("https?://");
           break;
         }
-        result.push("\\|");
+        regexp.push("\\|");
         break;
       case "/":
         if (!hostnameFinished &&
             text.charAt(i-2) == ":" && text.charAt(i-1) == "/")
         {
           hostnameStart = i + 1;
           canSafelyMatchAsLowercase = true;
         }
-        result.push("/");
+        regexp.push("/");
         break;
       case ".": case "+": case "$": case "?":
       case "{": case "}": case "(": case ")":
       case "[": case "]": case "\\":
-        result.push("\\", c);
+        regexp.push("\\", c);
         break;
       default:
         if (hostnameFinished && (c >= "a" && c <= "z" ||
                                  c >= "A" && c <= "Z"))
           canSafelyMatchAsLowercase = false;
-        result.push(c);
+        regexp.push(c);
     }
   }
 
-  return {regexp: result.join(""),
-          canSafelyMatchAsLowercase: canSafelyMatchAsLowercase};
-}
-
-function getRegExpTrigger(filter)
-{
-  let result = toRegExp(filter.regexpSource);
-
-  let trigger = {"url-filter": result.regexp};
-
-  // Limit rules to to HTTP(S) URLs
-  if (!/^(\^|http)/i.test(trigger["url-filter"]))
-    trigger["url-filter"] = "^https?://.*" + trigger["url-filter"];
-
-  // For rules containing only a hostname we know that we're matching against
-  // a lowercase string unless the matchCase option was passed.
-  if (result.canSafelyMatchAsLowercase && !filter.matchCase)
-    trigger["url-filter"] = trigger["url-filter"].toLowerCase();
-
-  if (result.canSafelyMatchAsLowercase || filter.matchCase)
-    trigger["url-filter-is-case-sensitive"] = true;
-
-  return trigger;
+  return {
+    regexp: regexp.join(""),
+    canSafelyMatchAsLowercase: canSafelyMatchAsLowercase,
+    hostname: hostname,
+    justHostname: justHostname
+  };
 }
 
 function getResourceTypes(filter)
 {
   let types = [];
 
   if (filter.contentType & typeMap.IMAGE)
     types.push("image");
   if (filter.contentType & typeMap.STYLESHEET)
     types.push("style-sheet");
@@ skipping 24 matching lines @@
   {
     result.push(domain);
 
     if (tldjs.getDomain(domain) == domain)
       result.push("www." + domain);
   }
 
   return result;
 }
 
-function convertFilter(filter, action, withResourceTypes)
+function convertFilterAddRules(rules, filter, action, withResourceTypes)
 {
-  let trigger = getRegExpTrigger(filter);
+  let parsed = parseFilterRegexpSource(filter.regexpSource);
+
+  // For the special case of $document whitelisting filters with just a domain
+  // we can generate an equivalent blocking rule exception using if-domain.
+  if (filter instanceof filterClasses.WhitelistFilter &&
+      filter.contentType & typeMap.DOCUMENT &&
+      parsed.justHostname)
+  {
+    rules.push({
+      trigger: {
+        "url-filter": ".*",
+        "if-domain": addDomainPrefix([parsed.hostname])
+      },
+      action: {type: "ignore-previous-rules"}
+    });
+    // If the filter contains multiple options we'll need to generate further
+    // rules for it, but if not we can simply return now.
+    if (filter.contentType == typeMap.DOCUMENT)

[Sebastian Noack, 2016/05/17 10:55:24]

What if the filter is @@||example.com$document,elemhide?

Then we wouldn't bail out here, though we don't have to create another rule
either. I think we have to check for IMAGE, SUBDOCUMENT, SCRIPT, etc. explicitly
below. Perhaps move the respective mask to a const to avoid duplication.

[kzar, 2016/05/17 11:23:34]

Done.

+      return rules;
+  }
+
+  let trigger = {"url-filter": parsed.regexp};
+
+  // Limit rules to HTTP(S) URLs
+  if (!/^(\^|http)/i.test(trigger["url-filter"]))
+    trigger["url-filter"] = "^https?://.*" + trigger["url-filter"];
+
+  // For rules containing only a hostname we know that we're matching against
+  // a lowercase string unless the matchCase option was passed.
+  if (parsed.canSafelyMatchAsLowercase && !filter.matchCase)
+    trigger["url-filter"] = trigger["url-filter"].toLowerCase();
+
+  if (parsed.canSafelyMatchAsLowercase || filter.matchCase)
+    trigger["url-filter-is-case-sensitive"] = true;
+
   let included = [];
   let excluded = [];
 
   parseDomains(filter.domains, included, excluded);
 
   if (withResourceTypes)
     trigger["resource-type"] = getResourceTypes(filter);
   if (filter.thirdParty != null)
     trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"];
 
   if (included.length > 0)
     trigger["if-domain"] = addDomainPrefix(included);
   else if (excluded.length > 0)
     trigger["unless-domain"] = addDomainPrefix(excluded);
 
-  return {trigger: trigger, action: {type: action}};
+  rules.push({trigger: trigger, action: {type: action}});
+
+  return rules;

[Sebastian Noack, 2016/05/17 10:55:24]

You don't have to return the rules here anymore.

[kzar, 2016/05/17 11:23:34]

Oops, Done.

 }
 
 function hasNonASCI(obj)
 {
   if (typeof obj == "string")
   {
     if (/[^\x00-\x7F]/.test(obj))
       return true;
   }
 
@@ skipping 90 matching lines @@
     return;
   if (filter instanceof filterClasses.RegExpFilter &&
       filter.regexpSource == null)
     return;
 
   if (filter instanceof filterClasses.BlockingFilter)
     this.requestFilters.push(filter);
 
   if (filter instanceof filterClasses.WhitelistFilter)
   {
-    if (filter.contentType & (typeMap.IMAGE
+    if (filter.contentType & (typeMap.DOCUMENT
+                              | typeMap.IMAGE
                               | typeMap.STYLESHEET
                               | typeMap.SCRIPT
                               | typeMap.FONT
                               | typeMap.MEDIA
                               | typeMap.POPUP
                               | typeMap.OBJECT
                               | typeMap.OBJECT_SUBREQUEST
                               | typeMap.XMLHTTPREQUEST
                               | typeMap.PING
                               | typeMap.SUBDOCUMENT
@@ skipping 19 matching lines @@
 
 /**
  * Generate content blocker list for all filters that were added
  *
  * @returns   {Filter}   filter    Filter to convert
  */
 ContentBlockerList.prototype.generateRules = function(filter)
 {
   let rules = [];
 
-  function addRule(rule)
-  {
-    if (!hasNonASCI(rule))
-      rules.push(rule);
-  }
-
   let groupedElemhideFilters = new Map();
   for (let filter of this.elemhideFilters)
   {
     let result = convertElemHideFilter(filter, this.elemhideSelectorExceptions);
     if (!result)
       continue;
 
     if (result.matchDomains.length == 0)
       result.matchDomains = ["^https?://"];
 
     for (let matchDomain of result.matchDomains)
     {
       let group = groupedElemhideFilters.get(matchDomain) || [];
       group.push(result.selector);
       groupedElemhideFilters.set(matchDomain, group);
     }
   }
 
   groupedElemhideFilters.forEach((selectors, matchDomain) =>
   {
     while (selectors.length)
     {
       let selector = selectors.splice(0, selectorLimit).join(", ");
 
       // As of Safari 9.0 element IDs are matched as lowercase. We work around
       // this by converting to the attribute format [id="elementID"]
       selector = convertIDSelectorsToAttributeSelectors(selector);
 
-      addRule({
+      rules.push({
         trigger: {"url-filter": matchDomain,
                   "url-filter-is-case-sensitive": true},
         action: {type: "css-display-none",
                  selector: selector}
       });
     }
   });
 
   for (let filter of this.elemhideExceptions)
-    addRule(convertFilter(filter, "ignore-previous-rules", false));
+    convertFilterAddRules(rules, filter, "ignore-previous-rules", false);
   for (let filter of this.requestFilters)
-    addRule(convertFilter(filter, "block", true));
+    convertFilterAddRules(rules, filter, "block", true);
   for (let filter of this.requestExceptions)
-    addRule(convertFilter(filter, "ignore-previous-rules", true));
+    convertFilterAddRules(rules, filter, "ignore-previous-rules", true);
 
-  return rules;
+  return rules.filter(rule => !hasNonASCI(rule));
 };