Issue 3585 - Merge element hiding rules for the same domain

abp2blocklist.js

 "use strict";
 
-var readline = require("readline");
-var punycode = require("punycode");
-var tldjs = require("tldjs");
-var filterClasses = require("./adblockplus.js");
-
-var typeMap = filterClasses.RegExpFilter.typeMap;
-
-var requestFilters = [];
-var requestExceptions = [];
-var elemhideFilters = [];
-var elemhideExceptions = [];
-var elemhideSelectorExceptions = Object.create(null);
+let readline = require("readline");
+let punycode = require("punycode");
+let tldjs = require("tldjs");
+let filterClasses = require("./adblockplus.js");
+
+let typeMap = filterClasses.RegExpFilter.typeMap;
+
+const selectorLimit = 5000;
+
+let requestFilters = [];
+let requestExceptions = [];
+let elemhideFilters = [];
+let elemhideExceptions = [];
+let elemhideSelectorExceptions = new Map();
 
 function recordException(filter)
 {
   if (filter.contentType & (typeMap.IMAGE
                             | typeMap.STYLESHEET
                             | typeMap.SCRIPT
                             | typeMap.FONT
                             | typeMap.MEDIA
                             | typeMap.POPUP
                             | typeMap.OBJECT
                             | typeMap.OBJECT_SUBREQUEST
                             | typeMap.XMLHTTPREQUEST
                             | typeMap.PING
                             | typeMap.SUBDOCUMENT
                             | typeMap.OTHER))
     requestExceptions.push(filter);
 
     if (filter.contentType & typeMap.ELEMHIDE)
       elemhideExceptions.push(filter);
 }
 
 function parseDomains(domains, included, excluded)
 {
-  for (var domain in domains)
+  for (let domain in domains)
   {
     if (domain != "")
     {
-      var enabled = domains[domain];
+      let enabled = domains[domain];
       domain = punycode.toASCII(domain.toLowerCase());
 
       if (!enabled)
         excluded.push(domain);
       else if (!domains[""])
         included.push(domain);
     }
   }
 }
 
 function recordSelectorException(filter)
 {
-  var domains = elemhideSelectorExceptions[filter.selector];
+  let domains = elemhideSelectorExceptions[filter.selector];
   if (!domains)
     domains = elemhideSelectorExceptions[filter.selector] = [];
 
   parseDomains(filter.domains, domains, []);
 }
 
 function parseFilter(line)
 {
   if (line.charAt(0) == "[")
     return;
 
-  var filter = filterClasses.Filter.fromText(line);
+  let filter = filterClasses.Filter.fromText(line);
 
   if (filter.sitekeys)
     return;
   if (filter instanceof filterClasses.RegExpFilter && !filter.regexpSource)
     return;
 
   if (filter instanceof filterClasses.BlockingFilter)
     requestFilters.push(filter);
   if (filter instanceof filterClasses.WhitelistFilter)
     recordException(filter);
   if (filter instanceof filterClasses.ElemHideFilter)
     elemhideFilters.push(filter);
   if (filter instanceof filterClasses.ElemHideException)
     recordSelectorException(filter);
 }
 
 function escapeRegExp(s)
 {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
 
 function matchDomain(domain)
 {
   return "^https?://([^/:]*\\.)?" + escapeRegExp(domain) + "[/:]";
 }
 
 function convertElemHideFilter(filter)
 {
-  var included = [];
-  var excluded = [];
-  var rules = [];
+  let included = [];
+  let excluded = [];
+  let rules = [];
 
   parseDomains(filter.domains, included, excluded);
 
   if (excluded.length == 0 && !(filter.selector in elemhideSelectorExceptions))
-    return [included.map(matchDomain), filter.selector];
+    return {matchDomains: included.map(matchDomain), selector: filter.selector};
 }
 
 function toRegExp(text)
 {
-  var result = "";
-  var lastIndex = text.length - 1;
-
-  for (var i = 0; i < text.length; i++)
-  {
-    var c = text[i];
+  let result = "";
+  let lastIndex = text.length - 1;
+
+  for (let i = 0; i < text.length; i++)
+  {
+    let c = text[i];
 
     switch (c)
     {
       case "*":
         if (result.length > 0 && i < lastIndex && text[i + 1] != "*")
           result += ".*";
         break;
       case "^":
         if (i < lastIndex)
           result += ".";
@@ skipping 21 matching lines @@
       default:
         result += c;
     }
   }
 
   return result;
 }
 
 function getRegExpSource(filter)
 {
-  var source = toRegExp(filter.regexpSource.replace(
+  let source = toRegExp(filter.regexpSource.replace(
     // Safari expects punycode, filter lists use unicode
     /^(\|\||\|?https?:\/\/)([\w\-.*\u0080-\uFFFF]+)/i,
     function (match, prefix, domain)
     {
       return prefix + punycode.toASCII(domain);
     }
   ));
 
   // Limit rules to to HTTP(S) URLs
   if (!/^(\^|http)/i.test(source))
     source = "^https?://.*" + source;
 
   return source;
 }
 
 function getResourceTypes(filter)
 {
-  var types = [];
+  let types = [];
 
   if (filter.contentType & typeMap.IMAGE)
     types.push("image");
   if (filter.contentType & typeMap.STYLESHEET)
     types.push("style-sheet");
   if (filter.contentType & typeMap.SCRIPT)
     types.push("script");
   if (filter.contentType & typeMap.FONT)
     types.push("font");
   if (filter.contentType & (typeMap.MEDIA | typeMap.OBJECT))
     types.push("media");
   if (filter.contentType & typeMap.POPUP)
     types.push("popup");
   if (filter.contentType & (typeMap.XMLHTTPREQUEST | typeMap.OBJECT_SUBREQUEST
       | typeMap.PING | typeMap.OTHER))
     types.push("raw");
   if (filter.contentType & typeMap.SUBDOCUMENT)
     types.push("document");
 
   return types;
 }
 
 function addDomainPrefix(domains)
 {
-  var result = [];
-
-  for (var i = 0; i < domains.length; i++)
-  {
-    var domain = domains[i];
+  let result = [];
+
+  for (let domain of domains)
+  {
     result.push(domain);
 
     if (tldjs.getSubdomain(domain) == "")
       result.push("www." + domain);
   }
 
   return result;
 }
 
 function convertFilter(filter, action, withResourceTypes)
 {
-  var trigger = {"url-filter": getRegExpSource(filter)};
-  var included = [];
-  var excluded = [];
+  let trigger = {"url-filter": getRegExpSource(filter)};
+  let included = [];
+  let excluded = [];
 
   parseDomains(filter.domains, included, excluded);
 
   if (withResourceTypes)
     trigger["resource-type"] = getResourceTypes(filter);
   if (filter.thirdParty != null)
     trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"];
 
   if (included.length > 0)
     trigger["if-domain"] = addDomainPrefix(included);
   else if (excluded.length > 0)
     trigger["unless-domain"] = addDomainPrefix(excluded);
 
   return {trigger: trigger, action: {type: action}};
 }
 
 function hasNonASCI(obj)
 {
   if (typeof obj == "string")
   {
     if (/[^\x00-\x7F]/.test(obj))
       return true;
   }
 
   if (typeof obj == "object")
   {
-    var i;
     if (obj instanceof Array)
-      for (i = 0; i < obj.length; i++)
-        if (hasNonASCI(obj[i]))
+      for (let item of obj)
+        if (hasNonASCI(item))
           return true;
 
-    var names = Object.getOwnPropertyNames(obj);
-    for (i = 0; i < names.length; i++)
-      if (hasNonASCI(obj[names[i]]))
+    for (let name of Object.getOwnPropertyNames(obj))
+      if (hasNonASCI(obj[name]))
         return true;
   }
 
   return false;
 }
 
 function logRules()
 {
-  var rules = [];
-  var i;
+  let rules = [];
 
   function addRule(rule)
   {
     if (!hasNonASCI(rule))
       rules.push(rule);
   }
 
   let groupedElemhideFilters = new Map();
   for (let filter of elemhideFilters)

[Sebastian Noack, 2016/02/15 14:33:11]

Does these ES2015 features even work in node.js without any additional command
line options? I remember that when I initially wrote that code I couldn't use
let statements.

Moreover, regardless of ES2015 support in the latest node version, I think we
should either stick to var, or use let consistently in the whole file.

[kzar, 2016/02/15 18:19:20]

(As discussed in IRC we will use ES2015 features that are supported by node.js
stable without special arguments.)

   {
     let result = convertElemHideFilter(filter);
     if (!result)
       continue;
-    let targetDomains = result[0];
-    let selector = result[1];
-
-    if (targetDomains.length == 0)
-      targetDomains = ["^https?://"];
-
-    for (let domain of targetDomains)
-    {
-      if (!groupedElemhideFilters.has(domain))
-        groupedElemhideFilters.set(domain, []);
-      groupedElemhideFilters.get(domain).push(selector);
-    }
-  }
-
-  groupedElemhideFilters.forEach((selectors, domain) =>
-  {
-    let rule = {trigger: {"url-filter": domain},
-                action: {type: "css-display-none"}};
-
-    if (selectors.length == 1)
-      rule["action"]["selector"] = selectors[0];
-    else
-      rule["action"]["selector"] = ":matches(" + selectors.join(", ") + ")";

[kzar, 2016/02/13 19:33:24]

Note: I wasn't sure if we need to escape the selectors here.

[Sebastian Noack, 2016/02/15 14:33:11]

Well, if any CSS selector is invalid it will break the whole group of selectors.
But that is the same with the logic we have on Chrome. So I don't think we need
to bother. And as far as I understand :matches, it's not possible to prevent
that by escaping as we need raw selectors here.

However, I wonder whether we even need :matches() or whether we can simply join
the selectors by comma. It seems that is what AdBlock is doing. While looking at
their code I also figured that they never merge more than 1000 selectors into
one rule. Perhaps, also Dean can give some advice here?

[kzar, 2016/02/15 18:19:20]

Acknowledged.

-
-    addRule(rule);
+
+    if (result.matchDomains.length == 0)
+      result.matchDomains = ["^https?://"];
+
+    for (let matchDomain of result.matchDomains)
+    {
+      let group = groupedElemhideFilters.get(matchDomain) || [];
+      group.push(result.selector);
+      groupedElemhideFilters.set(matchDomain, group);
+    }
+  }
+
+  groupedElemhideFilters.forEach((selectors, matchDomain) =>
+  {
+    while (selectors.length)
+    {
+      addRule({
+        trigger: {"url-filter": matchDomain},
+        action: {type: "css-display-none",
+                 selector: selectors.splice(0, selectorLimit).join(", ")}
+      });
+    }
   });
 
-  for (i = 0; i < elemhideExceptions.length; i++)
-    addRule(convertFilter(elemhideExceptions[i], "ignore-previous-rules", false));
-
-  for (i = 0; i < requestFilters.length; i++)
-    addRule(convertFilter(requestFilters[i], "block", true));
-  for (i = 0; i < requestExceptions.length; i++)
-    addRule(convertFilter(requestExceptions[i], "ignore-previous-rules", true));
+  for (let filter of elemhideExceptions)
+    addRule(convertFilter(filter, "ignore-previous-rules", false));
+
+  for (let filter of requestFilters)
+    addRule(convertFilter(filter, "block", true));
+  for (let filter of requestExceptions)
+    addRule(convertFilter(filter, "ignore-previous-rules", true));
 
   console.log(JSON.stringify(rules, null, "\t"));
 }
 
-var rl = readline.createInterface({input: process.stdin, terminal: false});
+let rl = readline.createInterface({input: process.stdin, terminal: false});
 rl.on("line", parseFilter);
 rl.on("close", logRules);