Issue 7016 - Convert serialization functions into generators

test/filterClasses.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-present eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ skipping 65 matching lines @@
       for (let [domain, isIncluded] of filter.domains)
       {
         if (domain != "")
           domains.push(isIncluded ? domain : "~" + domain);
       }
     }
     result.push("domains=" + domains.sort().join("|"));
 
     if (filter instanceof RegExpFilter)
     {
-      result.push("regexp=" + filter.regexp.source);
+      result.push("regexp=" + (filter.regexp ? filter.regexp.source : null));
       result.push("contentType=" + filter.contentType);
       result.push("matchCase=" + filter.matchCase);
 
       let sitekeys = filter.sitekeys || [];
       result.push("sitekeys=" + sitekeys.slice().sort().join("|"));
 
       result.push("thirdParty=" + filter.thirdParty);
       if (filter instanceof BlockingFilter)
       {
         result.push("type=filterlist");
@@ skipping 56 matching lines @@
   {
     addProperty("disabled", "false");
     addProperty("lastHit", "0");
     addProperty("hitCount", "0");
   }
   if (type == "whitelist" || type == "filterlist")
   {
     addProperty("contentType", 0x7FFFFFFF & ~(
       t.CSP | t.DOCUMENT | t.ELEMHIDE | t.POPUP | t.GENERICHIDE | t.GENERICBLOCK
     ));
+    addProperty("regexp", "null");
     addProperty("matchCase", "false");
     addProperty("thirdParty", "null");
     addProperty("domains", "");
     addProperty("sitekeys", "");
   }
   if (type == "filterlist")
   {
     addProperty("collapse", "null");
     addProperty("csp", "null");
     addProperty("rewrite", "null");
@@ skipping 16 matching lines @@
   addDefaults(expected);
 
   let filter = Filter.fromText(text);
   if (postInit)
     postInit(filter);
   let result = serializeFilter(filter);
   test.equal(result.sort().join("\n"), expected.sort().join("\n"), text);
 
   // Test round-trip
   let filter2;
-  filter2 = Filter.fromText(filter.text);
+  let buffer = [...filter.serialize()];
+  if (buffer.length)
+  {
+    let map = Object.create(null);
+    for (let line of buffer.slice(1))
+    {
+      if (/(.*?)=(.*)/.test(line))
+        map[RegExp.$1] = RegExp.$2;
+    }
+    filter2 = Filter.fromObject(map);
+  }
+  else
+    filter2 = Filter.fromText(filter.text);
 
   test.equal(serializeFilter(filter).join("\n"), serializeFilter(filter2).join("\n"), text + " deserialization");
 }
 
 exports.testFilterClassDefinitions = function(test)
 {
   test.equal(typeof Filter, "function", "typeof Filter");
   test.equal(typeof InvalidFilter, "function", "typeof InvalidFilter");
   test.equal(typeof CommentFilter, "function", "typeof CommentFilter");
   test.equal(typeof ActiveFilter, "function", "typeof ActiveFilter");
@@ skipping 39 matching lines @@
   checkElemHideEmulationFilterInvalid("~foo.com");
   checkElemHideEmulationFilterInvalid("~foo.com,~bar.com");
   checkElemHideEmulationFilterInvalid("foo");
   checkElemHideEmulationFilterInvalid("~foo.com,bar");
 
   test.done();
 };
 
 exports.testFiltersWithState = function(test)
 {
-  compareFilter(test, "blabla", ["type=filterlist", "text=blabla", "regexp=blabla"]);
+  compareFilter(test, "blabla", ["type=filterlist", "text=blabla"]);
   compareFilter(
-    test, "blabla_default", ["type=filterlist", "text=blabla_default", "regexp=blabla_default"],
+    test, "blabla_default", ["type=filterlist", "text=blabla_default"],
     filter =>
     {
       filter.disabled = false;
       filter.hitCount = 0;
       filter.lastHit = 0;
     }
   );
   compareFilter(
     test, "blabla_non_default",
-    ["type=filterlist", "text=blabla_non_default", "regexp=blabla_non_default", "disabled=true", "hitCount=12", "lastHit=20"],
+    ["type=filterlist", "text=blabla_non_default", "disabled=true", "hitCount=12", "lastHit=20"],
     filter =>
     {
       filter.disabled = true;
       filter.hitCount = 12;
       filter.lastHit = 20;
     }
   );
 
   test.done();
 };
 
 exports.testSpecialCharacters = function(test)
 {
   compareFilter(test, "/ddd|f?a[s]d/", ["type=filterlist", "text=/ddd|f?a[s]d/", "regexp=ddd|f?a[s]d"]);
   compareFilter(test, "*asdf*d**dd*", ["type=filterlist", "text=*asdf*d**dd*", "regexp=asdf.*d.*dd"]);
   compareFilter(test, "|*asd|f*d**dd*|", ["type=filterlist", "text=|*asd|f*d**dd*|", "regexp=^.*asd\\|f.*d.*dd.*$"]);
-  compareFilter(test, "dd[]{}$%<>&()d", ["type=filterlist", "text=dd[]{}$%<>&()d", "regexp=dd\\[\\]\\{\\}\\$\\%\\<\\>\\&\\(\\)d"]);
+  compareFilter(test, "dd[]{}$%<>&()*d", ["type=filterlist", "text=dd[]{}$%<>&()*d", "regexp=dd\\[\\]\\{\\}\\$\\%\\<\\>\\&\\(\\).*d"]);
 
   // Leading and trailing wildcards should be left in for rewrite filters (#6868).
   compareFilter(test, "*asdf*d**dd*$rewrite=", ["type=filterlist", "text=*asdf*d**dd*$rewrite=", "regexp=.*asdf.*d.*dd.*", "rewrite=", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.SUBDOCUMENT | t.OBJECT | t.OBJECT_SUBREQUEST))]);
 
   compareFilter(test, "@@/ddd|f?a[s]d/", ["type=whitelist", "text=@@/ddd|f?a[s]d/", "regexp=ddd|f?a[s]d", "contentType=" + defaultTypes]);
   compareFilter(test, "@@*asdf*d**dd*", ["type=whitelist", "text=@@*asdf*d**dd*", "regexp=asdf.*d.*dd", "contentType=" + defaultTypes]);
   compareFilter(test, "@@|*asd|f*d**dd*|", ["type=whitelist", "text=@@|*asd|f*d**dd*|", "regexp=^.*asd\\|f.*d.*dd.*$", "contentType=" + defaultTypes]);
-  compareFilter(test, "@@dd[]{}$%<>&()d", ["type=whitelist", "text=@@dd[]{}$%<>&()d", "regexp=dd\\[\\]\\{\\}\\$\\%\\<\\>\\&\\(\\)d", "contentType=" + defaultTypes]);
+  compareFilter(test, "@@dd[]{}$%<>&()*d", ["type=whitelist", "text=@@dd[]{}$%<>&()*d", "regexp=dd\\[\\]\\{\\}\\$\\%\\<\\>\\&\\(\\).*d", "contentType=" + defaultTypes]);
 
   test.done();
 };
 
 exports.testFilterOptions = function(test)
 {
-  compareFilter(test, "bla$match-case,csp=first csp,script,other,third-party,domain=FOO.cOm,sitekey=foo", ["type=filterlist", "text=bla$match-case,csp=first csp,script,other,third-party,domain=FOO.cOm,sitekey=foo", "regexp=bla", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER | t.CSP), "thirdParty=true", "domains=foo.com", "sitekeys=FOO", "csp=first csp"]);
-  compareFilter(test, "bla$~match-case,~csp=csp,~script,~other,~third-party,domain=~bAr.coM", ["type=filterlist", "text=bla$~match-case,~csp=csp,~script,~other,~third-party,domain=~bAr.coM", "regexp=bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER)), "thirdParty=false", "domains=~bar.com"]);
-  compareFilter(test, "@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bAR.foO.Com|~Foo.Bar.com,csp=c s p,sitekey=foo|bar", ["type=whitelist", "text=@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bAR.foO.Com|~Foo.Bar.com,csp=c s p,sitekey=foo|bar", "regexp=bla", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER | t.CSP), "thirdParty=true", "domains=bar.com|foo.com|~bar.foo.com|~foo.bar.com", "sitekeys=BAR|FOO"]);
-  compareFilter(test, "@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,sitekey=foo|bar", ["type=whitelist", "text=@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,sitekey=foo|bar", "regexp=bla", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER), "thirdParty=true", "domains=bar.com|foo.com|~bar.foo.com|~foo.bar.com", "sitekeys=BAR|FOO"]);
+  compareFilter(test, "bla$match-case,csp=first csp,script,other,third-party,domain=FOO.cOm,sitekey=foo", ["type=filterlist", "text=bla$match-case,csp=first csp,script,other,third-party,domain=FOO.cOm,sitekey=foo", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER | t.CSP), "thirdParty=true", "domains=foo.com", "sitekeys=FOO", "csp=first csp"]);
+  compareFilter(test, "bla$~match-case,~csp=csp,~script,~other,~third-party,domain=~bAr.coM", ["type=filterlist", "text=bla$~match-case,~csp=csp,~script,~other,~third-party,domain=~bAr.coM", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER)), "thirdParty=false", "domains=~bar.com"]);
+  compareFilter(test, "@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bAR.foO.Com|~Foo.Bar.com,csp=c s p,sitekey=foo|bar", ["type=whitelist", "text=@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bAR.foO.Com|~Foo.Bar.com,csp=c s p,sitekey=foo|bar", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER | t.CSP), "thirdParty=true", "domains=bar.com|foo.com|~bar.foo.com|~foo.bar.com", "sitekeys=BAR|FOO"]);
+  compareFilter(test, "@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,sitekey=foo|bar", ["type=whitelist", "text=@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,sitekey=foo|bar", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER), "thirdParty=true", "domains=bar.com|foo.com|~bar.foo.com|~foo.bar.com", "sitekeys=BAR|FOO"]);
   compareFilter(test, "||content.server.com/files/*.php$rewrite=$1", ["type=filterlist", "text=||content.server.com/files/*.php$rewrite=$1", "regexp=^[\\w\\-]+:\\/+(?!\\/)(?:[^\\/]+\\.)?content\\.server\\.com\\/files\\/.*\\.php", "matchCase=false", "rewrite=$1", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.SUBDOCUMENT | t.OBJECT | t.OBJECT_SUBREQUEST))]);
 
   // background and image should be the same for backwards compatibility
-  compareFilter(test, "bla$image", ["type=filterlist", "text=bla$image", "regexp=bla", "contentType=" + (t.IMAGE)]);
-  compareFilter(test, "bla$background", ["type=filterlist", "text=bla$background", "regexp=bla", "contentType=" + (t.IMAGE)]);
-  compareFilter(test, "bla$~image", ["type=filterlist", "text=bla$~image", "regexp=bla", "contentType=" + (defaultTypes & ~t.IMAGE)]);
-  compareFilter(test, "bla$~background", ["type=filterlist", "text=bla$~background", "regexp=bla", "contentType=" + (defaultTypes & ~t.IMAGE)]);
-
-  compareFilter(test, "@@bla$~script,~other", ["type=whitelist", "text=@@bla$~script,~other", "regexp=bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
-  compareFilter(test, "@@http://bla$~script,~other", ["type=whitelist", "text=@@http://bla$~script,~other", "regexp=http\\:\\/\\/bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
+  compareFilter(test, "bla$image", ["type=filterlist", "text=bla$image", "contentType=" + (t.IMAGE)]);
+  compareFilter(test, "bla$background", ["type=filterlist", "text=bla$background", "contentType=" + (t.IMAGE)]);
+  compareFilter(test, "bla$~image", ["type=filterlist", "text=bla$~image", "contentType=" + (defaultTypes & ~t.IMAGE)]);
+  compareFilter(test, "bla$~background", ["type=filterlist", "text=bla$~background", "contentType=" + (defaultTypes & ~t.IMAGE)]);
+
+  compareFilter(test, "@@bla$~script,~other", ["type=whitelist", "text=@@bla$~script,~other", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
+  compareFilter(test, "@@http://bla$~script,~other", ["type=whitelist", "text=@@http://bla$~script,~other", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
   compareFilter(test, "@@|ftp://bla$~script,~other", ["type=whitelist", "text=@@|ftp://bla$~script,~other", "regexp=^ftp\\:\\/\\/bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
-  compareFilter(test, "@@bla$~script,~other,document", ["type=whitelist", "text=@@bla$~script,~other,document", "regexp=bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER) | t.DOCUMENT)]);
-  compareFilter(test, "@@bla$~script,~other,~document", ["type=whitelist", "text=@@bla$~script,~other,~document", "regexp=bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
-  compareFilter(test, "@@bla$document", ["type=whitelist", "text=@@bla$document", "regexp=bla", "contentType=" + t.DOCUMENT]);
-  compareFilter(test, "@@bla$~script,~other,elemhide", ["type=whitelist", "text=@@bla$~script,~other,elemhide", "regexp=bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER) | t.ELEMHIDE)]);
-  compareFilter(test, "@@bla$~script,~other,~elemhide", ["type=whitelist", "text=@@bla$~script,~other,~elemhide", "regexp=bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
-  compareFilter(test, "@@bla$elemhide", ["type=whitelist", "text=@@bla$elemhide", "regexp=bla", "contentType=" + t.ELEMHIDE]);
+  compareFilter(test, "@@bla$~script,~other,document", ["type=whitelist", "text=@@bla$~script,~other,document", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER) | t.DOCUMENT)]);
+  compareFilter(test, "@@bla$~script,~other,~document", ["type=whitelist", "text=@@bla$~script,~other,~document", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
+  compareFilter(test, "@@bla$document", ["type=whitelist", "text=@@bla$document", "contentType=" + t.DOCUMENT]);
+  compareFilter(test, "@@bla$~script,~other,elemhide", ["type=whitelist", "text=@@bla$~script,~other,elemhide", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER) | t.ELEMHIDE)]);
+  compareFilter(test, "@@bla$~script,~other,~elemhide", ["type=whitelist", "text=@@bla$~script,~other,~elemhide", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
+  compareFilter(test, "@@bla$elemhide", ["type=whitelist", "text=@@bla$elemhide", "contentType=" + t.ELEMHIDE]);
 
   compareFilter(test, "@@bla$~script,~other,donottrack", ["type=invalid", "text=@@bla$~script,~other,donottrack", "reason=filter_unknown_option"]);
   compareFilter(test, "@@bla$~script,~other,~donottrack", ["type=invalid", "text=@@bla$~script,~other,~donottrack", "reason=filter_unknown_option"]);
   compareFilter(test, "@@bla$donottrack", ["type=invalid", "text=@@bla$donottrack", "reason=filter_unknown_option"]);
   compareFilter(test, "@@bla$foobar", ["type=invalid", "text=@@bla$foobar", "reason=filter_unknown_option"]);
   compareFilter(test, "@@bla$image,foobar", ["type=invalid", "text=@@bla$image,foobar", "reason=filter_unknown_option"]);
   compareFilter(test, "@@bla$foobar,image", ["type=invalid", "text=@@bla$foobar,image", "reason=filter_unknown_option"]);
 
   compareFilter(test, "bla$csp", ["type=invalid", "text=bla$csp", "reason=filter_invalid_csp"]);
   compareFilter(test, "bla$csp=", ["type=invalid", "text=bla$csp=", "reason=filter_invalid_csp"]);
+
+  // Blank CSP values are allowed for whitelist filters.
+  compareFilter(test, "@@bla$csp", ["type=whitelist", "text=@@bla$csp", "contentType=" + t.CSP]);
+  compareFilter(test, "@@bla$csp=", ["type=whitelist", "text=@@bla$csp=", "contentType=" + t.CSP]);
+
   compareFilter(test, "bla$csp=report-uri", ["type=invalid", "text=bla$csp=report-uri", "reason=filter_invalid_csp"]);
   compareFilter(test, "bla$csp=foo,csp=report-to", ["type=invalid", "text=bla$csp=foo,csp=report-to", "reason=filter_invalid_csp"]);
   compareFilter(test, "bla$csp=foo,csp=referrer foo", ["type=invalid", "text=bla$csp=foo,csp=referrer foo", "reason=filter_invalid_csp"]);
   compareFilter(test, "bla$csp=foo,csp=base-uri", ["type=invalid", "text=bla$csp=foo,csp=base-uri", "reason=filter_invalid_csp"]);
   compareFilter(test, "bla$csp=foo,csp=upgrade-insecure-requests", ["type=invalid", "text=bla$csp=foo,csp=upgrade-insecure-requests", "reason=filter_invalid_csp"]);
   compareFilter(test, "bla$csp=foo,csp=ReFeRReR", ["type=invalid", "text=bla$csp=foo,csp=ReFeRReR", "reason=filter_invalid_csp"]);
 
   test.done();
 };
 
@@ skipping 146 matching lines @@
     normalized,
     "b$la$sitekey=foo,domain=domain.com|foo.com,csp=c s p"
   );
   compareFilter(
     test, normalized, [
       "type=filterlist",
       "text=" + normalized,
       "csp=c s p",
       "domains=domain.com|foo.com",
       "sitekeys=FOO",
-      "regexp=b\\$la",
       "contentType=" + t.CSP
     ]
   );
 
   // Some $csp edge cases
   test.equal(Filter.normalize("$csp=  "),
              "$csp=");
   test.equal(Filter.normalize("$csp= c s p"),
              "$csp=c s p");
   test.equal(Filter.normalize("$$csp= c s p"),
@@ skipping 74 matching lines @@
   test.equal(
     filterStrip.rewriteUrl("http://example.com/?tag"),
     "http://example.com/?"
   );
 
   test.done();
 };
 
 exports.testDomainMapDeduplication = function(test)
 {
-  let filter1 = Filter.fromText("example.com##.foo");
-  let filter2 = Filter.fromText("example.com##.bar");
+  let filter1 = Filter.fromText("foo$domain=blocking.example.com");
+  let filter2 = Filter.fromText("bar$domain=blocking.example.com");
+  let filter3 = Filter.fromText("elemhide.example.com##.foo");
+  let filter4 = Filter.fromText("elemhide.example.com##.bar");
 
   // This compares the references to make sure that both refer to the same
   // object (#6815).
   test.equal(filter1.domains, filter2.domains);
 
-  let filter3 = Filter.fromText("www.example.com##.bar");
-
-  test.notEqual(filter2.domains, filter3.domains);
-
-  test.done();
-};
+  // For element hiding filters, the value of the property is cached internally
+  // only on second access.
+  test.notEqual(filter3.domains, filter4.domains);
+  test.equal(filter3.domains, filter4.domains);
+
+  let filter5 = Filter.fromText("bar$domain=www.example.com");
+  let filter6 = Filter.fromText("www.example.com##.bar");
+
+  test.notEqual(filter2.domains, filter5.domains);
+
+  // Check twice for element hiding filters to make sure the internal cached
+  // values are also not equal.
+  test.notEqual(filter4.domains, filter6.domains);
+  test.notEqual(filter4.domains, filter6.domains);
+
+  test.done();
+};