Issue 6735 - Store domains in lower case

test/filterClasses.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-present eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ skipping 273 matching lines @@
   compareFilter(test, "@@/ddd|f?a[s]d/", ["type=whitelist", "text=@@/ddd|f?a[s]d/", "regexp=ddd|f?a[s]d", "contentType=" + defaultTypes]);
   compareFilter(test, "@@*asdf*d**dd*", ["type=whitelist", "text=@@*asdf*d**dd*", "regexp=asdf.*d.*dd", "contentType=" + defaultTypes]);
   compareFilter(test, "@@|*asd|f*d**dd*|", ["type=whitelist", "text=@@|*asd|f*d**dd*|", "regexp=^.*asd\\|f.*d.*dd.*$", "contentType=" + defaultTypes]);
   compareFilter(test, "@@dd[]{}$%<>&()d", ["type=whitelist", "text=@@dd[]{}$%<>&()d", "regexp=dd\\[\\]\\{\\}\\$\\%\\<\\>\\&\\(\\)d", "contentType=" + defaultTypes]);
 
   test.done();
 };
 
 exports.testFilterOptions = function(test)
 {
-  compareFilter(test, "bla$match-case,csp=first csp,script,other,third-party,domain=foo.com,sitekey=foo", ["type=filterlist", "text=bla$match-case,csp=first csp,script,other,third-party,domain=foo.com,sitekey=foo", "regexp=bla", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER | t.CSP), "thirdParty=true", "domains=FOO.COM", "sitekeys=FOO", "csp=first csp"]);
-  compareFilter(test, "bla$~match-case,~csp=csp,~script,~other,~third-party,domain=~bar.com", ["type=filterlist", "text=bla$~match-case,~csp=csp,~script,~other,~third-party,domain=~bar.com", "regexp=bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER)), "thirdParty=false", "domains=~BAR.COM"]);
-  compareFilter(test, "@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,csp=c s p,sitekey=foo|bar", ["type=whitelist", "text=@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,csp=c s p,sitekey=foo|bar", "regexp=bla", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER | t.CSP), "thirdParty=true", "domains=BAR.COM|FOO.COM|~BAR.FOO.COM|~FOO.BAR.COM", "sitekeys=BAR|FOO"]);
-  compareFilter(test, "@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,sitekey=foo|bar", ["type=whitelist", "text=@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,sitekey=foo|bar", "regexp=bla", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER), "thirdParty=true", "domains=BAR.COM|FOO.COM|~BAR.FOO.COM|~FOO.BAR.COM", "sitekeys=BAR|FOO"]);
+  compareFilter(test, "bla$match-case,csp=first csp,script,other,third-party,domain=foo.com,sitekey=foo", ["type=filterlist", "text=bla$match-case,csp=first csp,script,other,third-party,domain=foo.com,sitekey=foo", "regexp=bla", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER | t.CSP), "thirdParty=true", "domains=foo.com", "sitekeys=FOO", "csp=first csp"]);
+  compareFilter(test, "bla$~match-case,~csp=csp,~script,~other,~third-party,domain=~bar.com", ["type=filterlist", "text=bla$~match-case,~csp=csp,~script,~other,~third-party,domain=~bar.com", "regexp=bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER)), "thirdParty=false", "domains=~bar.com"]);
+  compareFilter(test, "@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,csp=c s p,sitekey=foo|bar", ["type=whitelist", "text=@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,csp=c s p,sitekey=foo|bar", "regexp=bla", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER | t.CSP), "thirdParty=true", "domains=bar.com|foo.com|~bar.foo.com|~foo.bar.com", "sitekeys=BAR|FOO"]);
+  compareFilter(test, "@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,sitekey=foo|bar", ["type=whitelist", "text=@@bla$match-case,script,other,third-party,domain=foo.com|bar.com|~bar.foo.com|~foo.bar.com,sitekey=foo|bar", "regexp=bla", "matchCase=true", "contentType=" + (t.SCRIPT | t.OTHER), "thirdParty=true", "domains=bar.com|foo.com|~bar.foo.com|~foo.bar.com", "sitekeys=BAR|FOO"]);
   compareFilter(test, "||content.server.com/files/*.php$rewrite=$1", ["type=filterlist", "text=||content.server.com/files/*.php$rewrite=$1", "regexp=^[\\w\\-]+:\\/+(?!\\/)(?:[^\\/]+\\.)?content\\.server\\.com\\/files\\/.*\\.php", "matchCase=false", "rewrite=$1", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.SUBDOCUMENT | t.OBJECT | t.OBJECT_SUBREQUEST))]);
 
   // background and image should be the same for backwards compatibility
   compareFilter(test, "bla$image", ["type=filterlist", "text=bla$image", "regexp=bla", "contentType=" + (t.IMAGE)]);
   compareFilter(test, "bla$background", ["type=filterlist", "text=bla$background", "regexp=bla", "contentType=" + (t.IMAGE)]);
   compareFilter(test, "bla$~image", ["type=filterlist", "text=bla$~image", "regexp=bla", "contentType=" + (defaultTypes & ~t.IMAGE)]);
   compareFilter(test, "bla$~background", ["type=filterlist", "text=bla$~background", "regexp=bla", "contentType=" + (defaultTypes & ~t.IMAGE)]);
 
   compareFilter(test, "@@bla$~script,~other", ["type=whitelist", "text=@@bla$~script,~other", "regexp=bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
   compareFilter(test, "@@http://bla$~script,~other", ["type=whitelist", "text=@@http://bla$~script,~other", "regexp=http\\:\\/\\/bla", "contentType=" + (defaultTypes & ~(t.SCRIPT | t.OTHER))]);
@@ skipping 19 matching lines @@
   compareFilter(test, "bla$csp=foo,csp=upgrade-insecure-requests", ["type=invalid", "text=bla$csp=foo,csp=upgrade-insecure-requests", "reason=filter_invalid_csp"]);
   compareFilter(test, "bla$csp=foo,csp=ReFeRReR", ["type=invalid", "text=bla$csp=foo,csp=ReFeRReR", "reason=filter_invalid_csp"]);
 
   test.done();
 };
 
 exports.testElementHidingRules = function(test)
 {
   compareFilter(test, "##ddd", ["type=elemhide", "text=##ddd", "selector=ddd"]);
   compareFilter(test, "##body > div:first-child", ["type=elemhide", "text=##body > div:first-child", "selector=body > div:first-child"]);
-  compareFilter(test, "foo##ddd", ["type=elemhide", "text=foo##ddd", "selectorDomains=foo", "selector=ddd", "domains=FOO"]);
-  compareFilter(test, "foo,bar##ddd", ["type=elemhide", "text=foo,bar##ddd", "selectorDomains=foo,bar", "selector=ddd", "domains=BAR|FOO"]);
-  compareFilter(test, "foo,~bar##ddd", ["type=elemhide", "text=foo,~bar##ddd", "selectorDomains=foo", "selector=ddd", "domains=FOO|~BAR"]);
-  compareFilter(test, "foo,~baz,bar##ddd", ["type=elemhide", "text=foo,~baz,bar##ddd", "selectorDomains=foo,bar", "selector=ddd", "domains=BAR|FOO|~BAZ"]);
+  compareFilter(test, "foo##ddd", ["type=elemhide", "text=foo##ddd", "selectorDomains=foo", "selector=ddd", "domains=foo"]);
+  compareFilter(test, "foo,bar##ddd", ["type=elemhide", "text=foo,bar##ddd", "selectorDomains=foo,bar", "selector=ddd", "domains=bar|foo"]);
+  compareFilter(test, "foo,~bar##ddd", ["type=elemhide", "text=foo,~bar##ddd", "selectorDomains=foo", "selector=ddd", "domains=foo|~bar"]);
+  compareFilter(test, "foo,~baz,bar##ddd", ["type=elemhide", "text=foo,~baz,bar##ddd", "selectorDomains=foo,bar", "selector=ddd", "domains=bar|foo|~baz"]);
 
   test.done();
 };
 
 exports.testElementHidingExceptions = function(test)
 {
   compareFilter(test, "#@#ddd", ["type=elemhideexception", "text=#@#ddd", "selector=ddd"]);
   compareFilter(test, "#@#body > div:first-child", ["type=elemhideexception", "text=#@#body > div:first-child", "selector=body > div:first-child"]);
-  compareFilter(test, "foo#@#ddd", ["type=elemhideexception", "text=foo#@#ddd", "selectorDomains=foo", "selector=ddd", "domains=FOO"]);
-  compareFilter(test, "foo,bar#@#ddd", ["type=elemhideexception", "text=foo,bar#@#ddd", "selectorDomains=foo,bar", "selector=ddd", "domains=BAR|FOO"]);
-  compareFilter(test, "foo,~bar#@#ddd", ["type=elemhideexception", "text=foo,~bar#@#ddd", "selectorDomains=foo", "selector=ddd", "domains=FOO|~BAR"]);
-  compareFilter(test, "foo,~baz,bar#@#ddd", ["type=elemhideexception", "text=foo,~baz,bar#@#ddd", "selectorDomains=foo,bar", "selector=ddd", "domains=BAR|FOO|~BAZ"]);
+  compareFilter(test, "foo#@#ddd", ["type=elemhideexception", "text=foo#@#ddd", "selectorDomains=foo", "selector=ddd", "domains=foo"]);
+  compareFilter(test, "foo,bar#@#ddd", ["type=elemhideexception", "text=foo,bar#@#ddd", "selectorDomains=foo,bar", "selector=ddd", "domains=bar|foo"]);
+  compareFilter(test, "foo,~bar#@#ddd", ["type=elemhideexception", "text=foo,~bar#@#ddd", "selectorDomains=foo", "selector=ddd", "domains=foo|~bar"]);
+  compareFilter(test, "foo,~baz,bar#@#ddd", ["type=elemhideexception", "text=foo,~baz,bar#@#ddd", "selectorDomains=foo,bar", "selector=ddd", "domains=bar|foo|~baz"]);
 
   test.done();
 };
 
 exports.testElemHideEmulationFilters = function(test)
 {
   // Check valid domain combinations
-  compareFilter(test, "foo.com#?#:-abp-properties(abc)", ["type=elemhideemulation", "text=foo.com#?#:-abp-properties(abc)", "selectorDomains=foo.com", "selector=:-abp-properties(abc)", "domains=FOO.COM"]);
-  compareFilter(test, "foo.com,~bar.com#?#:-abp-properties(abc)", ["type=elemhideemulation", "text=foo.com,~bar.com#?#:-abp-properties(abc)", "selectorDomains=foo.com", "selector=:-abp-properties(abc)", "domains=FOO.COM|~BAR.COM"]);
-  compareFilter(test, "foo.com,~bar#?#:-abp-properties(abc)", ["type=elemhideemulation", "text=foo.com,~bar#?#:-abp-properties(abc)", "selectorDomains=foo.com", "selector=:-abp-properties(abc)", "domains=FOO.COM|~BAR"]);
-  compareFilter(test, "~foo.com,bar.com#?#:-abp-properties(abc)", ["type=elemhideemulation", "text=~foo.com,bar.com#?#:-abp-properties(abc)", "selectorDomains=bar.com", "selector=:-abp-properties(abc)", "domains=BAR.COM|~FOO.COM"]);
+  compareFilter(test, "foo.com#?#:-abp-properties(abc)", ["type=elemhideemulation", "text=foo.com#?#:-abp-properties(abc)", "selectorDomains=foo.com", "selector=:-abp-properties(abc)", "domains=foo.com"]);
+  compareFilter(test, "foo.com,~bar.com#?#:-abp-properties(abc)", ["type=elemhideemulation", "text=foo.com,~bar.com#?#:-abp-properties(abc)", "selectorDomains=foo.com", "selector=:-abp-properties(abc)", "domains=foo.com|~bar.com"]);
+  compareFilter(test, "foo.com,~bar#?#:-abp-properties(abc)", ["type=elemhideemulation", "text=foo.com,~bar#?#:-abp-properties(abc)", "selectorDomains=foo.com", "selector=:-abp-properties(abc)", "domains=foo.com|~bar"]);
+  compareFilter(test, "~foo.com,bar.com#?#:-abp-properties(abc)", ["type=elemhideemulation", "text=~foo.com,bar.com#?#:-abp-properties(abc)", "selectorDomains=bar.com", "selector=:-abp-properties(abc)", "domains=bar.com|~foo.com"]);
 
   // Check some special cases
   compareFilter(test, "#?#:-abp-properties(abc)", ["type=invalid", "text=#?#:-abp-properties(abc)", "reason=filter_elemhideemulation_nodomain"]);
-  compareFilter(test, "foo.com#?#abc", ["type=elemhideemulation", "text=foo.com#?#abc", "selectorDomains=foo.com", "selector=abc", "domains=FOO.COM"]);
-  compareFilter(test, "foo.com#?#:-abp-foobar(abc)", ["type=elemhideemulation", "text=foo.com#?#:-abp-foobar(abc)", "selectorDomains=foo.com", "selector=:-abp-foobar(abc)", "domains=FOO.COM"]);
-  compareFilter(test, "foo.com#?#aaa :-abp-properties(abc) bbb", ["type=elemhideemulation", "text=foo.com#?#aaa :-abp-properties(abc) bbb", "selectorDomains=foo.com", "selector=aaa :-abp-properties(abc) bbb", "domains=FOO.COM"]);
-  compareFilter(test, "foo.com#?#:-abp-properties(|background-image: url(data:*))", ["type=elemhideemulation", "text=foo.com#?#:-abp-properties(|background-image: url(data:*))", "selectorDomains=foo.com", "selector=:-abp-properties(|background-image: url(data:*))", "domains=FOO.COM"]);
+  compareFilter(test, "foo.com#?#abc", ["type=elemhideemulation", "text=foo.com#?#abc", "selectorDomains=foo.com", "selector=abc", "domains=foo.com"]);
+  compareFilter(test, "foo.com#?#:-abp-foobar(abc)", ["type=elemhideemulation", "text=foo.com#?#:-abp-foobar(abc)", "selectorDomains=foo.com", "selector=:-abp-foobar(abc)", "domains=foo.com"]);
+  compareFilter(test, "foo.com#?#aaa :-abp-properties(abc) bbb", ["type=elemhideemulation", "text=foo.com#?#aaa :-abp-properties(abc) bbb", "selectorDomains=foo.com", "selector=aaa :-abp-properties(abc) bbb", "domains=foo.com"]);
+  compareFilter(test, "foo.com#?#:-abp-properties(|background-image: url(data:*))", ["type=elemhideemulation", "text=foo.com#?#:-abp-properties(|background-image: url(data:*))", "selectorDomains=foo.com", "selector=:-abp-properties(|background-image: url(data:*))", "domains=foo.com"]);
 
   // Check conversion of legacy filters
-  compareFilter(test, "foo.com##[-abp-properties='abc']", ["type=elemhideemulation", "text=foo.com#?#:-abp-properties(abc)", "selectorDomains=foo.com", "selector=:-abp-properties(abc)", "domains=FOO.COM"]);
+  compareFilter(test, "foo.com##[-abp-properties='abc']", ["type=elemhideemulation", "text=foo.com#?#:-abp-properties(abc)", "selectorDomains=foo.com", "selector=:-abp-properties(abc)", "domains=foo.com"]);
   test.equal(Filter.fromText("foo.com##[-abp-properties='abc']"), Filter.fromText("foo.com#?#:-abp-properties(abc)"));
-  compareFilter(test, "foo.com#@#[-abp-properties='abc']", ["type=elemhideexception", "text=foo.com#@#[-abp-properties='abc']", "selectorDomains=foo.com", "selector=[-abp-properties='abc']", "domains=FOO.COM"]);
-  compareFilter(test, "foo.com#?#[-abp-properties='abc']", ["type=elemhideemulation", "text=foo.com#?#[-abp-properties='abc']", "selectorDomains=foo.com", "selector=[-abp-properties='abc']", "domains=FOO.COM"]);
-  compareFilter(test, "foo.com##aaa [-abp-properties='abc'] bbb", ["type=elemhideemulation", "text=foo.com#?#aaa :-abp-properties(abc) bbb", "selectorDomains=foo.com", "selector=aaa :-abp-properties(abc) bbb", "domains=FOO.COM"]);
+  compareFilter(test, "foo.com#@#[-abp-properties='abc']", ["type=elemhideexception", "text=foo.com#@#[-abp-properties='abc']", "selectorDomains=foo.com", "selector=[-abp-properties='abc']", "domains=foo.com"]);
+  compareFilter(test, "foo.com#?#[-abp-properties='abc']", ["type=elemhideemulation", "text=foo.com#?#[-abp-properties='abc']", "selectorDomains=foo.com", "selector=[-abp-properties='abc']", "domains=foo.com"]);
+  compareFilter(test, "foo.com##aaa [-abp-properties='abc'] bbb", ["type=elemhideemulation", "text=foo.com#?#aaa :-abp-properties(abc) bbb", "selectorDomains=foo.com", "selector=aaa :-abp-properties(abc) bbb", "domains=foo.com"]);
 
   // test matching -abp-properties= (https://issues.adblockplus.org/ticket/5037).
-  compareFilter(test, "foo.com##[-abp-properties-bogus='abc']", ["type=elemhide", "text=foo.com##[-abp-properties-bogus='abc']", "selectorDomains=foo.com", "selector=[-abp-properties-bogus='abc']", "domains=FOO.COM"]);
+  compareFilter(test, "foo.com##[-abp-properties-bogus='abc']", ["type=elemhide", "text=foo.com##[-abp-properties-bogus='abc']", "selectorDomains=foo.com", "selector=[-abp-properties-bogus='abc']", "domains=foo.com"]);
 
   test.done();
 };
 
 exports.testEmptyElemHideDomains = function(test)
 {
   let emptyDomainFilters = [
     ",##selector", ",,,##selector", "~,foo.com##selector", "foo.com,##selector",
     ",foo.com##selector", "foo.com,~##selector",
     "foo.com,,bar.com##selector", "foo.com,~,bar.com##selector"
@@ skipping 19 matching lines @@
       "selector=#foo\\7B color: red\\7D ",
       "domains="
     ]
   );
   compareFilter(
     test, "foo.com#?#:-abp-properties(/margin: [3-4]{2}/)", [
       "type=elemhideemulation",
       "text=foo.com#?#:-abp-properties(/margin: [3-4]{2}/)",
       "selectorDomains=foo.com",
       "selector=:-abp-properties(/margin: [3-4]\\7B 2\\7D /)",
-      "domains=FOO.COM"
+      "domains=foo.com"
     ]
   );
   test.done();
 };
 
 exports.testFilterNormalization = function(test)
 {
   // Line breaks etc
   test.equal(Filter.normalize("\n\t\nad\ns"),
              "ads");
@@ skipping 30 matching lines @@
   // to be a separator
   test.equal(Filter.normalize("   domain.c  om# @## sele ctor   "),
              "domain.com#@##selector");
 
   // Regular filters
   let normalized = Filter.normalize(
     "    b$l     a$sitekey=  foo  ,domain= do main.com |foo   .com,c sp= c   s p  "
   );
   test.equal(
     normalized,
     "b$la$sitekey=foo,domain=domain.com|foo.com,csp=c s p"

[sergei, 2018/06/07 09:25:17]

it seems the input should be rather something like
b$la$sitekey=foo,domain=domain.com|FOO.cOm,csp=c s p

[Manish Jethani, 2018/06/07 10:22:14]

The normalization does not affect the case of the filter text unfortunately.
Perhaps it should? Anyway, if we decide to do that then it should be a separate
Trac issue.

[sergei, 2018/06/07 10:44:48]

Right, sorry, I missed the beginning of the test, I thought the test checks the
values of the filter fields. BTW, the domains with capital letters should be in
the original filter texts in tests above.

[Manish Jethani, 2018/06/07 11:03:30]

Ah, makes sense, sorry I missed your point earlier.

Done.

   );
   compareFilter(
     test, normalized, [
       "type=filterlist",
       "text=" + normalized,
       "csp=c s p",
-      "domains=DOMAIN.COM|FOO.COM",
+      "domains=domain.com|foo.com",
       "sitekeys=FOO",
       "regexp=b\\$la",
       "contentType=" + t.CSP
     ]
   );
 
   // Some $csp edge cases
   test.equal(Filter.normalize("$csp= c s p"),
              "$csp=c s p");
   test.equal(Filter.normalize("$$csp= c s p"),
@@ skipping 52 matching lines @@
     filterRelative.rewriteUrl("http://content.server/file/foo.txt?bar"),
     "http://content.server/file/foo.txt/disable"
   );
   test.equal(
     filterRelative.rewriteUrl("http://example.com/file/foo.txt?bar"),
     "http://example.com/file/foo.txt/disable"
   );
 
   test.done();
 };