Issue 1801 - Use URL objects to process URLs in the background page

qunit/tests/url.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-2015 Eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ skipping 10 matching lines @@
   var url = require("url");
   var getDecodedHostname = url.getDecodedHostname;
   var extractHostFromFrame = url.extractHostFromFrame;
   var stringifyURL = url.stringifyURL;
   var isThirdParty = url.isThirdParty;
 
   module("URL/host tools");
 
   test("Extracting hostname from URL", function()
   {
-    equal(getDecodedHostname(new URL("http://example.com/foo")), "example.com", "with path");
-    equal(getDecodedHostname(new URL("http://example.com/?foo=bar")), "example.com", "with query");
-    equal(getDecodedHostname(new URL("http://example.com/#top")), "example.com", "with hash");
-    equal(getDecodedHostname(new URL("http://example.com/:8080")), "example.com", "with port");

[Wladimir Palant, 2015/02/09 12:54:29]

Port number before slash please.

[Sebastian Noack, 2015/02/11 10:55:51]

Done.

-    equal(getDecodedHostname(new URL("http://user:password@example.com/")), "example.com", "with auth credentials");
-    equal(getDecodedHostname(new URL("http://xn--f-1gaa.com/")), "f\u00f6\u00f6.com", "with punycode");
-    equal(getDecodedHostname(new URL("about:blank")), "", "about:blank");
-    equal(getDecodedHostname(new URL("data:text/plain,foo")), "", "data: URL");

[Wladimir Palant, 2015/02/09 12:54:29]

Could you add back tests that you removed? In particular, I would consider ftp
URL and IP addresses important.

[Sebastian Noack, 2015/02/11 10:55:51]

Not really removed them, since the function we test here is new. But I agree,
there are some cases that previously were tested for extractHostFromURL() that
makes sense to cover here as well.

+    function testURLHostname(url, expectedHostname, message)
+    {
+      equal(getDecodedHostname(new URL(url)), expectedHostname, message);
+    }
+
+    testURLHostname("http://example.com/foo", "example.com", "with path");
+    testURLHostname("http://example.com/?foo=bar", "example.com", "with query");
+    testURLHostname("http://example.com/#top", "example.com", "with hash");
+    testURLHostname("http://example.com:8080/", "example.com", "with port");
+    testURLHostname("http://user:password@example.com/", "example.com", "with auth credentials");
+    testURLHostname("http://xn--f-1gaa.com/", "f\u00f6\u00f6.com", "with punycode");
+    testURLHostname("about:blank", "", "about:blank");
+    testURLHostname("data:text/plain,foo", "", "data: URL");
+    testURLHostname("ftp://example.com/", "example.com", "ftp: URL");
+    testURLHostname("http://1.2.3.4:8000/", "1.2.3.4", "IPv4 address");
+    testURLHostname("http://[2001:db8:85a3::8a2e:370:7334]/", "[2001:db8:85a3::8a2e:370:7334]", "IPv6 address");
   });
 
   test("Extracting hostname from frame", function()
   {
-    function makeFrame(hierarchy)
+    function testFrameHostname(hierarchy, expectedHostname, message)
     {
       var frame = null;
 
       for (var i = 0; i < hierarchy.length; i++)
         frame = {parent: frame, url: new URL(hierarchy[i])};
 
-      return frame;
+      equal(extractHostFromFrame(frame), expectedHostname, message);
     }
 
-    equal(extractHostFromFrame(makeFrame(["http://example.com/"])), "example.com", "single frame");
-    equal(extractHostFromFrame(makeFrame(["http://example.com/", "http://example.org/"])), "example.org", "with parent frame");
-    equal(extractHostFromFrame(makeFrame(["http://example.com/", "data:text/plain,foo"])), "example.com", "data: URL, hostname in parent");
-    equal(extractHostFromFrame(makeFrame(["http://example.com/", "about:blank", "about:blank"])), "example.com", "about:blank, hostname in ancestor");
-    equal(extractHostFromFrame(makeFrame(["about:blank", "about:blank"])), "", "about:blank, no hostname");
-    equal(extractHostFromFrame(makeFrame(["http://xn--f-1gaa.com/"])), "f\u00f6\u00f6.com", "with punycode");
+    testFrameHostname(["http://example.com/"], "example.com", "single frame");
+    testFrameHostname(["http://example.com/", "http://example.org/"], "example.org", "with parent frame");
+    testFrameHostname(["http://example.com/", "data:text/plain,foo"], "example.com", "data: URL, hostname in parent");
+    testFrameHostname(["http://example.com/", "about:blank", "about:blank"], "example.com", "about:blank, hostname in ancestor");
+    testFrameHostname(["about:blank", "about:blank"], "", "about:blank, no hostname");
+    testFrameHostname(["http://xn--f-1gaa.com/"], "f\u00f6\u00f6.com", "with punycode");
   });
 
   test("Stringifying URLs", function()
   {
-    equal(stringifyURL(new URL("http://example.com/foo")),"http://example.com/foo", "includes path");
-    equal(stringifyURL(new URL("http://example.com/?foo=bar")),"http://example.com/?foo=bar", "includes query");
-    equal(stringifyURL(new URL("http://example.com:8080/")),"http://example.com:8080/", "includes port");
-    equal(stringifyURL(new URL("http://example.com/#top")),"http://example.com/", "stripped hash");
-    equal(stringifyURL(new URL("http://user:password@example.com/")),"http://example.com/", "stripped auth credentials");
-    equal(stringifyURL(new URL("http://xn--f-1gaa.com/")),"http://f\u00f6\u00f6.com/", "decoded punycode");
-    equal(stringifyURL(new URL("about:blank")), "about:blank", "about:blank");
-    equal(stringifyURL(new URL("data:text/plain,foo")), "data:text/plain,foo", "data: URL");
+    function testNormalizedURL(url, expectedURL, message)
+    {
+      equal(stringifyURL(new URL(url)), expectedURL, message);
+    }
+
+    function testPreservedURL(url, message)
+    {
+      testNormalizedURL(url, url, message);
+    }
+
+    testPreservedURL("http://example.com/foo", "includes path");
+    testPreservedURL("http://example.com/?foo=bar", "includes query");
+    testPreservedURL("http://example.com:8080/", "includes port");
+    testNormalizedURL("http://example.com/#top","http://example.com/", "stripped hash");
+    testNormalizedURL("http://user:password@example.com/","http://example.com/", "stripped auth credentials");
+    testNormalizedURL("http://xn--f-1gaa.com/","http://f\u00f6\u00f6.com/", "decoded punycode");
+    testPreservedURL("about:blank", "about:blank");
+    testPreservedURL("data:text/plain,foo", "data: URL");
   });
 
   test("Third-party checks", function()
   {
-    equal(isThirdParty(new URL("http://foo/"), "foo"), false, "same domain isn't third-party");
-    equal(isThirdParty(new URL("http://foo/"), "bar"), true, "different domain is third-party");
-    equal(isThirdParty(new URL("http://foo.com/"), "foo.com"), false, "same domain with TLD (.com) isn't third-party");
-    equal(isThirdParty(new URL("http://foo.com/"), "bar.com"), true, "same TLD (.com) but different domain is third-party");
-    equal(isThirdParty(new URL("http://foo.example.com/"), "bar.example.com"), false, "same basedomain (example.com) isn't third-party");

[Wladimir Palant, 2015/02/09 12:54:29]

Please add back http://foo.com/, www.foo.com scenario - domain + subdomain is
different from the case of two subdomains.

[Sebastian Noack, 2015/02/11 10:55:51]

Done.

-    equal(isThirdParty(new URL("http://foo.uk/"), "bar.uk"), true, "same TLD (.uk) but different domain is third-party");
-    equal(isThirdParty(new URL("http://foo.co.uk/"), "bar.co.uk"), true, "same TLD (.co.uk) but different domain is third-party");
-    equal(isThirdParty(new URL("http://foo.example.co.uk/"), "bar.example.co.uk"), false, "same basedomain (example.co.uk) isn't third-party");
-    equal(isThirdParty(new URL("http://1.2.3.4/"), "1.2.3.4"), false, "same IP address isn't third-party");
-    equal(isThirdParty(new URL("http://1.1.1.1/"), "2.1.1.1"), true, "different IP address is third-party");
-    equal(isThirdParty(new URL("http://xn--f-1gaa.com/"), "f\u00f6\u00f6.com"), false, "same IDN isn't third-party");
+    function hostnameToURL(hostname)
+    {
+      return new URL("http://" + hostname);
+    }
+
+    function testThirdParty(requestHost, documentHost, expected, message)
+    {
+      equal(
+        isThirdParty(
+          hostnameToURL(requestHost),
+
+          // Chrome's URL object normalizes IP addresses. So some test
+          // will fail if we don't normalize the document host as well.
+          getDecodedHostname(hostnameToURL(documentHost))
+        ),
+        expected,
+        message
+      );
+    }
+
+    testThirdParty("foo", "foo", false, "same domain isn't third-party");
+    testThirdParty("foo", "bar", true, "different domain is third-party");
+    testThirdParty("foo.com", "foo.com", false, "same domain with TLD (.com) isn't third-party");
+    testThirdParty("foo.com", "bar.com", true, "same TLD (.com) but different domain is third-party");
+    testThirdParty("foo.com", "www.foo.com", false, "same domain but differend subdomain isn't third-party");
+    testThirdParty("foo.example.com", "bar.example.com", false, "same basedomain (example.com) isn't third-party");
+    testThirdParty("foo.uk", "bar.uk", true, "same TLD (.uk) but different domain is third-party");
+    testThirdParty("foo.co.uk", "bar.co.uk", true, "same TLD (.co.uk) but different domain is third-party");
+    testThirdParty("foo.example.co.uk", "bar.example.co.uk", false, "same basedomain (example.co.uk) isn't third-party");
+    testThirdParty("1.2.3.4", "1.2.3.4", false, "same IPv4 address isn't third-party");
+    testThirdParty("1.1.1.1", "2.1.1.1", true, "different IPv4 address is third-party");
+    testThirdParty("0x01ff0101", "0x01ff0101", false, "same IPv4 hexadecimal address isn't third-party");
+    testThirdParty("0x01ff0101", "0x01ff0102", true, "different IPv4 hexadecimal address is third-party");
+    testThirdParty("1.0xff.3.4", "1.0xff.3.4", false, "same IPv4 address with hexadecimal octet isn't third-party");
+    testThirdParty("1.0xff.1.1", "2.0xff.1.1", true, "different IPv4 address with hexadecimal octet is third-party");
+    testThirdParty("0xff.example.com", "example.com", false, "domain starts like a hexadecimal IPv4 address but isn't one");
+    testThirdParty("[2001:db8:85a3::8a2e:370:7334]", "[2001:db8:85a3::8a2e:370:7334]", false, "same IPv6 address isn't third-party");
+    testThirdParty("[2001:db8:85a3::8a2e:370:7334]", "[5001:db8:85a3::8a2e:370:7334]", true, "different IPv6 address is third-party");
+    testThirdParty("[::ffff:192.0.2.128]", "[::ffff:192.0.2.128]", false, "same IPv4-mapped IPv6 address isn't third-party");
+    testThirdParty("[::ffff:192.0.2.128]", "[::ffff:192.1.2.128]", true, "different IPv4-mapped IPv6 address is third-party");
+    testThirdParty("xn--f-1gaa.com", "f\u00f6\u00f6.com", false, "same IDN isn't third-party");
+    testThirdParty("example.com..", "example.com....", false, "traling dots are ignored");
   });

[Wladimir Palant, 2015/02/09 12:54:29]

The original unit tests had a bunch more scenarios:

* IPv6 addresses (also dotted-decimal inside IPv6).
* Hexadecimal encoding in IPv4 addresses.
* "Invalid" IP addresses (e.g. 0xab.example.com).
* Trailing dot.

If you don't want to test base domain extraction directly, you need to cover
these cases here.

[Sebastian Noack, 2015/02/11 10:55:51]

Done.

 })();

[Wladimir Palant, 2015/02/09 12:54:29]

Nit: there is a reason why the original unit tests had lists with test data - to
avoid the boilerplace. E.g. equal(isThirdParty(new URL())) is all boilerplace
and obscures the parameters that are changing here. Up to you whether you want
to change that.

[Sebastian Noack, 2015/02/11 10:55:51]

The reason I didn't like that approach was that it made the test code rather
obscure IMO. Since it used a multidimensional array, it wasn't obvious which
item of the inner arrays were used how. Also the old tests didn't bother to use
meaningful messages documenting what we actually protecting against. I think a
better approach to eliminate duplication here would be to move the common
boilerplate into a function.