Issue 1801 - Use URL objects to process URLs in the background page

qunit/tests/url.js

Index: qunit/tests/url.js
===================================================================
new file mode 100644
--- /dev/null
+++ b/qunit/tests/url.js
@@ -0,0 +1,87 @@
+/*
+ * This file is part of Adblock Plus <https://adblockplus.org/>,
+ * Copyright (C) 2006-2015 Eyeo GmbH
+ *
+ * Adblock Plus is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * Adblock Plus is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+(function()
+{
+  var url = require("url");
+  var getDecodedHostname = url.getDecodedHostname;
+  var extractHostFromFrame = url.extractHostFromFrame;
+  var stringifyURL = url.stringifyURL;
+  var isThirdParty = url.isThirdParty;
+
+  module("URL/host tools");
+
+  test("Extracting hostname from URL", function()
+  {
+    equal(getDecodedHostname(new URL("http://example.com/foo")), "example.com", "with path");
+    equal(getDecodedHostname(new URL("http://example.com/?foo=bar")), "example.com", "with query");
+    equal(getDecodedHostname(new URL("http://example.com/#top")), "example.com", "with hash");
+    equal(getDecodedHostname(new URL("http://example.com/:8080")), "example.com", "with port");

[Wladimir Palant, 2015/02/09 12:54:29]

Port number before slash please.

[Sebastian Noack, 2015/02/11 10:55:51]

Done.

+    equal(getDecodedHostname(new URL("http://user:password@example.com/")), "example.com", "with auth credentials");
+    equal(getDecodedHostname(new URL("http://xn--f-1gaa.com/")), "f\u00f6\u00f6.com", "with punycode");
+    equal(getDecodedHostname(new URL("about:blank")), "", "about:blank");
+    equal(getDecodedHostname(new URL("data:text/plain,foo")), "", "data: URL");

[Wladimir Palant, 2015/02/09 12:54:29]

Could you add back tests that you removed? In particular, I would consider ftp
URL and IP addresses important.

[Sebastian Noack, 2015/02/11 10:55:51]

Not really removed them, since the function we test here is new. But I agree,
there are some cases that previously were tested for extractHostFromURL() that
makes sense to cover here as well.

+  });
+
+  test("Extracting hostname from frame", function()
+  {
+    function makeFrame(hierarchy)
+    {
+      var frame = null;
+
+      for (var i = 0; i < hierarchy.length; i++)
+        frame = {parent: frame, url: new URL(hierarchy[i])};
+
+      return frame;
+    }
+
+    equal(extractHostFromFrame(makeFrame(["http://example.com/"])), "example.com", "single frame");
+    equal(extractHostFromFrame(makeFrame(["http://example.com/", "http://example.org/"])), "example.org", "with parent frame");
+    equal(extractHostFromFrame(makeFrame(["http://example.com/", "data:text/plain,foo"])), "example.com", "data: URL, hostname in parent");
+    equal(extractHostFromFrame(makeFrame(["http://example.com/", "about:blank", "about:blank"])), "example.com", "about:blank, hostname in ancestor");
+    equal(extractHostFromFrame(makeFrame(["about:blank", "about:blank"])), "", "about:blank, no hostname");
+    equal(extractHostFromFrame(makeFrame(["http://xn--f-1gaa.com/"])), "f\u00f6\u00f6.com", "with punycode");
+  });
+
+  test("Stringifying URLs", function()
+  {
+    equal(stringifyURL(new URL("http://example.com/foo")),"http://example.com/foo", "includes path");
+    equal(stringifyURL(new URL("http://example.com/?foo=bar")),"http://example.com/?foo=bar", "includes query");
+    equal(stringifyURL(new URL("http://example.com:8080/")),"http://example.com:8080/", "includes port");
+    equal(stringifyURL(new URL("http://example.com/#top")),"http://example.com/", "stripped hash");
+    equal(stringifyURL(new URL("http://user:password@example.com/")),"http://example.com/", "stripped auth credentials");
+    equal(stringifyURL(new URL("http://xn--f-1gaa.com/")),"http://f\u00f6\u00f6.com/", "decoded punycode");
+    equal(stringifyURL(new URL("about:blank")), "about:blank", "about:blank");
+    equal(stringifyURL(new URL("data:text/plain,foo")), "data:text/plain,foo", "data: URL");
+  });
+
+  test("Third-party checks", function()
+  {
+    equal(isThirdParty(new URL("http://foo/"), "foo"), false, "same domain isn't third-party");
+    equal(isThirdParty(new URL("http://foo/"), "bar"), true, "different domain is third-party");
+    equal(isThirdParty(new URL("http://foo.com/"), "foo.com"), false, "same domain with TLD (.com) isn't third-party");
+    equal(isThirdParty(new URL("http://foo.com/"), "bar.com"), true, "same TLD (.com) but different domain is third-party");
+    equal(isThirdParty(new URL("http://foo.example.com/"), "bar.example.com"), false, "same basedomain (example.com) isn't third-party");

[Wladimir Palant, 2015/02/09 12:54:29]

Please add back http://foo.com/, www.foo.com scenario - domain + subdomain is
different from the case of two subdomains.

[Sebastian Noack, 2015/02/11 10:55:51]

Done.

+    equal(isThirdParty(new URL("http://foo.uk/"), "bar.uk"), true, "same TLD (.uk) but different domain is third-party");
+    equal(isThirdParty(new URL("http://foo.co.uk/"), "bar.co.uk"), true, "same TLD (.co.uk) but different domain is third-party");
+    equal(isThirdParty(new URL("http://foo.example.co.uk/"), "bar.example.co.uk"), false, "same basedomain (example.co.uk) isn't third-party");
+    equal(isThirdParty(new URL("http://1.2.3.4/"), "1.2.3.4"), false, "same IP address isn't third-party");
+    equal(isThirdParty(new URL("http://1.1.1.1/"), "2.1.1.1"), true, "different IP address is third-party");
+    equal(isThirdParty(new URL("http://xn--f-1gaa.com/"), "f\u00f6\u00f6.com"), false, "same IDN isn't third-party");
+  });

[Wladimir Palant, 2015/02/09 12:54:29]

The original unit tests had a bunch more scenarios:

* IPv6 addresses (also dotted-decimal inside IPv6).
* Hexadecimal encoding in IPv4 addresses.
* "Invalid" IP addresses (e.g. 0xab.example.com).
* Trailing dot.

If you don't want to test base domain extraction directly, you need to cover
these cases here.

[Sebastian Noack, 2015/02/11 10:55:51]

Done.

+})();

[Wladimir Palant, 2015/02/09 12:54:29]

Nit: there is a reason why the original unit tests had lists with test data - to
avoid the boilerplace. E.g. equal(isThirdParty(new URL())) is all boilerplace
and obscures the parameters that are changing here. Up to you whether you want
to change that.

[Sebastian Noack, 2015/02/11 10:55:51]

The reason I didn't like that approach was that it made the test code rather
obscure IMO. Since it used a multidimensional array, it wasn't obvious which
item of the inner arrays were used how. Also the old tests didn't bother to use
meaningful messages documenting what we actually protecting against. I think a
better approach to eliminate duplication here would be to move the common
boilerplate into a function.