Issue 6622 - Implement $rewrite filter option

lib/requestBlocker.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-present eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 /** @module requestBlocker */
 
 "use strict";
 
 const {Filter, RegExpFilter, BlockingFilter} =
   require("../adblockpluscore/lib/filterClasses");
 const {Subscription} = require("../adblockpluscore/lib/subscriptionClasses");
 const {defaultMatcher} = require("../adblockpluscore/lib/matcher");
 const {FilterNotifier} = require("../adblockpluscore/lib/filterNotifier");
 const {Prefs} = require("./prefs");
 const {checkWhitelisted, getKey} = require("./whitelisting");
-const {stringifyURL, extractHostFromFrame, isThirdParty} = require("./url");
+const {extractHostFromFrame, isThirdParty} = require("./url");
 const {port} = require("./messaging");
-const devtools = require("./devtools");
+const {logRequest: hitLoggerLogRequest} = require("./hitLogger");
 
 const extensionProtocol = new URL(browser.extension.getURL("")).protocol;
 
 // Chrome can't distinguish between OBJECT_SUBREQUEST and OBJECT requests.
 if (!browser.webRequest.ResourceType ||
     !("OBJECT_SUBREQUEST" in browser.webRequest.ResourceType))
 {
   RegExpFilter.typeMap.OBJECT_SUBREQUEST = RegExpFilter.typeMap.OBJECT;
 }
 
@@ skipping 21 matching lines @@
   if (!(browser.webRequest.ResourceType))
     return;
 
   for (let type in browser.webRequest.ResourceType)
     yield resourceTypes.get(browser.webRequest.ResourceType[type]) || "OTHER";
 
   // WEBRTC gets addressed through a workaround, even if the webRequest API is
   // lacking support to block this kind of a request.
   yield "WEBRTC";
 
-  // These filter types aren't mapped to resource types.
+  // POPUP, CSP and ELEMHIDE filters aren't mapped to resource types.
   yield "POPUP";
   yield "ELEMHIDE";
   yield "CSP";
-  yield "REWRITE";
 }());
 
 function getDocumentInfo(page, frame, originUrl)
 {
   return [
     extractHostFromFrame(frame, originUrl),
     getKey(page, frame, originUrl),
     !!checkWhitelisted(page, frame, originUrl,
                        RegExpFilter.typeMap.GENERICBLOCK)
   ];
 }
 
 function matchRequest(url, type, docDomain, sitekey, specificOnly)
 {
-  let urlString = stringifyURL(url);
   let thirdParty = isThirdParty(url, docDomain);
-
-  return [
-    defaultMatcher.matchesAny(urlString, RegExpFilter.typeMap[type],
-                              docDomain, thirdParty, sitekey, specificOnly),
-    urlString,
-    thirdParty
-  ];
+  let filter = defaultMatcher.matchesAny(url.href, RegExpFilter.typeMap[type],
+                                         docDomain, thirdParty,
+                                         sitekey, specificOnly);
+  return [filter, thirdParty];
 }
 
 function getRelatedTabIds(details)
 {
   // This is the common case, the request is associated with a single tab.
   // If tabId is -1, its not (e.g. the request was sent by
   // a Service/Shared Worker) and we have to identify the related tabs.
   if (details.tabId != -1)
     return Promise.resolve([details.tabId]);
 
   let url;                    // Firefox provides "originUrl" indicating the
   if (details.originUrl)      // URL of the tab that caused this request.
     url = details.originUrl;  // In case of Service/Shared Worker, this is the
                               // URL of the tab that caused the worker to spawn.
 
-  else if (details.initiator)        // Chromium >=63 provides "intiator" which
-    url = details.initiator + "/*";  // is equivalent to "originUrl" on Firefox
+  else if (details.initiator && details.initiator != "null")
+    url = details.initiator + "/*";  // Chromium >=63 provides "intiator" which
+                                     // is equivalent to "originUrl" on Firefox
                                      // except that its not a full URL but just
                                      // an origin (proto + host).
   else
     return Promise.resolve([]);
 
   return browser.tabs.query({url}).then(tabs => tabs.map(tab => tab.id));
 }
 
-function logRequest(tabIds, url, type, docDomain, thirdParty,
-                    sitekey, specificOnly, filter)
+function logRequest(tabIds, request, filter)
 {
   if (filter)
     FilterNotifier.emit("filter.hitCount", filter, 0, 0, tabIds);
 
-  devtools.logRequest(
-    tabIds, url, type, docDomain,
-    thirdParty, sitekey,
-    specificOnly, filter
-  );
+  hitLoggerLogRequest(tabIds, request, filter);
 }
 
 browser.webRequest.onBeforeRequest.addListener(details =>
 {
   // Never block top-level documents.
   if (details.type == "main_frame")
     return;
 
   // Filter out requests from non web protocols. Ideally, we'd explicitly
   // specify the protocols we are interested in (i.e. http://, https://,
   // ws:// and wss://) with the url patterns, given below, when adding this
   // listener. But unfortunately, Chrome <=57 doesn't support the WebSocket
   // protocol and is causing an error if it is given.
   let url = new URL(details.url);
   if (url.protocol != "http:" && url.protocol != "https:" &&
       url.protocol != "ws:" && url.protocol != "wss:")
     return;
 
   // Firefox provides us with the full origin URL, while Chromium (>=63)
   // provides only the protocol + host of the (top-level) document which
   // the request originates from through the "initiator" property.
-  let originUrl = details.originUrl ? new URL(details.originUrl) :
-                  details.initiator ? new URL(details.initiator) : null;
+  let originUrl = null;
+  if (details.originUrl)
+    originUrl = new URL(details.originUrl);
+  else if (details.initiator && details.initiator != "null")
+    originUrl = new URL(details.initiator);
 
   // Ignore requests sent by extensions or by Firefox itself:
   // * Firefox intercepts requests sent by any extensions, indicated with
   //   an "originURL" starting with "moz-extension:".
   // * Chromium intercepts requests sent by this extension only, indicated
   //   on Chromium >=63 with an "initiator" starting with "chrome-extension:".
   // * On Firefox, requests that don't relate to any document or extension are
   //   indicated with an "originUrl" starting with "chrome:".
   if (originUrl && (originUrl.protocol == extensionProtocol ||
                     originUrl.protocol == "chrome:"))
     return;
 
   let page = new ext.Page({id: details.tabId});
   let frame = ext.getFrame(
     details.tabId,
     // We are looking for the frame that contains the element which
     // has triggered this request. For most requests (e.g. images) we
     // can just use the request's frame ID, but for subdocument requests
     // (e.g. iframes) we must instead use the request's parent frame ID.
     details.type == "sub_frame" ? details.parentFrameId : details.frameId
   );
 
   // On Chromium >= 63, if both the frame is unknown and we haven't get
-  // an "initator", this implies a request sent by the browser itself
-  // (on older versions of Chromium, due to the lack of "initator",
+  // an "initiator", this implies a request sent by the browser itself
+  // (on older versions of Chromium, due to the lack of "initiator",
   // this can also indicate a request sent by a Shared/Service Worker).
   if (!frame && !originUrl)
     return;
 
   if (checkWhitelisted(page, frame, originUrl))
     return;
 
   let type = resourceTypes.get(details.type) || "OTHER";
   let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame,
                                                            originUrl);
-  let [filter, urlString, thirdParty] = matchRequest(url, type, docDomain,
-                                                     sitekey, specificOnly);
+  let [filter, thirdParty] = matchRequest(url, type, docDomain,
+                                          sitekey, specificOnly);
 
   let result;
+  let rewrittenUrl;
 
   if (filter instanceof BlockingFilter)
   {
     if (filter.rewrite)
     {
-      let rewritten = filter.rewriteUrl(urlString);
-      if (rewritten == urlString)

[Manish Jethani, 2018/05/04 15:37:47]

Do we still need this check? If the filter matched there was a rewrite, even if
the result was the same.

[hub, 2018/05/04 18:27:58]

definitely. a redirectUrl will cause onBeforeRequest to be called again - which
will likely be an infinite loop until the browser cancel it. By just doing a
return we say "let's go". 

-        // we couldn't do the rewrite, so just let it through.
-        return;
-
-      type = "REWRITE";

[Manish Jethani, 2018/05/04 15:37:47]

Also I'm wondering if it makes sense to set the type here. The original type may
be "SCRIPT" and now we're overwriting it with "REWRITE", but we don't provide
information on what it was rewritten to. Technically we did block the original
script, we just redirected it to a new resource. I'm wondering if I would prefer
to see "SCRIPT" rather than "REWRITE" in the DevTools panel in this case.

[Sebastian Noack, 2018/05/04 15:46:49]

I already thought the same, and tend to agree to not indicate requests matching
a $rewrite filter with a type of REWRITE. This would be a leaky abstraction. But
then we need a different way to indicate $rewrite filters in the devtools panel.

[hub, 2018/05/04 18:27:58]

I have a solution to indicate the rewrite in the devtools panel. It requires a
change in adblockplusui (for which I have a patch) to actually display it.

But then I wanted to be able to filter REWRITE explicitly. Can't have both the
type and REWRITE in the filter though. Not sure there is an easy way without an
overhaul of the logging.

If you think we don't need to filter REWRITE, then I'm fine. It will simplify
this part.

-      result = {redirectUrl: rewritten};
+      rewrittenUrl = filter.rewriteUrl(details.url);
+      // If no rewrite happened (error, different origin), we'll
+      // return undefined in order to avoid an "infinite" loop.
+      if (rewrittenUrl != details.url)
+        result = {redirectUrl: rewrittenUrl};
     }
     else
       result = {cancel: true};
   }
 
   getRelatedTabIds(details).then(tabIds =>
   {
-    logRequest(tabIds, urlString, type, docDomain,
-               thirdParty, sitekey, specificOnly, filter);
+    logRequest(
+      tabIds,
+      {
+        url: details.url, type, docDomain, thirdParty,
+        sitekey, specificOnly, rewrittenUrl
+      },
+      filter
+    );
   });
 
   return result;
 }, {urls: ["<all_urls>"]}, ["blocking"]);
 
 port.on("filters.collapse", (message, sender) =>
 {
   let {page, frame} = sender;
 
   if (checkWhitelisted(page, frame))
@@ skipping 19 matching lines @@
   return blocked && Prefs.hidePlaceholders;
 });
 
 port.on("request.blockedByRTCWrapper", (msg, sender) =>
 {
   let {page, frame} = sender;
 
   if (checkWhitelisted(page, frame))
     return false;
 
+  let {url} = msg;
   let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame);
-  let [filter, url, thirdParty] = matchRequest(new URL(msg.url),
-                                               "WEBRTC", docDomain,
-                                               sitekey, specificOnly);
-
-  logRequest([sender.page.id], url, "WEBRTC", docDomain,
-             thirdParty, sitekey, specificOnly, filter);
+  let [filter, thirdParty] = matchRequest(new URL(url), "WEBRTC", docDomain,
+                                          sitekey, specificOnly);
+  logRequest(
+    [sender.page.id],
+    {url, type: "WEBRTC", docDomain, thirdParty, sitekey, specificOnly},
+    filter
+  );
 
   return filter instanceof BlockingFilter;
 });
 
 let ignoreFilterNotifications = false;
 let handlerBehaviorChangedQuota =
   browser.webRequest.MAX_HANDLER_BEHAVIOR_CHANGED_CALLS_PER_10_MINUTES;
 
 function propagateHandlerBehaviorChange()
 {
@@ skipping 51 matching lines @@
 }
 
 FilterNotifier.on("subscription.added", onFilterChange);
 FilterNotifier.on("subscription.removed", onFilterChange);
 FilterNotifier.on("subscription.updated", onFilterChange);
 FilterNotifier.on("subscription.disabled", arg => onFilterChange(arg, true));
 FilterNotifier.on("filter.added", onFilterChange);
 FilterNotifier.on("filter.removed", onFilterChange);
 FilterNotifier.on("filter.disabled", arg => onFilterChange(arg, true));
 FilterNotifier.on("load", onFilterChange);