Issue 6622 - Implement $rewrite filter option

lib/requestBlocker.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-present eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 /** @module requestBlocker */
 
 "use strict";
 
 const {Filter, RegExpFilter, BlockingFilter} =
   require("../adblockpluscore/lib/filterClasses");
 const {Subscription} = require("../adblockpluscore/lib/subscriptionClasses");
 const {defaultMatcher} = require("../adblockpluscore/lib/matcher");
 const {FilterNotifier} = require("../adblockpluscore/lib/filterNotifier");
 const {Prefs} = require("./prefs");
 const {checkWhitelisted, getKey} = require("./whitelisting");
-const {stringifyURL, extractHostFromFrame, isThirdParty} = require("./url");
+const {extractHostFromFrame, isThirdParty} = require("./url");
 const {port} = require("./messaging");
-const devtools = require("./devtools");
+const {logRequest: hitLoggerLogRequest} = require("./hitLogger");
 
 const extensionProtocol = new URL(browser.extension.getURL("")).protocol;
 
 // Chrome can't distinguish between OBJECT_SUBREQUEST and OBJECT requests.
 if (!browser.webRequest.ResourceType ||
     !("OBJECT_SUBREQUEST" in browser.webRequest.ResourceType))
 {
   RegExpFilter.typeMap.OBJECT_SUBREQUEST = RegExpFilter.typeMap.OBJECT;
 }
 
@@ skipping 21 matching lines @@
   if (!(browser.webRequest.ResourceType))
     return;
 
   for (let type in browser.webRequest.ResourceType)
     yield resourceTypes.get(browser.webRequest.ResourceType[type]) || "OTHER";
 
   // WEBRTC gets addressed through a workaround, even if the webRequest API is
   // lacking support to block this kind of a request.
   yield "WEBRTC";
 
-  // These filter types aren't mapped to resource types.
+  // POPUP, CSP and ELEMHIDE filters aren't mapped to resource types.
   yield "POPUP";
   yield "ELEMHIDE";
   yield "CSP";
-  yield "REWRITE";
 }());
 
 function getDocumentInfo(page, frame, originUrl)
 {
   return [
     extractHostFromFrame(frame, originUrl),
     getKey(page, frame, originUrl),
     !!checkWhitelisted(page, frame, originUrl,
                        RegExpFilter.typeMap.GENERICBLOCK)
   ];
 }
 
 function matchRequest(url, type, docDomain, sitekey, specificOnly)
 {
-  let urlString = stringifyURL(url);
   let thirdParty = isThirdParty(url, docDomain);
-
-  return [
-    defaultMatcher.matchesAny(urlString, RegExpFilter.typeMap[type],
-                              docDomain, thirdParty, sitekey, specificOnly),
-    urlString,
-    thirdParty
-  ];
+  let filter = defaultMatcher.matchesAny(url.href, RegExpFilter.typeMap[type],
+                                         docDomain, thirdParty,
+                                         sitekey, specificOnly);
+  return [filter, thirdParty];
 }
 
 function getRelatedTabIds(details)
 {
   // This is the common case, the request is associated with a single tab.
   // If tabId is -1, its not (e.g. the request was sent by
   // a Service/Shared Worker) and we have to identify the related tabs.
   if (details.tabId != -1)
     return Promise.resolve([details.tabId]);
 
   let url;                    // Firefox provides "originUrl" indicating the
   if (details.originUrl)      // URL of the tab that caused this request.
     url = details.originUrl;  // In case of Service/Shared Worker, this is the
                               // URL of the tab that caused the worker to spawn.
 
-  else if (details.initiator)        // Chromium >=63 provides "intiator" which
-    url = details.initiator + "/*";  // is equivalent to "originUrl" on Firefox
+  else if (details.initiator && details.initiator != "null")
+    url = details.initiator + "/*";  // Chromium >=63 provides "intiator" which
+                                     // is equivalent to "originUrl" on Firefox
                                      // except that its not a full URL but just
                                      // an origin (proto + host).
   else
     return Promise.resolve([]);
 
   return browser.tabs.query({url}).then(tabs => tabs.map(tab => tab.id));
 }
 
-function logRequest(tabIds, url, type, docDomain, thirdParty,
-                    sitekey, specificOnly, filter)
+function logRequest(tabIds, request, filter)
 {
   if (filter)
     FilterNotifier.emit("filter.hitCount", filter, 0, 0, tabIds);
 
-  devtools.logRequest(
-    tabIds, url, type, docDomain,
-    thirdParty, sitekey,
-    specificOnly, filter
-  );
+  hitLoggerLogRequest(tabIds, request, filter);
 }
 
 browser.webRequest.onBeforeRequest.addListener(details =>
 {
   // Never block top-level documents.
   if (details.type == "main_frame")
     return;
 
   // Filter out requests from non web protocols. Ideally, we'd explicitly
   // specify the protocols we are interested in (i.e. http://, https://,
   // ws:// and wss://) with the url patterns, given below, when adding this
   // listener. But unfortunately, Chrome <=57 doesn't support the WebSocket
   // protocol and is causing an error if it is given.
   let url = new URL(details.url);
   if (url.protocol != "http:" && url.protocol != "https:" &&
       url.protocol != "ws:" && url.protocol != "wss:")
     return;
 
   // Firefox provides us with the full origin URL, while Chromium (>=63)
   // provides only the protocol + host of the (top-level) document which
   // the request originates from through the "initiator" property.
-  let originUrl = details.originUrl ? new URL(details.originUrl) :
-                  details.initiator ? new URL(details.initiator) : null;
+  let originUrl = null;
+  if (details.originUrl)
+    originUrl = new URL(details.originUrl);
+  else if (details.initiator && details.initiator != "null")
+    originUrl = new URL(details.initiator);
 
   // Ignore requests sent by extensions or by Firefox itself:
   // * Firefox intercepts requests sent by any extensions, indicated with
   //   an "originURL" starting with "moz-extension:".
   // * Chromium intercepts requests sent by this extension only, indicated
   //   on Chromium >=63 with an "initiator" starting with "chrome-extension:".
   // * On Firefox, requests that don't relate to any document or extension are
   //   indicated with an "originUrl" starting with "chrome:".
   if (originUrl && (originUrl.protocol == extensionProtocol ||
                     originUrl.protocol == "chrome:"))
     return;
 
   let page = new ext.Page({id: details.tabId});
   let frame = ext.getFrame(
     details.tabId,
     // We are looking for the frame that contains the element which
     // has triggered this request. For most requests (e.g. images) we
     // can just use the request's frame ID, but for subdocument requests
     // (e.g. iframes) we must instead use the request's parent frame ID.
     details.type == "sub_frame" ? details.parentFrameId : details.frameId
   );
 
   // On Chromium >= 63, if both the frame is unknown and we haven't get
-  // an "initator", this implies a request sent by the browser itself
-  // (on older versions of Chromium, due to the lack of "initator",
+  // an "initiator", this implies a request sent by the browser itself
+  // (on older versions of Chromium, due to the lack of "initiator",
   // this can also indicate a request sent by a Shared/Service Worker).
   if (!frame && !originUrl)
     return;
 
   if (checkWhitelisted(page, frame, originUrl))
     return;
 
   let type = resourceTypes.get(details.type) || "OTHER";
   let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame,
                                                            originUrl);
-  let [filter, urlString, thirdParty] = matchRequest(url, type, docDomain,
-                                                     sitekey, specificOnly);
+  let [filter, thirdParty] = matchRequest(url, type, docDomain,
+                                          sitekey, specificOnly);
 
   let result;
+  let rewrittenUrl;
 
   if (filter instanceof BlockingFilter)
   {
     if (filter.rewrite)
     {
-      let rewritten = filter.rewriteUrl(urlString);
-      if (!rewritten || rewritten == urlString)

[Manish Jethani, 2018/05/03 02:51:16]

This should probably be updated now, since rewriteUrl never returns a falsey
value. Actually we know at this point that filter.regexp matches, this means the
rewrite is happening even if the result is the same as the original URL. That's
OK, we can just remove this check here entirely.

[hub, 2018/05/03 18:43:48]

Done.

-        // we couldn't do the rewrite, so just let it through.
-        return;
-
-      type = "REWRITE";
-      result = {redirectUrl: rewritten};
+      rewrittenUrl = filter.rewriteUrl(details.url);
+      // If no rewrite happened (error, different origin), we'll
+      // return undefined in order to avoid an "infinite" loop.
+      if (rewrittenUrl != details.url)
+        result = {redirectUrl: rewrittenUrl};
     }
     else
       result = {cancel: true};
   }
 
   getRelatedTabIds(details).then(tabIds =>
   {
-    logRequest(tabIds, urlString, type, docDomain,
-               thirdParty, sitekey, specificOnly, filter);
+    logRequest(
+      tabIds,
+      {
+        url: details.url, type, docDomain, thirdParty,
+        sitekey, specificOnly, rewrittenUrl
+      },
+      filter
+    );
   });
 
   return result;
 }, {urls: ["<all_urls>"]}, ["blocking"]);
 
 port.on("filters.collapse", (message, sender) =>
 {
   let {page, frame} = sender;
 
   if (checkWhitelisted(page, frame))
@@ skipping 19 matching lines @@
   return blocked && Prefs.hidePlaceholders;
 });
 
 port.on("request.blockedByRTCWrapper", (msg, sender) =>
 {
   let {page, frame} = sender;
 
   if (checkWhitelisted(page, frame))
     return false;
 
+  let {url} = msg;
   let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame);
-  let [filter, url, thirdParty] = matchRequest(new URL(msg.url),
-                                               "WEBRTC", docDomain,
-                                               sitekey, specificOnly);
-
-  logRequest([sender.page.id], url, "WEBRTC", docDomain,
-             thirdParty, sitekey, specificOnly, filter);
+  let [filter, thirdParty] = matchRequest(new URL(url), "WEBRTC", docDomain,
+                                          sitekey, specificOnly);
+  logRequest(
+    [sender.page.id],
+    {url, type: "WEBRTC", docDomain, thirdParty, sitekey, specificOnly},
+    filter
+  );
 
   return filter instanceof BlockingFilter;
 });
 
 let ignoreFilterNotifications = false;
 let handlerBehaviorChangedQuota =
   browser.webRequest.MAX_HANDLER_BEHAVIOR_CHANGED_CALLS_PER_10_MINUTES;
 
 function propagateHandlerBehaviorChange()
 {
@@ skipping 51 matching lines @@
 }
 
 FilterNotifier.on("subscription.added", onFilterChange);
 FilterNotifier.on("subscription.removed", onFilterChange);
 FilterNotifier.on("subscription.updated", onFilterChange);
 FilterNotifier.on("subscription.disabled", arg => onFilterChange(arg, true));
 FilterNotifier.on("filter.added", onFilterChange);
 FilterNotifier.on("filter.removed", onFilterChange);
 FilterNotifier.on("filter.disabled", arg => onFilterChange(arg, true));
 FilterNotifier.on("load", onFilterChange);