Issue 6402 - Split filter hit / request logging out into own API

lib/requestBlocker.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-present eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 /** @module requestBlocker */
 
 "use strict";
 
 const {Filter, RegExpFilter, BlockingFilter} =
   require("../adblockpluscore/lib/filterClasses");
 const {Subscription} = require("../adblockpluscore/lib/subscriptionClasses");
 const {defaultMatcher} = require("../adblockpluscore/lib/matcher");
 const {FilterNotifier} = require("../adblockpluscore/lib/filterNotifier");
 const {Prefs} = require("./prefs");
 const {checkWhitelisted, getKey} = require("./whitelisting");
-const {stringifyURL, extractHostFromFrame, isThirdParty} = require("./url");
+const {extractHostFromFrame, isThirdParty} = require("./url");
 const {port} = require("./messaging");
 const {logRequest: hitLoggerLogRequest} = require("./hitLogger");
 
 const extensionProtocol = new URL(browser.extension.getURL("")).protocol;
 
 // Chrome can't distinguish between OBJECT_SUBREQUEST and OBJECT requests.
 if (!browser.webRequest.ResourceType ||
     !("OBJECT_SUBREQUEST" in browser.webRequest.ResourceType))
 {
   RegExpFilter.typeMap.OBJECT_SUBREQUEST = RegExpFilter.typeMap.OBJECT;
@@ skipping 28 matching lines @@
 
   // WEBRTC gets addressed through a workaround, even if the webRequest API is
   // lacking support to block this kind of a request.
   yield "WEBRTC";
 
   // POPUP, CSP and ELEMHIDE filters aren't mapped to resource types.
   yield "POPUP";
   yield "ELEMHIDE";
   yield "CSP";
 }());
+
+function getDocumentInfo(page, frame, originUrl)
+{
+  return [
+    extractHostFromFrame(frame, originUrl),
+    getKey(page, frame, originUrl),
+    !!checkWhitelisted(page, frame, originUrl,
+                       RegExpFilter.typeMap.GENERICBLOCK)
+  ];
+}
+
+function matchRequest(url, type, docDomain, sitekey, specificOnly)
+{
+  let thirdParty = isThirdParty(url, docDomain);
+  let filter = defaultMatcher.matchesAny(url.href, RegExpFilter.typeMap[type],
+                                         docDomain, thirdParty,
+                                         sitekey, specificOnly);
+  return [filter, thirdParty];
+}
 
 function getRelatedTabIds(details)
 {
   // This is the common case, the request is associated with a single tab.
   // If tabId is -1, its not (e.g. the request was sent by
   // a Service/Shared Worker) and we have to identify the related tabs.
   if (details.tabId != -1)
     return Promise.resolve([details.tabId]);
 
   let url;                    // Firefox provides "originUrl" indicating the
   if (details.originUrl)      // URL of the tab that caused this request.
     url = details.originUrl;  // In case of Service/Shared Worker, this is the
                               // URL of the tab that caused the worker to spawn.
 
   else if (details.initiator)        // Chromium >=63 provides "intiator" which
     url = details.initiator + "/*";  // is equivalent to "originUrl" on Firefox
                                      // except that its not a full URL but just
                                      // an origin (proto + host).
   else
     return Promise.resolve([]);
 
   return browser.tabs.query({url}).then(tabs => tabs.map(tab => tab.id));
 }
 
-function logRequest(details, url, type, docDomain, thirdParty,
-                    sitekey, specificOnly, filter)
-{
-  getRelatedTabIds(details).then(tabIds =>
-  {
-    if (filter)
-      FilterNotifier.emit("filter.hitCount", filter, 0, 0, tabIds);
-
-    hitLoggerLogRequest(
-      tabIds, url, type, docDomain,
-      thirdParty, sitekey,
-      specificOnly, filter
-    );
-  });
+function logRequest(tabIds, request, filter)
+{
+  if (filter)
+    FilterNotifier.emit("filter.hitCount", filter, 0, 0, tabIds);
+
+  hitLoggerLogRequest(tabIds, request, filter);
 }
 
 browser.webRequest.onBeforeRequest.addListener(details =>
 {
   // Never block top-level documents.
   if (details.type == "main_frame")
     return;
 
   // Filter out requests from non web protocols. Ideally, we'd explicitly
   // specify the protocols we are interested in (i.e. http://, https://,
   // ws:// and wss://) with the url patterns, given below, when adding this
   // listener. But unfortunately, Chrome <=57 doesn't support the WebSocket
   // protocol and is causing an error if it is given.
   let url = new URL(details.url);
   if (url.protocol != "http:" && url.protocol != "https:" &&
       url.protocol != "ws:" && url.protocol != "wss:")
     return;
 
   // Firefox provides us with the full origin URL, while Chromium (>=63)
   // provides only the protocol + host of the (top-level) document which
   // the request originates from through the "initiator" property.
   let originUrl = details.originUrl ? new URL(details.originUrl) :
                   details.initiator ? new URL(details.initiator) : null;
 
-  // Ignore requests sent by extensions or by the browser itself:
+  // Ignore requests sent by extensions or by Firefox itself:
   // * Firefox intercepts requests sent by any extensions, indicated with
   //   an "originURL" starting with "moz-extension:".
   // * Chromium intercepts requests sent by this extension only, indicated
   //   on Chromium >=63 with an "initiator" starting with "chrome-extension:".
   // * On Firefox, requests that don't relate to any document or extension are
   //   indicated with an "originUrl" starting with "chrome:".
-  // * On Chromium >=63, requests that don't relate to any document or extension
-  //   have no "initiator". But since on older Chromium versions, no request
-  //   has an "initiator", we have to check for the tabId as well.
-  if (originUrl)
+  if (originUrl && (originUrl.protocol == extensionProtocol ||
+                    originUrl.protocol == "chrome:"))
+    return;
+
+  let page = new ext.Page({id: details.tabId});
+  let frame = ext.getFrame(
+    details.tabId,
+    // We are looking for the frame that contains the element which
+    // has triggered this request. For most requests (e.g. images) we
+    // can just use the request's frame ID, but for subdocument requests
+    // (e.g. iframes) we must instead use the request's parent frame ID.
+    details.type == "sub_frame" ? details.parentFrameId : details.frameId
+  );
+
+  // On Chromium >= 63, if both the frame is unknown and we haven't get
+  // an "initator", this implies a request sent by the browser itself
+  // (on older versions of Chromium, due to the lack of "initator",
+  // this can also indicate a request sent by a Shared/Service Worker).
+  if (!frame && !originUrl)
+    return;
+
+  if (checkWhitelisted(page, frame, originUrl))
+    return;
+
+  let type = resourceTypes.get(details.type) || "OTHER";
+  let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame,
+                                                           originUrl);
+  let [filter, thirdParty] = matchRequest(url, type, docDomain,
+                                          sitekey, specificOnly);
+
+  getRelatedTabIds(details).then(tabIds =>
   {
-    if (originUrl.protocol == extensionProtocol ||
-        originUrl.protocol == "chrome:")
-      return;
-  }
-  else if (details.tabId == -1)
-    return;
-
-  let page = null;
-  let frame = null;
-  if (details.tabId != -1)
-  {
-    page = new ext.Page({id: details.tabId});
-    frame = ext.getFrame(
-      details.tabId,
-      // We are looking for the frame that contains the element which
-      // has triggered this request. For most requests (e.g. images) we
-      // can just use the request's frame ID, but for subdocument requests
-      // (e.g. iframes) we must instead use the request's parent frame ID.
-      details.type == "sub_frame" ? details.parentFrameId : details.frameId
+    logRequest(
+      tabIds,
+      {url: details.url, type, docDomain, thirdParty, sitekey, specificOnly},
+      filter
     );
-  }
-
-  if (checkWhitelisted(page, frame, originUrl))
-    return;
-
-  let urlString = stringifyURL(url);
-  let type = resourceTypes.get(details.type) || "OTHER";
-  let docDomain = extractHostFromFrame(frame, originUrl);
-  let thirdParty = isThirdParty(url, docDomain);
-  let sitekey = getKey(page, frame, originUrl);
-  let specificOnly = !!checkWhitelisted(page, frame, originUrl,
-                                        RegExpFilter.typeMap.GENERICBLOCK);
-
-  let filter = defaultMatcher.matchesAny(
-    urlString, RegExpFilter.typeMap[type],
-    docDomain, thirdParty, sitekey, specificOnly
-  );
-
-  logRequest(details, urlString, type, docDomain,
-             thirdParty, sitekey, specificOnly, filter);
+  });
 
   if (filter instanceof BlockingFilter)
     return {cancel: true};
 }, {urls: ["<all_urls>"]}, ["blocking"]);
 
 port.on("filters.collapse", (message, sender) =>
 {
-  if (checkWhitelisted(sender.page, sender.frame))
+  let {page, frame} = sender;
+
+  if (checkWhitelisted(page, frame))
     return false;
 
-  let typeMask = RegExpFilter.typeMap[message.mediatype];
-  let documentHost = extractHostFromFrame(sender.frame);
-  let sitekey = getKey(sender.page, sender.frame);
   let blocked = false;
-
-  let specificOnly = checkWhitelisted(
-    sender.page, sender.frame, null,
-    RegExpFilter.typeMap.GENERICBLOCK
-  );
+  let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame);
 
   for (let url of message.urls)
   {
-    let urlObj = new URL(url, message.baseURL);
-    let filter = defaultMatcher.matchesAny(
-      stringifyURL(urlObj),
-      typeMask, documentHost,
-      isThirdParty(urlObj, documentHost),
-      sitekey, specificOnly
-    );
+    let [filter] = matchRequest(new URL(url, message.baseURL),
+                                message.mediatype, docDomain,
+                                sitekey, specificOnly);
 
     if (filter instanceof BlockingFilter)
     {
       if (filter.collapse != null)
         return filter.collapse;
       blocked = true;
     }
   }
 
   return blocked && Prefs.hidePlaceholders;
 });
 
+port.on("request.blockedByRTCWrapper", (msg, sender) =>
+{
+  let {page, frame} = sender;
+
+  if (checkWhitelisted(page, frame))
+    return false;
+
+  let {url} = msg;
+  let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame);
+  let [filter, thirdParty] = matchRequest(new URL(url), "WEBRTC", docDomain,
+                                          sitekey, specificOnly);
+  logRequest(
+    [sender.page.id],
+    {url, type: "WEBRTC", docDomain, thirdParty, sitekey, specificOnly},
+    filter
+  );
+
+  return filter instanceof BlockingFilter;
+});
+
 let ignoreFilterNotifications = false;
+let handlerBehaviorChangedQuota =
+  browser.webRequest.MAX_HANDLER_BEHAVIOR_CHANGED_CALLS_PER_10_MINUTES;
+
+function propagateHandlerBehaviorChange()
+{
+  // Make sure to not call handlerBehaviorChanged() more often than allowed
+  // by browser.webRequest.MAX_HANDLER_BEHAVIOR_CHANGED_CALLS_PER_10_MINUTES.
+  // Otherwise Chrome notifies the user that this extension is causing issues.
+  if (handlerBehaviorChangedQuota > 0)
+  {
+    browser.webNavigation.onBeforeNavigate.removeListener(
+      propagateHandlerBehaviorChange
+    );
+    browser.webRequest.handlerBehaviorChanged();
+    handlerBehaviorChangedQuota--;
+    setTimeout(() => { handlerBehaviorChangedQuota++; }, 600000);
+  }
+}
 
 function onFilterChange(arg, isDisabledAction)
 {
   // Avoid triggering filters.behaviorChanged multiple times
   // when multiple filter hanges happen at the same time.
   if (ignoreFilterNotifications)
     return;
 
   // Ignore disabled subscriptions and filters, unless they just got
   // disabled, otherwise they have no effect on the handler behavior.
   if (arg && arg.disabled && !isDisabledAction)
     return;
 
   // Ignore empty subscriptions. This includes subscriptions
   // that have just been added, but not downloaded yet.
   if (arg instanceof Subscription && arg.filters.length == 0)
     return;
 
   // Ignore all types of filters but request filters,
   // only these have an effect on the handler behavior.
   if (arg instanceof Filter && !(arg instanceof RegExpFilter))
     return;
 
   ignoreFilterNotifications = true;
   setTimeout(() =>
   {
+    // Defer handlerBehaviorChanged() until navigation occurs.
+    // There wouldn't be any visible effect when calling it earlier,
+    // but it's an expensive operation and that way we avoid to call
+    // it multiple times, if multiple filters are added/removed.
+    if (!browser.webNavigation.onBeforeNavigate
+                              .hasListener(propagateHandlerBehaviorChange))
+      browser.webNavigation.onBeforeNavigate
+                           .addListener(propagateHandlerBehaviorChange);
+
     ignoreFilterNotifications = false;
-    ext.webRequest.handlerBehaviorChanged();
     FilterNotifier.emit("filter.behaviorChanged");
   });
 }
 
 FilterNotifier.on("subscription.added", onFilterChange);
 FilterNotifier.on("subscription.removed", onFilterChange);
 FilterNotifier.on("subscription.updated", onFilterChange);
 FilterNotifier.on("subscription.disabled", arg => onFilterChange(arg, true));
 FilterNotifier.on("filter.added", onFilterChange);
 FilterNotifier.on("filter.removed", onFilterChange);
 FilterNotifier.on("filter.disabled", arg => onFilterChange(arg, true));
 FilterNotifier.on("load", onFilterChange);
-
-port.on("request.blockedByRTCWrapper", (msg, sender) =>
-{
-  return ext.webRequest.onBeforeRequest._dispatch(
-     new URL(msg.url),
-     "webrtc",
-     sender.page,
-     sender.frame
-  ).includes(false);
-});