Issue 394 - hit statistics tool data collection

lib/contentPolicy.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
- * Copyright (C) 2006-2015 Eyeo GmbH
+ * Copyright (C) 2006-2016 Eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 /**
  * @fileOverview Content policy implementation, responsible for blocking things.
  */
 
 "use strict";
 
 let {XPCOMUtils} = Cu.import("resource://gre/modules/XPCOMUtils.jsm", {});
 let {Services} = Cu.import("resource://gre/modules/Services.jsm", {});
 
 let {Utils} = require("utils");
+let {port} = require("messaging");
 let {Prefs} = require("prefs");
 let {FilterStorage} = require("filterStorage");
 let {BlockingFilter, WhitelistFilter, RegExpFilter} = require("filterClasses");
 let {defaultMatcher} = require("matcher");
 let {ElemHide} = require("elemHide");
 
 /**
  * Public policy checking functions and auxiliary objects
  * @class
  */
 var Policy = exports.Policy =
 {
   /**
-   * Set of explicitly supported content types
-   * @type Set.<string>
-   */
-  contentTypes: new Set([
-    "OTHER", "SCRIPT", "IMAGE", "STYLESHEET", "OBJECT", "SUBDOCUMENT", "DOCUMENT",
-    "XMLHTTPREQUEST", "OBJECT_SUBREQUEST", "FONT", "MEDIA", "ELEMHIDE", "POPUP",
-    "GENERICHIDE", "GENERICBLOCK"
-  ]),
+   * Map of content types reported by Firefox to the respecitve content types
+   * used by Adblock Plus. Other content types are simply mapped to OTHER.
+   * @type Map.<string,string>
+   */
+  contentTypes: new Map(function* ()
+  {
+    // Treat navigator.sendBeacon() the same as <a ping>,
+    // it's essentially the same concept - merely generalized.
+    yield ["BEACON", "PING"];
+
+    // Treat <img srcset> and <picture> the same as other images.
+    yield ["IMAGESET", "IMAGE"];
+
+    // Treat fetch() the same as XMLHttpRequest,
+    // it's essentially the same - merely a more modern API.
+    yield ["FETCH", "XMLHTTPREQUEST"];
+
+    // Everything else is mapped to itself
+    for (let contentType of ["OTHER", "SCRIPT", "IMAGE", "STYLESHEET", "OBJECT",
+                             "SUBDOCUMENT", "DOCUMENT", "XMLHTTPREQUEST",
+                             "OBJECT_SUBREQUEST", "FONT", "MEDIA", "PING",
+                             "ELEMHIDE", "POPUP", "GENERICHIDE", "GENERICBLOCK"])
+      yield [contentType, contentType];
+  }()),
 
   /**
    * Set of content types that aren't associated with a visual document area
    * @type Set.<string>
    */
   nonVisualTypes: new Set([
     "SCRIPT", "STYLESHEET", "XMLHTTPREQUEST", "OBJECT_SUBREQUEST", "FONT",
-    "ELEMHIDE", "POPUP", "GENERICHIDE", "GENERICBLOCK"
+    "PING", "ELEMHIDE", "POPUP", "GENERICHIDE", "GENERICBLOCK"
   ]),
 
   /**
    * Map containing all schemes that should be ignored by content policy.
    * @type Set.<string>
    */
   whitelistSchemes: new Set(),
 
   /**
    * Called on module startup, initializes various exported properties.
    */
   init: function()
   {
     // whitelisted URL schemes
     for (let scheme of Prefs.whitelistschemes.toLowerCase().split(" "))
       this.whitelistSchemes.add(scheme);
 
-    Utils.addChildMessageListener("AdblockPlus:ShouldAllow", message => this.shouldAllow(message));
+    port.on("shouldAllow", (message, sender) => this.shouldAllow(message));
 
     // Generate class identifier used to collapse nodes and register
     // corresponding stylesheet.
     let collapsedClass = "";
     let offset = "a".charCodeAt(0);
     for (let i = 0; i < 20; i++)
       collapsedClass +=  String.fromCharCode(offset + Math.random() * 26);
-    Utils.addChildMessageListener("AdblockPlus:GetCollapsedClass", () => collapsedClass);
+    port.on("getCollapsedClass", (message, sender) => collapsedClass);
 
     let collapseStyle = Services.io.newURI("data:text/css," +
         encodeURIComponent("." + collapsedClass +
         "{-moz-binding: url(chrome://global/content/bindings/general.xml#foobarbazdummy) !important;}"), null, null);
     Utils.styleService.loadAndRegisterSheet(collapseStyle, Ci.nsIStyleSheetService.USER_SHEET);
     onShutdown.add(() =>
     {
       Utils.styleService.unregisterSheet(collapseStyle, Ci.nsIStyleSheetService.USER_SHEET);
     });
   },
 
   /**
    * Checks whether a node should be blocked, hides it if necessary
    * @param {Object} data  request data
    * @param {String} data.contentType
    * @param {String} data.location  location of the request, filter key if contentType is ELEMHIDE
    * @param {Object[]} data.frames
    * @param {Boolean} data.isPrivate  true if the request belongs to a private browsing window
    * @return {Object} An object containing properties allow, collapse and hits
    *                  indicating how this request should be handled.
    */
   shouldAllow: function({contentType, location, frames, isPrivate})
   {
     let hits = [];
 
     function addHit(frameIndex, contentType, docDomain, thirdParty, location, filter)
     {
       if (filter && !isPrivate)
-      {
-        let topWindowLocation = frames[frames.length-1].location;
-        FilterStorage.increaseHitCount(filter, getHostname(topWindowLocation));

[Wladimir Palant, 2016/02/15 12:25:40]

getHostname(topWindowLocation) should be identical to docDomain.

[saroyanm, 2016/02/19 17:38:11]

Done.

-      }
+        FilterStorage.increaseHitCount(filter, docDomain, thirdParty);
+
       hits.push({
         frameIndex, contentType, docDomain, thirdParty, location,
         filter: filter ? filter.text : null,
         filterType: filter ? filter.type : null
       });
     }
 
     function response(allow, collapse)
     {
       return {allow, collapse, hits};
     }
 
     // Ignore whitelisted schemes
     if (!this.isBlockableScheme(location))
       return response(true, false);
 
     // Interpret unknown types as "other"
-    if (!this.contentTypes.has(contentType))
-      contentType = "OTHER";
-
+    contentType = this.contentTypes.get(contentType) || "OTHER";
+
+    let nogeneric = false;
+    if (Prefs.enabled)
+    {
+      let whitelistHit =
+          this.isFrameWhitelisted(frames, contentType == "ELEMHIDE");
+      if (whitelistHit)
+      {
+        let [frameIndex, matchType, docDomain, thirdParty, location, filter] = whitelistHit;
+        addHit(frameIndex, matchType, docDomain, thirdParty, location, filter);
+        if (matchType == "DOCUMENT" || matchType == "ELEMHIDE")
+          return response(true, false);
+        else
+          nogeneric = true;
+      }
+    }
+
+    let match = null;
     let wndLocation = frames[0].location;
     let docDomain = getHostname(wndLocation);
-    let match = null;
     let [sitekey, sitekeyFrame] = getSitekey(frames);
-    let nogeneric = false;
-    if (!match && Prefs.enabled)
-    {
-      let testSitekey = sitekey;
-      let testSitekeyFrame = sitekeyFrame;
-      for (let i = 0; i < frames.length; i++)
-      {
-        let frame = frames[i];
-        let testWndLocation = frame.location;
-        let parentWndLocation = frames[Math.min(i + 1, frames.length - 1)].location;
-        let parentDocDomain = getHostname(parentWndLocation);
-
-        let typeMap = RegExpFilter.typeMap.DOCUMENT;
-        if (contentType == "ELEMHIDE")
-          typeMap = typeMap | RegExpFilter.typeMap.ELEMHIDE;
-        let whitelistMatch = defaultMatcher.matchesAny(testWndLocation, typeMap, parentDocDomain, false, testSitekey);
-        if (whitelistMatch instanceof WhitelistFilter)
-        {
-          let whitelistType = (whitelistMatch.contentType & RegExpFilter.typeMap.DOCUMENT) ? "DOCUMENT" : "ELEMHIDE";
-          addHit(i, whitelistType, parentDocDomain, false, testWndLocation,
-              whitelistMatch);
-          return response(true, false);
-        }
-
-        let genericType = (contentType == "ELEMHIDE" ? "GENERICHIDE" : "GENERICBLOCK");
-        let nogenericMatch = defaultMatcher.matchesAny(testWndLocation,
-            RegExpFilter.typeMap[genericType], parentDocDomain, false, testSitekey);
-        if (nogenericMatch instanceof WhitelistFilter)
-        {
-          nogeneric = true;
-          addHit(i, genericType, parentDocDomain, false, testWndLocation,
-              nogenericMatch);
-        }
-
-        if (frame == testSitekeyFrame)
-          [testSitekey, testSitekeyFrame] = getSitekey(frames.slice(i + 1));
-      }
-    }
-
-    if (!match && contentType == "ELEMHIDE")
+    if (contentType == "ELEMHIDE")
     {
       match = ElemHide.getFilterByKey(location);
       location = match.text.replace(/^.*?#/, '#');
 
       if (!match.isActiveOnDomain(docDomain))
         return response(true, false);
 
       let exception = ElemHide.getException(match, docDomain);
       if (exception)
       {
@@ skipping 31 matching lines @@
     {
       let match = /^([\w\-]+):/.exec(location);
       scheme = match ? match[1] : null;
     }
     else
       scheme = location.scheme;
     return !this.whitelistSchemes.has(scheme);
   },
 
   /**
-   * Checks whether a page is whitelisted.
+   * Checks whether a top-level window is whitelisted.
    * @param {String} url
-   * @param {String} [parentUrl] location of the parent page
-   * @param {String} [sitekey] public key provided on the page
-   * @return {Filter} filter that matched the URL or null if not whitelisted
-   */
-  isWhitelisted: function(url, parentUrl, sitekey)
+   *    URL of the document loaded into the window
+   * @return {?WhitelistFilter}
+   *    exception rule that matched the URL if any
+   */
+  isWhitelisted: function(url)
   {
     if (!url)
       return null;
 
     // Do not apply exception rules to schemes on our whitelistschemes list.
     if (!this.isBlockableScheme(url))
       return null;
-
-    if (!parentUrl)
-      parentUrl = url;
 
     // Ignore fragment identifier
     let index = url.indexOf("#");
     if (index >= 0)
       url = url.substring(0, index);
 
-    let result = defaultMatcher.matchesAny(url, RegExpFilter.typeMap.DOCUMENT, getHostname(parentUrl), false, sitekey);
+    let result = defaultMatcher.matchesAny(url, RegExpFilter.typeMap.DOCUMENT,
+        getHostname(url), false, null);
     return (result instanceof WhitelistFilter ? result : null);
   },
 
   /**
-   * Checks whether the page loaded in a window is whitelisted for indication in the UI.
-   * @param wnd {nsIDOMWindow}
-   * @return {Filter} matching exception rule or null if not whitelisted
-   */
-  isWindowWhitelisted: function(wnd)
-  {
-    return this.isWhitelisted(getWindowLocation(wnd));
+   * Checks whether a frame is whitelisted.
+   * @param {Array} frames
+   *    frame structure as returned by getFrames() in child/utils module.
+   * @param {boolean} isElemHide
+   *    true if element hiding whitelisting should be considered
+   * @return {?Array}
+   *    An array with the hit parameters: frameIndex, contentType, docDomain,
+   *    thirdParty, location, filter. Note that the filter could be a
+   *    genericblock/generichide exception rule. If nothing matched null is
+   *    returned.
+   */
+  isFrameWhitelisted: function(frames, isElemHide)
+  {
+    let [sitekey, sitekeyFrame] = getSitekey(frames);
+    let nogenericHit = null;
+
+    let typeMap = RegExpFilter.typeMap.DOCUMENT;
+    if (isElemHide)
+      typeMap = typeMap | RegExpFilter.typeMap.ELEMHIDE;
+    let genericType = (isElemHide ? "GENERICHIDE" : "GENERICBLOCK");
+
+    for (let i = 0; i < frames.length; i++)
+    {
+      let frame = frames[i];
+      let wndLocation = frame.location;
+      let parentWndLocation = frames[Math.min(i + 1, frames.length - 1)].location;
+      let parentDocDomain = getHostname(parentWndLocation);
+
+      let match = defaultMatcher.matchesAny(wndLocation, typeMap, parentDocDomain, false, sitekey);
+      if (match instanceof WhitelistFilter)
+      {
+        let whitelistType = (whitelistMatch.contentType & RegExpFilter.typeMap.DOCUMENT) ? "DOCUMENT" : "ELEMHIDE";
+        return [i, whitelistType, parentDocDomain, false, wndLocation, match];
+      }
+
+      if (!nogenericHit)
+      {
+        match = defaultMatcher.matchesAny(wndLocation,
+            RegExpFilter.typeMap[genericType], parentDocDomain, false, sitekey);
+        if (match instanceof WhitelistFilter)
+          nogenericHit = [i, genericType, parentDocDomain, false, wndLocation, match];
+      }
+
+      if (frame == sitekeyFrame)
+        [sitekey, sitekeyFrame] = getSitekey(frames.slice(i + 1));
+    }
+
+    return nogenericHit;
   },
 
   /**
    * Deletes nodes that were previously stored with a
    * RequestNotifier.storeNodesForEntries() call or similar.
    * @param {string} id  unique ID of the nodes
    */
   deleteNodes: function(id)
   {
-    let messageManager = Cc["@mozilla.org/parentprocessmessagemanager;1"]
-                           .getService(Ci.nsIMessageBroadcaster);
-    messageManager.broadcastAsyncMessage("AdblockPlus:DeleteNodes", id);
+    port.emit("deleteNodes", id);
   },
 
   /**
    * Asynchronously re-checks filters for nodes given by an ID previously
    * returned by a RequestNotifier.storeNodesForEntries() call or similar.
    * @param {string} id  unique ID of the nodes
    * @param {RequestEntry} entry
    */
   refilterNodes: function(id, entry)
   {
-    let messageManager = Cc["@mozilla.org/parentprocessmessagemanager;1"]
-                           .getService(Ci.nsIMessageBroadcaster);
-    messageManager.broadcastAsyncMessage("AdblockPlus:RefilterNodes", {
+    port.emit("refilterNodes", {
       nodesID: id,
       entry: entry
     });
   }
 };
 Policy.init();
 
 /**
  * Extracts the hostname from a URL (might return null).
  */
@@ skipping 98 matching lines @@
   {
     // EffectiveTLDService throws on IP addresses, just compare the host name
     let host = "";
     try
     {
       host = uri.host;
     } catch (e) {}
     return host != docDomain;
   }
 }