Issue 154 - Added devtools panel showing blocked and blockable items

lib/devtools.js

Index: lib/devtools.js
===================================================================
new file mode 100644
--- /dev/null
+++ b/lib/devtools.js
@@ -0,0 +1,347 @@
+/*
+ * This file is part of Adblock Plus <https://adblockplus.org/>,
+ * Copyright (C) 2006-2016 Eyeo GmbH
+ *
+ * Adblock Plus is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * Adblock Plus is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+"use strict";
+
+let {RegExpFilter, WhitelistFilter, ElemHideFilter} = require("filterClasses");
+let {SpecialSubscription} = require("subscriptionClasses");
+let {FilterStorage} = require("filterStorage");
+let {defaultMatcher} = require("matcher");
+let {FilterNotifier} = require("filterNotifier");
+
+// Mapping of inspected tabs to their devpanel page
+// and recorded items. We can't use a PageMap here,
+// because data must persist after navigation/reload.
+let panels = Object.create(null);
+
+function hasPanels()
+{
+  return Object.getOwnPropertyNames(panels).length > 0;
+}
+
+function getRequestInfo(request)
+{
+  return {
+    url: request.url,
+    type: request.type,
+    docDomain: request.docDomain
+  };
+}
+
+function getFilterInfo(filter)
+{
+  if (!filter)
+    return null;
+
+  let userDefined = false;
+  let subscriptionTitle = null;
+
+  for (let subscription of filter.subscriptions)
+  {
+    if (!subscription.disabled)
+    {
+      if (subscription instanceof SpecialSubscription)
+        userDefined = true;
+      else
+        subscriptionTitle = subscription.title;
+    }
+  }
+
+  return {
+    text: filter.text,
+    whitelisted: filter instanceof WhitelistFilter,
+    userDefined: userDefined,
+    subscription: subscriptionTitle
+  };
+}
+
+function hasRecord(panel, request, filter)
+{
+  return panel.records.some(record =>
+    record.request.type      == request.type      &&
+    record.request.url       == request.url       &&
+    record.request.docDomain == request.docDomain &&
+
+    // Matched element hiding filters don't relate to a particular request,
+    // so we also have to match the CSS selector in order to distinguish them.
+    (request.type != "ELEMHIDE" || record.filter.selector == filter.selector)
+  );
+}
+
+function addRecord(panel, request, filter)
+{
+  if (!hasRecord(panel, request, filter))
+  {
+    panel.panel.sendMessage({
+      type: "add-record",
+      request: getRequestInfo(request),
+      filter: getFilterInfo(filter)
+    });
+
+    panel.records.push({
+      request: request,
+      filter: filter
+    });
+  }
+}
+
+function matchRequest(request)
+{
+  return defaultMatcher.matchesAny(
+    request.url,
+    RegExpFilter.typeMap[request.type],
+    request.docDomain,
+    request.thirdParty,
+    request.sitekey,
+    request.specificOnly
+  );
+}
+
+/**
+ * Logs a request in the devtools panel.
+ *
+ * @param {Page}    page           The page the request occured on
+ * @param {string]  url            The URL of the request
+ * @param {string}  type           The request type
+ * @param {string}  docDomain      The IDN-decoded hostname of the document
+ * @param {boolean} thirdParty     Whether the origin of the request and document differs
+ * @param {string}  [sitekey]      The active sitekey if there is any
+ * @param {boolean} [specificOnly] Whether generic filters should be ignored
+ * @param {filter}  [filter]       The matched filter or null if there is no match
+ */
+exports.logRequest = function(page, url, type, docDomain, thirdParty, sitekey, specificOnly, filter)

[kzar, 2016/01/31 13:33:52]

Nit: Mind wrapping this?

[Sebastian Noack, 2016/02/02 10:39:53]

Done.

+{
+  let panel = panels[page._id];
+  if (panel && !panel.reload && !panel.reloading)
+  {
+    let request = {
+      url: url,
+      type: type,
+      docDomain: docDomain,
+      thirdParty: thirdParty,
+      sitekey: sitekey,
+      specificOnly: specificOnly
+    };
+
+    addRecord(panel, request, filter);
+  }
+};
+
+/**
+ * Logs active element hiding filters in the devtools panel.
+ *
+ * @param {Page}     page       The page the elements were hidden on
+ * @param {string[]} selectors  The CSS selectors of active elemhide filters
+ * @param {string}   docDomain  The IDN-decoded hostname of the document
+ */
+exports.logHiddenElements = function(page, selectors, docDomain)
+{
+  let panel = panels[page._id];
+  if (panel && !panel.reload && !panel.reloading)
+  {
+    for (let subscription of FilterStorage.subscriptions)
+    {
+      if (subscription.disabled)
+        continue;
+
+      for (let filter of subscription.filters)
+      {
+        if (!(filter instanceof ElemHideFilter))
+          continue;
+        if (selectors.indexOf(filter.selector) == -1)
+          continue;
+        if (!filter.isActiveOnDomain(docDomain))
+          continue;
+
+        addRecord(panel, {type: "ELEMHIDE", docDomain: docDomain}, filter);
+      }
+    }
+  }
+};
+
+/**
+ * Checks whether a page is inspected by the devtools panel.
+ *
+ * @param {Page} page
+ * @return {boolean}
+ */
+exports.hasPanel = function(page)
+{
+  return page._id in panels;
+};
+
+function onBeforeRequest(details)
+{
+  let panel = panels[details.tabId];
+
+  // Clear the devtools panel and reload the inspected tab without caching
+  // when a new request is issued. However, make sure that we don't end up
+  // in an infinite recursion if we already triggered a reload.
+  if (panel.reloading)
+  {
+    panel.reloading = false;
+  }
+  else
+  {
+    panel.records = [];
+    panel.panel.sendMessage({type: "reset"});
+
+    // We can't repeat the request if it isn't a GET request. Chrome would
+    // prompt the user to confirm reloading the page, and POST requests are
+    // known to cause issues on many websites if repeated.
+    if (details.method == "GET")
+      panel.reload = true;
+  }
+}
+
+function onLoading(page)
+{
+  let tabId = page._id;
+  let panel = panels[tabId];
+
+  // Reloading the tab is the only way that allows bypassing all caches, in
+  // order to see all requests in the devtools panel. Reloading must not be
+  // performed before the tab changes to "loading", otherwise it will load the
+  // previous URL.
+  if (panel && panel.reload)
+  {
+    chrome.tabs.reload(tabId, {bypassCache: true});
+
+    panel.reload = false;
+    panel.reloading = true;
+  }
+}
+
+function onFilterChange(action, arg)
+{
+  let added, filters;
+  switch (action)
+  {
+    case "filter.added":
+      added = true;
+      filters = [arg];
+      break;
+
+    case "filter.removed":
+      added = false;
+      filters = [arg];
+      break;
+
+    // When there haven't ever been any user filters before, the subscription is
+    // added, triggering a "subscription.added" instead of a "filter.added" event.
+    case "subscription.added":
+      if (arg instanceof SpecialSubscription)
+      {
+        added = true;
+        filters = arg.filters;
+        break;
+      }
+
+    default:
+      return;
+  }
+
+  for (let tabId in panels)
+  {
+    let panel = panels[tabId];
+
+    for (let i = 0; i < panel.records.length; i++)
+    {
+      let record = panel.records[i];
+
+      // If an added filter matches a request shown in the devtools panel,
+      // update that record to show the new filter. Ignore element hiding
+      // filters since there are only records for element hiding filters that
+      // already match, and also we don't know if any new element matches.
+      if (added)
+      {
+        if (record.request.type == "ELEMHIDE")
+          continue;
+
+        let filter = matchRequest(record.request);
+        if (filters.indexOf(filter) == -1)
+          continue;
+
+        record.filter = filter;
+      }
+
+      // If a filter shown in the devtools panel got removed, update that
+      // record to show the filter (if any) that matches now instead.
+      // However, for element hiding, just remove the record, since we don't
+      // know whether another filter would match until the page is reloaded.
+      else
+      {
+        if (filters.indexOf(record.filter) == -1)
+          continue;
+
+        if (record.request.type == "ELEMHIDE")
+        {
+          panel.panel.sendMessage({
+            type: "remove-record",
+            index: i
+          });
+          panel.records.splice(i--, 1);
+          continue;
+        }
+
+        record.filter = matchRequest(record.request);
+      }
+
+      panel.panel.sendMessage({
+        type: "update-record",
+        index: i,
+        request: getRequestInfo(record.request),
+        filter: getFilterInfo(record.filter)
+      });
+    }
+  }
+}
+
+ext.devtools.onCreated.addListener(function(panel)

[kzar, 2016/01/31 13:33:52]

I guess I'm undecided about the ext.devtools abstraction. We're only using it
for Chrome anyway, so couldn't we just listen for `chrome.runtime.onConnect`
here instead?

[Sebastian Noack, 2016/02/02 10:39:53]

Yes, I would much prefer that as well. But note that we need some kind of
abstraction for adblockplusui. Feel free to look at the code there. Suggestions
are welcome.

[Sebastian Noack, 2016/02/02 11:22:11]

OK, I looked into it. And indeed, as we are in the background page, we don't
need to bother about the mock implementation in adblockplusui. And using
chrome.runtime.onConnect directly, seems to be more appropriate here. Done.

+{
+  let inspectedTabId = panel.inspectedTabId;
+  let localOnBeforeRequest = onBeforeRequest.bind();
+
+  chrome.webRequest.onBeforeRequest.addListener(
+    localOnBeforeRequest,
+    {
+      urls:  ["<all_urls>"],
+      types: ["main_frame"],
+      tabId: inspectedTabId
+    }
+  );
+
+  if (!hasPanels())
+  {
+    ext.pages.onLoading.addListener(onLoading);
+    FilterNotifier.addListener(onFilterChange);
+  }
+
+  panel.onRemoved.addListener(function()
+  {
+    delete panels[inspectedTabId];
+    chrome.webRequest.onBeforeRequest.removeListener(localOnBeforeRequest);
+
+    if (!hasPanels())
+    {
+      FilterNotifier.removeListener(onFilterChange);
+      ext.pages.onLoading.removeListener(onLoading);
+    }
+  });
+
+  panels[inspectedTabId] = {panel: panel, records: []};
+});