Issue 3775 - fix saving of requests

lib/crawler.js

 /*
  * This Source Code is subject to the terms of the Mozilla Public License
  * version 2.0 (the "License"). You can obtain a copy of the License at
  * http://mozilla.org/MPL/2.0/.
  */
 
 /**
  * @module crawler
  */
 
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/Promise.jsm");
 
 function abprequire(module)
 {
   let result = {};
   result.wrappedJSObject = result;
   Services.obs.notifyObservers(result, "adblockplus-require", module);
   return result.exports;
 }
 
-let {Policy} = abprequire("contentPolicy");
 let {RequestNotifier} = abprequire("requestNotifier");
-let {Utils} = abprequire("utils");
 
-let dataForTab = new WeakMap();
 
 /**
  * Creates a pool of tabs and allocates them to tasks on request.
  *
  * @param {tabbrowser} browser
  *    The tabbed browser where tabs should be created
  * @param {int} maxtabs
  *    The maximum number of tabs to be allocated
  * @constructor
  */
@@ skipping 148 matching lines @@
         window.document.documentElement.acceptDialog();
       else
         window.close();
     }, false);
   },
 
   QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver, Ci.nsISupportsWeakReference])
 };
 
 /**
- * Retrieves crawler results associated with a particular content window.
- *
- * @param {Window} window
- *    Content window to retrieve crawler results for
- * @result {Object}
- *    Crawler results or undefined if the window wasn't created by the crawler.
- */
-function getDataForWindow(window)
-{
-  let topWindow = window.top;
-  if (!topWindow.document)
-    throw new Error("No document associated with the node's top window");
-  let tabbrowser = Utils.getChromeWindow(topWindow).getBrowser();
-  if (!tabbrowser)
-    throw new Error("Unable to get a tabbrowser reference from the window");
-  let browser = tabbrowser.getBrowserForDocument(topWindow.document);
-  if (!browser)
-    throw new Error("Unable to get browser for the content window");
-  let tab = tabbrowser.getTabForBrowser(browser);
-  if (!tab)
-    throw new Error("Unable to get tab for the browser");
-  return dataForTab.get(tab);
-};
-
-/**
  * Starts the crawling session. The crawler opens each URL in a tab and stores
  * the results.
  *
  * @param {Window} window
  *    The browser window we're operating in
  * @param {String[]} urls
  *    URLs to be crawled
  * @param {int} number_of_tabs
  *    Maximum number of tabs to be opened
  * @param {String} targetURL
  *    URL that should receive the results
  */
 function run(window, urls, timeout, maxtabs, targetURL, onDone)
 {
-  let requestNotifier = new RequestNotifier(null, function() {});
-
-  let origProcessNode = Policy.processNode;
-  Policy.processNode = processNodeReplacement.bind(null, origProcessNode, requestNotifier);
-
   let tabAllocator = new TabAllocator(window.getBrowser(), maxtabs);
   let loadListener = new LoadListener(window.getBrowser(), timeout);
   let running = 0;
   let windowCloser = new WindowCloser();
   let taskDone = function()
   {
     running--;
     if (running <= 0)
     {
-      Policy.processNode = origProcessNode;
-      requestNotifier.shutdown();
       loadListener.stop();
       windowCloser.stop();
       onDone();
     }
   };
 
   for (let url of urls)
   {
     running++;
     Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(function(result)
@@ skipping 26 matching lines @@
  *
  * @param {String} url
  * @param {TabAllocator} tabAllocator
  * @param {loadListener} loadListener
  * @result {Object}
  *    Crawling result
  */
 function* crawl_url(url, tabAllocator, loadListener)
 {
   let tab = yield tabAllocator.getTab();
-  let result = {url: url};
+  let result = {url, requests: []};
 
-  dataForTab.set(tab, result);
   try
   {
     result.startTime = Date.now();
+    let requestNotifier = new RequestNotifier(tab.linkedBrowser.outerWindowID, function({type, location, filter}, scanComplete)

[Wladimir Palant, 2016/03/14 19:50:43]

When the scan is completed the listener is called with null as first parameter -
this will produce an error right now. You need to check the parameter first,
before decomposing.

Style nit: while we didn't strictly enforce this rule here, please keep the code
reasonably within 80 characters.

[sergei, 2016/03/15 10:59:56]

Fixed, is the indentation correct now?

[sergei, 2016/03/15 10:59:56]

Fixed.

However, I guess, it should be documented in `RequestNotifier` constructor and I
actually cannot reproduce it, how to reproduce it in crawler?

[Wladimir Palant, 2016/03/15 11:05:44]

Yes, it should be documented. As to reproducing - I don't know, but
RequestNotifier.onComplete() will trigger the listener with null parameter.

+    {
+      result.requests.push({location, contentType: type, filter});
+    });

[Wladimir Palant, 2016/03/14 19:50:43]

You need to shut down this notifier when you are done, just like the tab is
being released. Otherwise the content side won't be shut down - you'll leak
memory and produce unnecessary IPC messages.

[sergei, 2016/03/15 10:59:56]

Done. Thanks, overlooked it.

+
     tab.linkedBrowser.loadURI(url, null, null);
     [result.channelStatus, result.headers] = yield loadListener.waitForLoad(tab);
     result.endTime = Date.now();
     result.finalUrl = tab.linkedBrowser.currentURI.spec;
 
     let document = tab.linkedBrowser.contentDocument;
     if (document.documentElement)
     {
       try
       {
@@ skipping 25 matching lines @@
 
 function reportException(e)
 {
   let stack = "";
   if (e && typeof e == "object" && "stack" in e)
     stack = e.stack + "\n";
 
   Cu.reportError(e);
   dump(e + "\n" + stack + "\n");
 }
-
-/**
- * Wrapper for the Policy.processNode() function in ABP. Calls the original
- * function and records all the data.
- *
- * @param {Function} origProcessNode
- *      The original processNode function.
- * @param {RequestNotifier} requestNotifier
- *      The crawler's RequestNotifier object instance.
- * @param {nsIDOMWindow} wnd
- * @param {nsIDOMElement} node
- * @param {Number} contentType
- * @param {nsIURI} location
- * @param  {Boolean} collapse
- * @return {Boolean}
- */
-function processNodeReplacement(origProcessNode, requestNotifier, wnd, node, contentType, location, collapse)
-{
-  let filters = [];
-  let origListener = requestNotifier.listener;
-  requestNotifier.listener = function(window, node, entry)
-  {
-    if (entry.filter)
-      filters.push(entry.filter.text);
-  };
-
-  /*
-   * Call the original processNode. If the original throws, then we will too, so this is outside a try clause.
-   */
-  let result;
-  try
-  {
-    result = origProcessNode(wnd, node, contentType, location, collapse);
-  }
-  finally
-  {
-    requestNotifier.listener = origListener;
-  }
-
-  try
-  {
-    let data = getDataForWindow(wnd);
-    if (data)
-    {
-      if (!("requests" in data))
-        data.requests = [];
-      data.requests.push({
-        contentType: contentType,
-        location: (contentType == Policy.type.ELEMHIDE ? location.text : location.spec),
-        blocked: result != Ci.nsIContentPolicy.ACCEPT,
-        filters: filters
-      });
-    }
-  }
-  catch (e)
-  {
-    reportException(e);
-  }
-  return result;
-};