Issue 3792 - Fix to support multiprocess firefox

lib/crawler.js

 /*
  * This Source Code is subject to the terms of the Mozilla Public License
  * version 2.0 (the "License"). You can obtain a copy of the License at
  * http://mozilla.org/MPL/2.0/.
  */
 
 'use strict';
 
 /**
  * @module crawler
  */
 
 const {XPCOMUtils} = Cu.import("resource://gre/modules/XPCOMUtils.jsm", {});
 const {Task} = Cu.import("resource://gre/modules/Task.jsm");
+const {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm", {});
 
 function abprequire(module)
 {
   let result = {};
   result.wrappedJSObject = result;
   Services.obs.notifyObservers(result, "adblockplus-require", module);
   return result.exports;
 }
 
 let {RequestNotifier} = abprequire("requestNotifier");
@@ skipping 76 matching lines @@
    * @param {tab} tab
    */
   releaseTab: function(tab)
   {
     // If we are about to close last tab don't close it immediately to keep
     // the window alive. It will be closed when a new tab is created.
     if (this._tabs > 1)
       this._browser.removeTab(tab);
     else
     {
       // navigate away from early opened URL

[Wladimir Palant, 2016/09/29 11:44:59]

early => previously

[sergei, 2016/09/29 15:36:22]

Done.

       tab.linkedBrowser.loadURI('about:blank', null, null);

[Wladimir Palant, 2016/09/29 11:44:59]

Double quotation marks please.

[sergei, 2016/09/29 15:36:22]

Done.

       this._tabKeepingWindowAlive = tab;
     }
 
     this._tabs--;
     if (this._resolvers.length && this._tabs < this._maxtabs)
     {
       this._resolvers.shift()(this._createTab());
     }
   },
 };
 
 /**
- * Observes page loads in a particular tabbed browser.
- *
- * @param {tabbrowser} browser
- *    The tabbed browser to be observed
- * @param {int} timeout
- *    Load timeout in milliseconds
- * @constructor
- */
-function LoadListener(browser, timeout)
-{
-  this._browser = browser;
-  this._deferred = new Map();
-  this._timeout = timeout;
-  browser.addTabsProgressListener(this);
-}
-LoadListener.prototype = {
-  /**
-   * Returns a promise that will be resolved when the page in the specified tab
-   * finishes loading. Loading will be stopped if the timeout is reached.
-   *
-   * @param {tab} tab
-   * @result {Promise}
-   */
-  waitForLoad: function(tab)
-  {
-    let deferred = Promise.defer();
-    this._deferred.set(tab.linkedBrowser, deferred);
-
-    tab.ownerDocument.defaultView.setTimeout(function()
-    {
-      tab.linkedBrowser.stop();
-    }, this._timeout);
-
-    return deferred.promise;
-  },
-
-  /**
-   * Deactivates this object.
-   */
-  stop: function()
-  {
-    this._browser.removeTabsProgressListener(this);
-  },
-
-  onStateChange: function(browser, progress, request, flags, status)
-  {
-    if ((flags & Ci.nsIWebProgressListener.STATE_STOP) && (flags & Ci.nsIWebProgressListener.STATE_IS_WINDOW))
-    {
-      let deferred = this._deferred.get(browser);
-      if (deferred)
-      {
-        this._deferred.delete(browser);
-
-        let headers = [];
-        if (request instanceof Ci.nsIHttpChannel)
-        {
-          try
-          {
-            headers.push("HTTP/x.x " + request.responseStatus + " " + request.responseStatusText);
-            request.visitResponseHeaders((header, value) => headers.push(header + ": " + value));
-          }
-          catch (e)
-          {
-            // Exceptions are expected here
-          }
-        }
-        deferred.resolve([status, headers]);
-      }
-    }
-  }
-};
-
-/**
  * Once created, this object will make sure all new windows are dismissed
  * immediately.
  *
  * @constructor
  */
 function WindowCloser()
 {
   Services.obs.addObserver(this, "xul-window-registered", true)
 }
 WindowCloser.prototype = {
@@ skipping 14 matching lines @@
       if (window.document.documentElement.localName == 'dialog')
         window.document.documentElement.acceptDialog();
       else
         window.close();
     }, false);
   },
 
   QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver, Ci.nsISupportsWeakReference])
 };
 
+function configureFrameScript()
+{
+  const info = require("info");
+  const frameScriptPath = info.addonRoot + '/lib/child/frameScript.js?' + Math.random() +
+    '&info=' + encodeURIComponent(JSON.stringify(info));

[Wladimir Palant, 2016/09/29 11:44:59]

Double quotation marks please.

What's the point of sending the info object to the frame script? It doesn't need
this information.

[sergei, 2016/09/29 15:36:22]

Done.
I use info.addonRoot to load modules, however it should be in the commit when it
is actually needed, so I have removed it.

+  Services.mm.loadFrameScript(frameScriptPath, true);
+
+  onShutdown.add(() =>
+  {
+    Services.mm.removeDelayedFrameScript(frameScriptPath);
+  });
+}
+
 /**
  * Starts the crawling session. The crawler opens each URL in a tab and stores
  * the results.
  *
  * @param {Window} window
  *    The browser window we're operating in
  * @param {String[]} urls
  *    URLs to be crawled
  * @param {int} timeout
  *    Load timeout in milliseconds
  * @param {int} maxtabs
  *    Maximum number of tabs to be opened
  * @param {String} targetURL
  *    URL that should receive the results
  * @param {Function} onDone
  *    The callback which is called after finishing of crawling of all URLs.
  */
 function run(window, urls, timeout, maxtabs, targetURL, onDone)
 {
+  configureFrameScript();
   new Promise((resolve, reject) =>
   {
     if (FilterStorage.subscriptions.length > 0)
     {
       resolve();
       return;
     }
     let onFiltersLoaded = (action, item, newValue, oldValue) =>
     {
       if (action == "load")
@@ skipping 20 matching lines @@
  * @param {int} maxtabs
  *    Maximum number of tabs to be opened
  * @param {String} targetURL
  *    URL that should receive the results
  * @param {Function} onDone
  *    The callback which is called after finishing of all tasks.
  */
 function crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)
 {
   let tabAllocator = new TabAllocator(window.getBrowser(), maxtabs);
-  let loadListener = new LoadListener(window.getBrowser(), timeout);
+
   let running = 0;
   let windowCloser = new WindowCloser();
   let taskDone = function()
   {
     running--;
     if (running <= 0)
     {
-      loadListener.stop();
       windowCloser.stop();
       onDone();
     }
   };
 
   for (let url of urls)
   {
     running++;
-    Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(function(result)
+    Task.spawn(crawl_url.bind(null, url, tabAllocator, timeout)).then(function(result)
     {
       let request = new XMLHttpRequest();
       request.open("POST", targetURL);
       request.addEventListener("load", taskDone, false);
       request.addEventListener("error", taskDone, false);
       request.send(JSON.stringify(result));
     }, function(url, exception)
     {
       reportException(exception);
 
       let request = new XMLHttpRequest();
       request.open("POST", targetURL);
       request.addEventListener("load", taskDone, false);
       request.addEventListener("error", taskDone, false);
       request.send(JSON.stringify({
         url: url,
         startTime: Date.now(),
         error: String(exception)
       }));
     }.bind(null, url));
   }
 }
 
 /**
+ * Expects to receive page info gathered in a content process for the specified
+ * `tab`. If there is no relevant message within specified `timeout` then
+ * the result promise is resolved with error object.
+ * @param tab
+ *    Tab in which we are interested in
+ * @param {int} timeout
+ *    Timeout in milliseconds
+ * @return {Promise} promise which will be resolved with the received page info
+ */
+function getPageInfo(tab, timeout)
+{
+  return new Promise((resolve, result) =>
+  {
+    const mm = tab.linkedBrowser.messageManager;
+    let timerID;
+    let onDone = (msg) =>
+    {
+      mm.removeMessageListener("abpcrawler:pageInfoGathered", onDone);
+      clearTimeout(timerID);
+      resolve(msg.data);
+    }
+    mm.addMessageListener("abpcrawler:pageInfoGathered", onDone);
+    timerID = setTimeout(() => onDone({data: {error: "timeout"}}), timeout);
+  });
+}
+
+/**
  * Crawls a URL. This is a generator meant to be used via a Task object.
  *
  * @param {String} url
  * @param {TabAllocator} tabAllocator
- * @param {loadListener} loadListener
+ * @param {int} timeout
+ *    Load timeout in milliseconds
  * @result {Object}
  *    Crawling result
  */
-function* crawl_url(url, tabAllocator, loadListener)
+function* crawl_url(url, tabAllocator, timeout)
 {
   let tab = yield tabAllocator.getTab();
   let result = {url, requests: []};
   let requestNotifier;
   try
   {
     result.startTime = Date.now();
     requestNotifier = new RequestNotifier(tab.linkedBrowser.outerWindowID,
       function(entry, scanComplete)
     {
       if (!entry)
         return;
       let {type: contentType, location, filter} = entry;
       result.requests.push({location, contentType, filter});
     });
 
     tab.linkedBrowser.loadURI(url, null, null);
-    [result.channelStatus, result.headers] = yield loadListener.waitForLoad(tab);
+
+    Object.assign(result, yield getPageInfo(tab, timeout));
+    result.finalUrl = tab.linkedBrowser.currentURI.spec;
     result.endTime = Date.now();
-    result.finalUrl = tab.linkedBrowser.currentURI.spec;
-
-    let document = tab.linkedBrowser.contentDocument;
-    if (document.documentElement)
-    {
-      try
-      {
-        let canvas = document.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
-        canvas.width = document.documentElement.scrollWidth;
-        canvas.height = document.documentElement.scrollHeight;
-
-        let context = canvas.getContext("2d");
-        context.drawWindow(document.defaultView, 0, 0, canvas.width, canvas.height, "rgb(255, 255, 255)");
-        result.screenshot = canvas.toDataURL("image/jpeg", 0.8);
-      }
-      catch (e)
-      {
-        reportException(e);
-        result.error = "Capturing screenshot failed: " + e;
-      }
-
-      // TODO: Capture frames as well?
-      let serializer = new tab.ownerDocument.defaultView.XMLSerializer();
-      result.source = serializer.serializeToString(document.documentElement);
-    }
   }
   finally
   {
     if (requestNotifier)
       requestNotifier.shutdown();
     tabAllocator.releaseTab(tab);
   }
   return result;
 }
 
 function reportException(e)
 {
   let stack = "";
   if (e && typeof e == "object" && "stack" in e)
     stack = e.stack + "\n";
 
   Cu.reportError(e);
   dump(e + "\n" + stack + "\n");
 }