Issue 3792 - Fix to support multiprocess firefox

lib/crawler.js

Index: lib/crawler.js
diff --git a/lib/crawler.js b/lib/crawler.js
index f01f6f81851bec00f337718d7b46e9f1543a4add..c0fd025ca7aa52b188263a287c6ba35a7e24a730 100644
--- a/lib/crawler.js
+++ b/lib/crawler.js
@@ -11,6 +11,7 @@
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/Promise.jsm");

[Wladimir Palant, 2016/09/14 16:13:18]

Just realized that this import hasn't been removed - please do so, we want to
use the default Promise object.

+Cu.import("resource://gre/modules/Timer.jsm");

[Wladimir Palant, 2016/09/14 16:13:18]

Nit: We don't import like that any more, symbols should be imported explicitly.

  let {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm",
{});

[sergei, 2016/09/29 09:58:14]

Fixed and the rest related to coding style is addressed in codereview
https://codereview.adblockplus.org/29355247/

 
 function abprequire(module)
 {
@@ -134,79 +135,6 @@ TabAllocator.prototype = {
 };
 
 /**
- * Observes page loads in a particular tabbed browser.
- *
- * @param {tabbrowser} browser
- *    The tabbed browser to be observed
- * @param {int} timeout
- *    Load timeout in milliseconds
- * @constructor
- */
-function LoadListener(browser, timeout)
-{
-  this._browser = browser;
-  this._deferred = new Map();
-  this._timeout = timeout;
-  browser.addTabsProgressListener(this);
-}
-LoadListener.prototype = {
-  /**
-   * Returns a promise that will be resolved when the page in the specified tab
-   * finishes loading. Loading will be stopped if the timeout is reached.
-   *
-   * @param {tab} tab
-   * @result {Promise}
-   */
-  waitForLoad: function(tab)
-  {
-    let deferred = Promise.defer();
-    this._deferred.set(tab.linkedBrowser, deferred);
-
-    tab.ownerDocument.defaultView.setTimeout(function()
-    {
-      tab.linkedBrowser.stop();
-    }, this._timeout);
-
-    return deferred.promise;
-  },
-
-  /**
-   * Deactivates this object.
-   */
-  stop: function()
-  {
-    this._browser.removeTabsProgressListener(this);
-  },
-
-  onStateChange: function(browser, progress, request, flags, status)
-  {
-    if ((flags & Ci.nsIWebProgressListener.STATE_STOP) && (flags & Ci.nsIWebProgressListener.STATE_IS_WINDOW))
-    {
-      let deferred = this._deferred.get(browser);
-      if (deferred)
-      {
-        this._deferred.delete(browser);
-
-        let headers = [];
-        if (request instanceof Ci.nsIHttpChannel)
-        {
-          try
-          {
-            headers.push("HTTP/x.x " + request.responseStatus + " " + request.responseStatusText);
-            request.visitResponseHeaders((header, value) => headers.push(header + ": " + value));
-          }
-          catch (e)
-          {
-            // Exceptions are expected here
-          }
-        }
-        deferred.resolve([status, headers]);
-      }
-    }
-  }
-};
-
-/**
  * Once created, this object will make sure all new windows are dismissed
  * immediately.
  *
@@ -300,7 +228,7 @@ exports.run = run;
 function crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)
 {
   let tabAllocator = new TabAllocator(window.getBrowser(), maxtabs);
-  let loadListener = new LoadListener(window.getBrowser(), timeout);
+
   let running = 0;
   let windowCloser = new WindowCloser();
   let taskDone = function()
@@ -308,7 +236,6 @@ function crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)
     running--;
     if (running <= 0)
     {
-      loadListener.stop();
       windowCloser.stop();
       onDone();
     }
@@ -317,7 +244,7 @@ function crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)
   for (let url of urls)
   {
     running++;
-    Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(function(result)
+    Task.spawn(crawl_url.bind(null, url, tabAllocator, timeout)).then(function(result)
     {
       let request = new XMLHttpRequest();
       request.open("POST", targetURL);
@@ -342,15 +269,43 @@ function crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)
 }
 
 /**
+ * Expects to receive page info gathered in a content process for the specified
+ * `tab`. If there is no relevant message within specified `timeout` then
+ * the result promise is resolve with error object.
+ * @param tab
+ *    Tab in which we are interested in
+ * @param {int} timeout
+ *    Timeout in milliseconds
+ * @return {Promise} promise which will be resolved with the received page info
+ */
+function getPageInfo(tab, timeout)
+{
+  return new Promise((resolve, result) =>
+  {
+    let mm = tab.linkedBrowser.messageManager;
+    let timerID;
+    let onDone = (pageInfo) =>
+    {
+      mm.removeMessageListener("abpcrawler:pageInfoGathered", onDone);
+      clearTimeout(timerID);
+      resolve(pageInfo);
+    }
+    mm.addMessageListener("abpcrawler:pageInfoGathered", (msg) => onDone(msg.data));;

[Wladimir Palant, 2016/09/14 16:11:48]

That's not the callback you are removing above. Also: duplicate semicolon.

[sergei, 2016/09/29 09:58:13]

Fixed. Sorry, overlooked.

+    timerID = setTimeout(onDone.bind(this, {error: "timeout"}), timeout);

[Wladimir Palant, 2016/09/14 16:11:49]

How about not using bind() here, for clarity and consistency?

() => onDone({data: {error: "timeout"}})

[sergei, 2016/09/29 09:58:13]

Done. Good idea.

+  });
+}
+
+/**
  * Crawls a URL. This is a generator meant to be used via a Task object.
  *
  * @param {String} url
  * @param {TabAllocator} tabAllocator
- * @param {loadListener} loadListener
+ * @param {int} timeout
+ *    Load timeout in milliseconds
  * @result {Object}
  *    Crawling result
  */
-function* crawl_url(url, tabAllocator, loadListener)
+function* crawl_url(url, tabAllocator, timeout)
 {
   let tab = yield tabAllocator.getTab();
   let result = {url, requests: []};
@@ -368,33 +323,10 @@ function* crawl_url(url, tabAllocator, loadListener)
     });
 
     tab.linkedBrowser.loadURI(url, null, null);
-    [result.channelStatus, result.headers] = yield loadListener.waitForLoad(tab);
-    result.endTime = Date.now();
-    result.finalUrl = tab.linkedBrowser.currentURI.spec;
-
-    let document = tab.linkedBrowser.contentDocument;
-    if (document.documentElement)
-    {
-      try
-      {
-        let canvas = document.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
-        canvas.width = document.documentElement.scrollWidth;
-        canvas.height = document.documentElement.scrollHeight;
-
-        let context = canvas.getContext("2d");
-        context.drawWindow(document.defaultView, 0, 0, canvas.width, canvas.height, "rgb(255, 255, 255)");
-        result.screenshot = canvas.toDataURL("image/jpeg", 0.8);
-      }
-      catch (e)
-      {
-        reportException(e);
-        result.error = "Capturing screenshot failed: " + e;
-      }
 
-      // TODO: Capture frames as well?
-      let serializer = new tab.ownerDocument.defaultView.XMLSerializer();
-      result.source = serializer.serializeToString(document.documentElement);
-    }
+    Object.assign(result, yield getPageInfo(tab, timeout));
+    result.finalUrl = tab.linkedBrowser.currentURI.spec;
+    result.endTime = Date.now();
   }
   finally
   {
@@ -414,3 +346,13 @@ function reportException(e)
   Cu.reportError(e);
   dump(e + "\n" + stack + "\n");
 }
+
+let {addonRoot} = require("info");
+let frameScriptPath = addonRoot + "/lib/child/frameScript.js";
+let globalMessageManager = Services.mm;
+globalMessageManager.loadFrameScript(frameScriptPath, true);
+
+onShutdown.add(() =>
+{
+  globalMessageManager.removeDelayedFrameScript(frameScriptPath);
+});