Issue 3792 - Fix to support multiprocess firefox

lib/crawler.js

 /*
  * This Source Code is subject to the terms of the Mozilla Public License
  * version 2.0 (the "License"). You can obtain a copy of the License at
  * http://mozilla.org/MPL/2.0/.
  */
 
 /**
  * @module crawler
  */
 
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/Promise.jsm");

[Wladimir Palant, 2016/09/14 16:13:18]

Just realized that this import hasn't been removed - please do so, we want to
use the default Promise object.

+Cu.import("resource://gre/modules/Timer.jsm");

[Wladimir Palant, 2016/09/14 16:13:18]

Nit: We don't import like that any more, symbols should be imported explicitly.

  let {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm",
{});

[sergei, 2016/09/29 09:58:14]

Fixed and the rest related to coding style is addressed in codereview
https://codereview.adblockplus.org/29355247/

 
 function abprequire(module)
 {
   let result = {};
   result.wrappedJSObject = result;
   Services.obs.notifyObservers(result, "adblockplus-require", module);
   return result.exports;
 }
 
 let {RequestNotifier} = abprequire("requestNotifier");
@@ skipping 103 matching lines @@
       }
       else if (this._tabs < this._maxtabs)
       {
         this._resolvers.shift()(this._createTab());
       }
     }
   },
 };
 
 /**
- * Observes page loads in a particular tabbed browser.
- *
- * @param {tabbrowser} browser
- *    The tabbed browser to be observed
- * @param {int} timeout
- *    Load timeout in milliseconds
- * @constructor
- */
-function LoadListener(browser, timeout)
-{
-  this._browser = browser;
-  this._deferred = new Map();
-  this._timeout = timeout;
-  browser.addTabsProgressListener(this);
-}
-LoadListener.prototype = {
-  /**
-   * Returns a promise that will be resolved when the page in the specified tab
-   * finishes loading. Loading will be stopped if the timeout is reached.
-   *
-   * @param {tab} tab
-   * @result {Promise}
-   */
-  waitForLoad: function(tab)
-  {
-    let deferred = Promise.defer();
-    this._deferred.set(tab.linkedBrowser, deferred);
-
-    tab.ownerDocument.defaultView.setTimeout(function()
-    {
-      tab.linkedBrowser.stop();
-    }, this._timeout);
-
-    return deferred.promise;
-  },
-
-  /**
-   * Deactivates this object.
-   */
-  stop: function()
-  {
-    this._browser.removeTabsProgressListener(this);
-  },
-
-  onStateChange: function(browser, progress, request, flags, status)
-  {
-    if ((flags & Ci.nsIWebProgressListener.STATE_STOP) && (flags & Ci.nsIWebProgressListener.STATE_IS_WINDOW))
-    {
-      let deferred = this._deferred.get(browser);
-      if (deferred)
-      {
-        this._deferred.delete(browser);
-
-        let headers = [];
-        if (request instanceof Ci.nsIHttpChannel)
-        {
-          try
-          {
-            headers.push("HTTP/x.x " + request.responseStatus + " " + request.responseStatusText);
-            request.visitResponseHeaders((header, value) => headers.push(header + ": " + value));
-          }
-          catch (e)
-          {
-            // Exceptions are expected here
-          }
-        }
-        deferred.resolve([status, headers]);
-      }
-    }
-  }
-};
-
-/**
  * Once created, this object will make sure all new windows are dismissed
  * immediately.
  *
  * @constructor
  */
 function WindowCloser()
 {
   Services.obs.addObserver(this, "xul-window-registered", true)
 }
 WindowCloser.prototype = {
@@ skipping 73 matching lines @@
  * @param {int} maxtabs
  *    Maximum number of tabs to be opened
  * @param {String} targetURL
  *    URL that should receive the results
  * @param {Function} onDone
  *    The callback which is called after finishing of all tasks.
  */
 function crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)
 {
   let tabAllocator = new TabAllocator(window.getBrowser(), maxtabs);
-  let loadListener = new LoadListener(window.getBrowser(), timeout);
+
   let running = 0;
   let windowCloser = new WindowCloser();
   let taskDone = function()
   {
     running--;
     if (running <= 0)
     {
-      loadListener.stop();
       windowCloser.stop();
       onDone();
     }
   };
 
   for (let url of urls)
   {
     running++;
-    Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(function(result)
+    Task.spawn(crawl_url.bind(null, url, tabAllocator, timeout)).then(function(result)
     {
       let request = new XMLHttpRequest();
       request.open("POST", targetURL);
       request.addEventListener("load", taskDone, false);
       request.addEventListener("error", taskDone, false);
       request.send(JSON.stringify(result));
     }, function(url, exception)
     {
       reportException(exception);
 
       let request = new XMLHttpRequest();
       request.open("POST", targetURL);
       request.addEventListener("load", taskDone, false);
       request.addEventListener("error", taskDone, false);
       request.send(JSON.stringify({
         url: url,
         startTime: Date.now(),
         error: String(exception)
       }));
     }.bind(null, url));
   }
 }
 
 /**
+ * Expects to receive page info gathered in a content process for the specified
+ * `tab`. If there is no relevant message within specified `timeout` then
+ * the result promise is resolve with error object.
+ * @param tab
+ *    Tab in which we are interested in
+ * @param {int} timeout
+ *    Timeout in milliseconds
+ * @return {Promise} promise which will be resolved with the received page info
+ */
+function getPageInfo(tab, timeout)
+{
+  return new Promise((resolve, result) =>
+  {
+    let mm = tab.linkedBrowser.messageManager;
+    let timerID;
+    let onDone = (pageInfo) =>
+    {
+      mm.removeMessageListener("abpcrawler:pageInfoGathered", onDone);
+      clearTimeout(timerID);
+      resolve(pageInfo);
+    }
+    mm.addMessageListener("abpcrawler:pageInfoGathered", (msg) => onDone(msg.data));;

[Wladimir Palant, 2016/09/14 16:11:48]

That's not the callback you are removing above. Also: duplicate semicolon.

[sergei, 2016/09/29 09:58:13]

Fixed. Sorry, overlooked.

+    timerID = setTimeout(onDone.bind(this, {error: "timeout"}), timeout);

[Wladimir Palant, 2016/09/14 16:11:49]

How about not using bind() here, for clarity and consistency?

() => onDone({data: {error: "timeout"}})

[sergei, 2016/09/29 09:58:13]

Done. Good idea.

+  });
+}
+
+/**
  * Crawls a URL. This is a generator meant to be used via a Task object.
  *
  * @param {String} url
  * @param {TabAllocator} tabAllocator
- * @param {loadListener} loadListener
+ * @param {int} timeout
+ *    Load timeout in milliseconds
  * @result {Object}
  *    Crawling result
  */
-function* crawl_url(url, tabAllocator, loadListener)
+function* crawl_url(url, tabAllocator, timeout)
 {
   let tab = yield tabAllocator.getTab();
   let result = {url, requests: []};
   let requestNotifier;
   try
   {
     result.startTime = Date.now();
     requestNotifier = new RequestNotifier(tab.linkedBrowser.outerWindowID,
       function(entry, scanComplete)
     {
       if (!entry)
         return;
       let {type: contentType, location, filter} = entry;
       result.requests.push({location, contentType, filter});
     });
 
     tab.linkedBrowser.loadURI(url, null, null);
-    [result.channelStatus, result.headers] = yield loadListener.waitForLoad(tab);
+
+    Object.assign(result, yield getPageInfo(tab, timeout));
+    result.finalUrl = tab.linkedBrowser.currentURI.spec;
     result.endTime = Date.now();
-    result.finalUrl = tab.linkedBrowser.currentURI.spec;
-
-    let document = tab.linkedBrowser.contentDocument;
-    if (document.documentElement)
-    {
-      try
-      {
-        let canvas = document.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
-        canvas.width = document.documentElement.scrollWidth;
-        canvas.height = document.documentElement.scrollHeight;
-
-        let context = canvas.getContext("2d");
-        context.drawWindow(document.defaultView, 0, 0, canvas.width, canvas.height, "rgb(255, 255, 255)");
-        result.screenshot = canvas.toDataURL("image/jpeg", 0.8);
-      }
-      catch (e)
-      {
-        reportException(e);
-        result.error = "Capturing screenshot failed: " + e;
-      }
-
-      // TODO: Capture frames as well?
-      let serializer = new tab.ownerDocument.defaultView.XMLSerializer();
-      result.source = serializer.serializeToString(document.documentElement);
-    }
   }
   finally
   {
     if (requestNotifier)
       requestNotifier.shutdown();
     tabAllocator.releaseTab(tab);
   }
   return result;
 }
 
 function reportException(e)
 {
   let stack = "";
   if (e && typeof e == "object" && "stack" in e)
     stack = e.stack + "\n";
 
   Cu.reportError(e);
   dump(e + "\n" + stack + "\n");
 }
+
+let {addonRoot} = require("info");
+let frameScriptPath = addonRoot + "/lib/child/frameScript.js";
+let globalMessageManager = Services.mm;
+globalMessageManager.loadFrameScript(frameScriptPath, true);
+
+onShutdown.add(() =>
+{
+  globalMessageManager.removeDelayedFrameScript(frameScriptPath);
+});