Issue 3792 - Fix to support multiprocess firefox

lib/crawler.js

 /*
  * This Source Code is subject to the terms of the Mozilla Public License
  * version 2.0 (the "License"). You can obtain a copy of the License at
  * http://mozilla.org/MPL/2.0/.
  */
 
+"use strict";
+
 /**
  * @module crawler
  */
 
-Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/Task.jsm");
-Cu.import("resource://gre/modules/Promise.jsm");

[Wladimir Palant, 2016/09/14 16:13:18]

Just realized that this import hasn't been removed - please do so, we want to
use the default Promise object.

-Cu.import("resource://gre/modules/Timer.jsm");

[Wladimir Palant, 2016/09/14 16:13:18]

Nit: We don't import like that any more, symbols should be imported explicitly.

  let {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm",
{});

[sergei, 2016/09/29 09:58:14]

Fixed and the rest related to coding style is addressed in codereview
https://codereview.adblockplus.org/29355247/

+const {Services} = Cu.import("resource://gre/modules/Services.jsm", {});
+const {XPCOMUtils} = Cu.import("resource://gre/modules/XPCOMUtils.jsm", {});
+const {Task} = Cu.import("resource://gre/modules/Task.jsm", {});
+const {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm", {});
 
 function abprequire(module)
 {
   let result = {};
   result.wrappedJSObject = result;
   Services.obs.notifyObservers(result, "adblockplus-require", module);
   return result.exports;
 }
 
 let {RequestNotifier} = abprequire("requestNotifier");
@@ skipping 10 matching lines @@
  * @constructor
  */
 function TabAllocator(browser, maxtabs)
 {
   this._browser = browser;
   this._tabs = 0;
   this._maxtabs = maxtabs;
   // The queue containing resolve functions of promises waiting for a tab.
   this._resolvers = [];
   // Keep at least one tab alive to prevent browser from closing itself.
-  let tabToRemove = this._browser.tabs[0];
-  this._browser.removeAllTabsBut(tabToRemove);
-  // this._tab is a keep alive tab
-  this._tab = this._createTab().then(tab =>
-  {
-    // Starting from Firefox 48 (nightly) the sequence of calls addTab and
-    // removeTab can cause a closing of the browser because a new tab is still
-    // not here. Because of that we need to remove the previous tab only after
-    // the new tab is ready.
-    this._browser.removeTab(tabToRemove);
-    return tab;
-  });
+  this._tabKeepingWindowAlive = this._browser.tabs[0];
+  this._browser.removeAllTabsBut(this._tabKeepingWindowAlive);
 }
 TabAllocator.prototype = {
+  _removeTabKeepingWindowAlive: function()
+  {
+    if (!this._tabKeepingWindowAlive)
+      return;
+    this._browser.removeTab(this._tabKeepingWindowAlive);
+    delete this._tabKeepingWindowAlive;
+  },
+
   /**
    * Creates a blank tab in this._browser.
    *
    * @return {Promise.<tab>} promise which resolves once the tab is fully initialized.
    */
   _createTab: function()
   {
     this._tabs++;
     let tab = this._browser.addTab("about:blank");
     if (tab.linkedBrowser.outerWindowID)
+    {
+      this._removeTabKeepingWindowAlive();
       return Promise.resolve(tab);
+    }
     return new Promise((resolve, reject) =>
     {
       let onBrowserInit = (msg) =>
       {
-        // https://bugzilla.mozilla.org/show_bug.cgi?id=1256602#c1
         tab.linkedBrowser.messageManager.removeMessageListener("Browser:Init", onBrowserInit);
+        this._removeTabKeepingWindowAlive();
         resolve(tab);
       };
+      // "Browser:Init" message is sent once the browser is ready, see
+      // https://bugzil.la/1256602#c1
       tab.linkedBrowser.messageManager.addMessageListener("Browser:Init", onBrowserInit);
     });
   },
 
   /**
    * Returns a promise that will resolve into a tab once a tab is allocated.
    * The tab cannot be used by other tasks until releaseTab() is called.
    *
    * @result {Promise.<tab>}
    */
   getTab: function()
   {
-    if (this._tab)
-    {
-      let tab = this._tab;
-      delete this._tab;
-      return tab;
-    }
     if (this._tabs < this._maxtabs)
       return this._createTab();
     return new Promise((resolve, reject) => this._resolvers.push(resolve));
   },
 
   /**
    * Adds a tab back to the pool so that it can be used by other tasks.
    *
    * @param {tab} tab
    */
   releaseTab: function(tab)
   {
-    // If we are about to close last tab don't close it immediately rather
-    // allocate a new blank tab and close the current one afterwards.
-    if (this._tabs == 1)
-    {
-      this._tab = this._createTab().then((resultTab) =>
-      {
-        this.releaseTab(tab);
-        return resultTab;
-      });
-      return;
-    }
-
-    this._browser.removeTab(tab);
+    // If we are about to close last tab don't close it immediately to keep
+    // the window alive. It will be closed when a new tab is created.
+    if (this._tabs > 1)
+      this._browser.removeTab(tab);
+    else
+    {
+      // navigate away from previously opened URL
+      tab.linkedBrowser.loadURI("about:blank", null, null);
+      this._tabKeepingWindowAlive = tab;
+    }
+
     this._tabs--;
-    if (this._resolvers.length)
-    {
-      if (this._tab)
-      {
-        this._resolvers.shift()(this._tab);
-        delete this._tab;
-      }
-      else if (this._tabs < this._maxtabs)
-      {
-        this._resolvers.shift()(this._createTab());
-      }
+    if (this._resolvers.length && this._tabs < this._maxtabs)
+    {
+      this._resolvers.shift()(this._createTab());
     }
   },
 };
 
 /**
  * Once created, this object will make sure all new windows are dismissed
  * immediately.
  *
  * @constructor
  */
@@ skipping 18 matching lines @@
     {
       if (window.document.documentElement.localName == 'dialog')
         window.document.documentElement.acceptDialog();
       else
         window.close();
     }, false);
   },
 
   QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver, Ci.nsISupportsWeakReference])
 };
+
+function configureFrameScript()
+{
+  const info = require("info");
+  let frameScriptPath = info.addonRoot + "/lib/child/frameScript.js";
+  Services.mm.loadFrameScript(frameScriptPath, true);
+
+  onShutdown.add(() =>
+  {
+    Services.mm.removeDelayedFrameScript(frameScriptPath);
+  });
+}
 
 /**
  * Starts the crawling session. The crawler opens each URL in a tab and stores
  * the results.
  *
  * @param {Window} window
  *    The browser window we're operating in
  * @param {String[]} urls
  *    URLs to be crawled
  * @param {int} timeout
  *    Load timeout in milliseconds
  * @param {int} maxtabs
  *    Maximum number of tabs to be opened
  * @param {String} targetURL
  *    URL that should receive the results
  * @param {Function} onDone
  *    The callback which is called after finishing of crawling of all URLs.
  */
 function run(window, urls, timeout, maxtabs, targetURL, onDone)
 {
+  configureFrameScript();
   new Promise((resolve, reject) =>
   {
     if (FilterStorage.subscriptions.length > 0)
     {
       resolve();
       return;
     }
     let onFiltersLoaded = (action, item, newValue, oldValue) =>
     {
       if (action == "load")
@@ skipping 63 matching lines @@
         startTime: Date.now(),
         error: String(exception)
       }));
     }.bind(null, url));
   }
 }
 
 /**
  * Expects to receive page info gathered in a content process for the specified
  * `tab`. If there is no relevant message within specified `timeout` then
- * the result promise is resolve with error object.
+ * the result promise is resolved with error object.
  * @param tab
  *    Tab in which we are interested in
  * @param {int} timeout
  *    Timeout in milliseconds
  * @return {Promise} promise which will be resolved with the received page info
  */
 function getPageInfo(tab, timeout)
 {
   return new Promise((resolve, result) =>
   {
     let mm = tab.linkedBrowser.messageManager;
     let timerID;
-    let onDone = (pageInfo) =>
+    let onDone = (msg) =>
     {
       mm.removeMessageListener("abpcrawler:pageInfoGathered", onDone);
       clearTimeout(timerID);
-      resolve(pageInfo);
-    }
-    mm.addMessageListener("abpcrawler:pageInfoGathered", (msg) => onDone(msg.data));;

[Wladimir Palant, 2016/09/14 16:11:48]

That's not the callback you are removing above. Also: duplicate semicolon.

[sergei, 2016/09/29 09:58:13]

Fixed. Sorry, overlooked.

-    timerID = setTimeout(onDone.bind(this, {error: "timeout"}), timeout);

[Wladimir Palant, 2016/09/14 16:11:49]

How about not using bind() here, for clarity and consistency?

() => onDone({data: {error: "timeout"}})

[sergei, 2016/09/29 09:58:13]

Done. Good idea.

+      resolve(msg.data);
+    }
+    mm.addMessageListener("abpcrawler:pageInfoGathered", onDone);
+    timerID = setTimeout(() => onDone({data: {error: "timeout"}}), timeout);
   });
 }
 
 /**
  * Crawls a URL. This is a generator meant to be used via a Task object.
  *
  * @param {String} url
  * @param {TabAllocator} tabAllocator
  * @param {int} timeout
  *    Load timeout in milliseconds
@@ skipping 34 matching lines @@
 
 function reportException(e)
 {
   let stack = "";
   if (e && typeof e == "object" && "stack" in e)
     stack = e.stack + "\n";
 
   Cu.reportError(e);
   dump(e + "\n" + stack + "\n");
 }
-
-let {addonRoot} = require("info");
-let frameScriptPath = addonRoot + "/lib/child/frameScript.js";
-let globalMessageManager = Services.mm;
-globalMessageManager.loadFrameScript(frameScriptPath, true);
-
-onShutdown.add(() =>
-{
-  globalMessageManager.removeDelayedFrameScript(frameScriptPath);
-});