| OLD | NEW | 
|---|
| 1 /* | 1 /* | 
| 2  * This Source Code is subject to the terms of the Mozilla Public License | 2  * This Source Code is subject to the terms of the Mozilla Public License | 
| 3  * version 2.0 (the "License"). You can obtain a copy of the License at | 3  * version 2.0 (the "License"). You can obtain a copy of the License at | 
| 4  * http://mozilla.org/MPL/2.0/. | 4  * http://mozilla.org/MPL/2.0/. | 
| 5  */ | 5  */ | 
| 6 | 6 | 
| 7 /** | 7 /** | 
| 8  * @module crawler | 8  * @module crawler | 
| 9  */ | 9  */ | 
| 10 | 10 | 
| 11 Cu.import("resource://gre/modules/Services.jsm"); | 11 Cu.import("resource://gre/modules/Services.jsm"); | 
| 12 Cu.import("resource://gre/modules/Task.jsm"); | 12 Cu.import("resource://gre/modules/Task.jsm"); | 
| 13 Cu.import("resource://gre/modules/Promise.jsm"); | 13 Cu.import("resource://gre/modules/Promise.jsm"); | 
| 14 | 14 | 
| 15 function abprequire(module) | 15 function abprequire(module) | 
| 16 { | 16 { | 
| 17   let result = {}; | 17   let result = {}; | 
| 18   result.wrappedJSObject = result; | 18   result.wrappedJSObject = result; | 
| 19   Services.obs.notifyObservers(result, "adblockplus-require", module); | 19   Services.obs.notifyObservers(result, "adblockplus-require", module); | 
| 20   return result.exports; | 20   return result.exports; | 
| 21 } | 21 } | 
| 22 | 22 | 
| 23 let {RequestNotifier} = abprequire("requestNotifier"); | 23 let {RequestNotifier} = abprequire("requestNotifier"); | 
| 24 let {FilterNotifier} = abprequire("filterNotifier"); | 24 let {FilterNotifier} = abprequire("filterNotifier"); | 
| 25 let {FilterStorage} = abprequire("filterStorage"); | 25 let {FilterStorage} = abprequire("filterStorage"); | 
| 26 | 26 | 
| 27 /** | 27 /** | 
| 28  * Creates a pool of tabs and allocates them to tasks on request. | 28  * Allocates tabs on request but not more than maxtabs at the same time. | 
| 29  * | 29  * | 
| 30  * @param {tabbrowser} browser | 30  * @param {tabbrowser} browser | 
| 31  *    The tabbed browser where tabs should be created | 31  *    The tabbed browser where tabs should be created | 
| 32  * @param {int} maxtabs | 32  * @param {int} maxtabs | 
| 33  *    The maximum number of tabs to be allocated | 33  *    The maximum number of tabs to be allocated | 
| 34  * @constructor | 34  * @constructor | 
| 35  */ | 35  */ | 
| 36 function TabAllocator(browser, maxtabs) | 36 function TabAllocator(browser, maxtabs) | 
| 37 { | 37 { | 
| 38   browser.removeAllTabsBut(browser.tabs[0]) | 38   this._browser = browser; | 
| 39 | 39   this._tabs = 0; | 
| 40   this._tabs = []; | 40   this._maxtabs = maxtabs; | 
| 41   for (let i = 0; i < maxtabs; i++) | 41   // The queue containing resolve functions of promises waiting for a tab. | 
| 42     this._tabs.push(browser.addTab("about:blank")); | 42   this._resolvers = []; | 
| 43 | 43   // Keep at least one tab alive to prevent browser from closing itself. | 
| 44   browser.removeTab(browser.tabs[0]); | 44   this._tabKeepingWindowAlive = this._browser.tabs[0]; | 
| 45 | 45   this._browser.removeAllTabsBut(this._tabKeepingWindowAlive); | 
| 46   this._deferred = []; |  | 
| 47 } | 46 } | 
| 48 TabAllocator.prototype = { | 47 TabAllocator.prototype = { | 
|  | 48   _removeTabKeepingWindowAlive: function() | 
|  | 49   { | 
|  | 50     if (!this._tabKeepingWindowAlive) | 
|  | 51       return; | 
|  | 52     this._browser.removeTab(this._tabKeepingWindowAlive); | 
|  | 53     delete this._tabKeepingWindowAlive; | 
|  | 54   }, | 
|  | 55 | 
| 49   /** | 56   /** | 
| 50    * Returns a promise that will resolve into a tab once a tab can be allocated. | 57    * Creates a blank tab in this._browser. | 
|  | 58    * | 
|  | 59    * @return {Promise.<tab>} promise which resolves once the tab is fully initia
     lized. | 
|  | 60    */ | 
|  | 61   _createTab: function() | 
|  | 62   { | 
|  | 63     this._tabs++; | 
|  | 64     let tab = this._browser.addTab("about:blank"); | 
|  | 65     if (tab.linkedBrowser.outerWindowID) | 
|  | 66     { | 
|  | 67       this._removeTabKeepingWindowAlive(); | 
|  | 68       return Promise.resolve(tab); | 
|  | 69     } | 
|  | 70     return new Promise((resolve, reject) => | 
|  | 71     { | 
|  | 72       let onBrowserInit = (msg) => | 
|  | 73       { | 
|  | 74         tab.linkedBrowser.messageManager.removeMessageListener("Browser:Init", o
     nBrowserInit); | 
|  | 75         this._removeTabKeepingWindowAlive(); | 
|  | 76         resolve(tab); | 
|  | 77       }; | 
|  | 78       // "Browser:Init" message is sent once the browser is ready, see | 
|  | 79       // https://bugzil.la/1256602#c1 | 
|  | 80       tab.linkedBrowser.messageManager.addMessageListener("Browser:Init", onBrow
     serInit); | 
|  | 81     }); | 
|  | 82   }, | 
|  | 83 | 
|  | 84   /** | 
|  | 85    * Returns a promise that will resolve into a tab once a tab is allocated. | 
| 51    * The tab cannot be used by other tasks until releaseTab() is called. | 86    * The tab cannot be used by other tasks until releaseTab() is called. | 
| 52    * | 87    * | 
| 53    * @result {Promise} | 88    * @result {Promise.<tab>} | 
| 54    */ | 89    */ | 
| 55   getTab: function() | 90   getTab: function() | 
| 56   { | 91   { | 
| 57     if (this._tabs.length) | 92     if (this._tabs < this._maxtabs) | 
| 58       return this._tabs.shift(); | 93       return this._createTab(); | 
| 59     else | 94     return new Promise((resolve, reject) => this._resolvers.push(resolve)); | 
| 60     { |  | 
| 61       let deferred = Promise.defer(); |  | 
| 62       this._deferred.push(deferred); |  | 
| 63       return deferred.promise; |  | 
| 64     } |  | 
| 65   }, | 95   }, | 
| 66 | 96 | 
| 67   /** | 97   /** | 
| 68    * Adds a tab back to the pool so that it can be used by other tasks. | 98    * Adds a tab back to the pool so that it can be used by other tasks. | 
| 69    * | 99    * | 
| 70    * @param {tab} tab | 100    * @param {tab} tab | 
| 71    */ | 101    */ | 
| 72   releaseTab: function(tab) | 102   releaseTab: function(tab) | 
| 73   { | 103   { | 
| 74     let browser = tab.parentNode.tabbrowser; | 104     // If we are about to close last tab don't close it immediately to keep | 
| 75     browser.removeTab(tab); | 105     // the window alive. It will be closed when a new tab is created. | 
| 76     tab = browser.addTab("about:blank"); | 106     if (this._tabs > 1) | 
|  | 107       this._browser.removeTab(tab); | 
|  | 108     else | 
|  | 109     { | 
|  | 110       // navigate away from early opened URL | 
|  | 111       tab.linkedBrowser.loadURI('about:blank', null, null); | 
|  | 112       this._tabKeepingWindowAlive = tab; | 
|  | 113     } | 
| 77 | 114 | 
| 78     if (this._deferred.length) | 115     this._tabs--; | 
| 79       this._deferred.shift().resolve(tab); | 116     if (this._resolvers.length && this._tabs < this._maxtabs) | 
| 80     else | 117     { | 
| 81       this._tabs.push(tab); | 118       this._resolvers.shift()(this._createTab()); | 
| 82   } | 119     } | 
|  | 120   }, | 
| 83 }; | 121 }; | 
| 84 | 122 | 
| 85 /** | 123 /** | 
| 86  * Observes page loads in a particular tabbed browser. | 124  * Observes page loads in a particular tabbed browser. | 
| 87  * | 125  * | 
| 88  * @param {tabbrowser} browser | 126  * @param {tabbrowser} browser | 
| 89  *    The tabbed browser to be observed | 127  *    The tabbed browser to be observed | 
| 90  * @param {int} timeout | 128  * @param {int} timeout | 
| 91  *    Load timeout in milliseconds | 129  *    Load timeout in milliseconds | 
| 92  * @constructor | 130  * @constructor | 
| (...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 224         resolve(); | 262         resolve(); | 
| 225       } | 263       } | 
| 226     }; | 264     }; | 
| 227     FilterNotifier.addListener(onFiltersLoaded); | 265     FilterNotifier.addListener(onFiltersLoaded); | 
| 228   }).then(() => crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)) | 266   }).then(() => crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)) | 
| 229   .catch(reportException); | 267   .catch(reportException); | 
| 230 } | 268 } | 
| 231 exports.run = run; | 269 exports.run = run; | 
| 232 | 270 | 
| 233 /** | 271 /** | 
| 234  * Spawns a {Task} task to crawl each url from `urls` argument and calls | 272  * Spawns a {Task} task to crawl each url from urls argument and calls | 
| 235  * `onDone` when all tasks are finished. | 273  * onDone when all tasks are finished. | 
| 236  * @param {Window} window | 274  * @param {Window} window | 
| 237  *   The browser window we're operating in | 275  *   The browser window we're operating in | 
| 238  * @param {String[]} urls | 276  * @param {String[]} urls | 
| 239  *   URLs to be crawled | 277  *   URLs to be crawled | 
| 240  * @param {int} timeout | 278  * @param {int} timeout | 
| 241  *    Load timeout in milliseconds | 279  *    Load timeout in milliseconds | 
| 242  * @param {int} maxtabs | 280  * @param {int} maxtabs | 
| 243  *    Maximum number of tabs to be opened | 281  *    Maximum number of tabs to be opened | 
| 244  * @param {String} targetURL | 282  * @param {String} targetURL | 
| 245  *    URL that should receive the results | 283  *    URL that should receive the results | 
| (...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 356 | 394 | 
| 357 function reportException(e) | 395 function reportException(e) | 
| 358 { | 396 { | 
| 359   let stack = ""; | 397   let stack = ""; | 
| 360   if (e && typeof e == "object" && "stack" in e) | 398   if (e && typeof e == "object" && "stack" in e) | 
| 361     stack = e.stack + "\n"; | 399     stack = e.stack + "\n"; | 
| 362 | 400 | 
| 363   Cu.reportError(e); | 401   Cu.reportError(e); | 
| 364   dump(e + "\n" + stack + "\n"); | 402   dump(e + "\n" + stack + "\n"); | 
| 365 } | 403 } | 
| OLD | NEW | 
|---|