| OLD | NEW |
| 1 /* | 1 /* |
| 2 * This Source Code is subject to the terms of the Mozilla Public License | 2 * This Source Code is subject to the terms of the Mozilla Public License |
| 3 * version 2.0 (the "License"). You can obtain a copy of the License at | 3 * version 2.0 (the "License"). You can obtain a copy of the License at |
| 4 * http://mozilla.org/MPL/2.0/. | 4 * http://mozilla.org/MPL/2.0/. |
| 5 */ | 5 */ |
| 6 | 6 |
| 7 /** | 7 /** |
| 8 * @module crawler | 8 * @module crawler |
| 9 */ | 9 */ |
| 10 | 10 |
| 11 Cu.import("resource://gre/modules/Services.jsm"); | 11 Cu.import("resource://gre/modules/Services.jsm"); |
| 12 Cu.import("resource://gre/modules/Task.jsm"); | 12 Cu.import("resource://gre/modules/Task.jsm"); |
| 13 Cu.import("resource://gre/modules/Promise.jsm"); | 13 Cu.import("resource://gre/modules/Promise.jsm"); |
| 14 | 14 |
| 15 function abprequire(module) | 15 function abprequire(module) |
| 16 { | 16 { |
| 17 let result = {}; | 17 let result = {}; |
| 18 result.wrappedJSObject = result; | 18 result.wrappedJSObject = result; |
| 19 Services.obs.notifyObservers(result, "adblockplus-require", module); | 19 Services.obs.notifyObservers(result, "adblockplus-require", module); |
| 20 return result.exports; | 20 return result.exports; |
| 21 } | 21 } |
| 22 | 22 |
| 23 let {RequestNotifier} = abprequire("requestNotifier"); | 23 let {RequestNotifier} = abprequire("requestNotifier"); |
| 24 let {FilterNotifier} = abprequire("filterNotifier"); | 24 let {FilterNotifier} = abprequire("filterNotifier"); |
| 25 let {FilterStorage} = abprequire("filterStorage"); | 25 let {FilterStorage} = abprequire("filterStorage"); |
| 26 | 26 |
| 27 /** | 27 /** |
| 28 * Creates a pool of tabs and allocates them to tasks on request. | 28 * Allocates tabs on request but not more than maxtabs at the same time. |
| 29 * | 29 * |
| 30 * @param {tabbrowser} browser | 30 * @param {tabbrowser} browser |
| 31 * The tabbed browser where tabs should be created | 31 * The tabbed browser where tabs should be created |
| 32 * @param {int} maxtabs | 32 * @param {int} maxtabs |
| 33 * The maximum number of tabs to be allocated | 33 * The maximum number of tabs to be allocated |
| 34 * @constructor | 34 * @constructor |
| 35 */ | 35 */ |
| 36 function TabAllocator(browser, maxtabs) | 36 function TabAllocator(browser, maxtabs) |
| 37 { | 37 { |
| 38 browser.removeAllTabsBut(browser.tabs[0]) | 38 this._browser = browser; |
| 39 | 39 this._tabs = 0; |
| 40 this._tabs = []; | 40 this._maxtabs = maxtabs; |
| 41 for (let i = 0; i < maxtabs; i++) | 41 // The queue containing resolve functions of promises waiting for a tab. |
| 42 this._tabs.push(browser.addTab("about:blank")); | 42 this._resolvers = []; |
| 43 | 43 // Keep at least one tab alive to prevent browser from closing itself. |
| 44 browser.removeTab(browser.tabs[0]); | 44 let tabToRemove = this._browser.tabs[0]; |
| 45 | 45 this._browser.removeAllTabsBut(tabToRemove); |
| 46 this._deferred = []; | 46 // this._tab is a keep alive tab |
| 47 this._tab = this._createTab().then(tab => |
| 48 { |
| 49 // Starting from Firefox 48 (nightly) the sequence of calls addTab and |
| 50 // removeTab can cause a closing of the browser because a new tab is still |
| 51 // not here. Because of that we need to remove the previous tab only after |
| 52 // the new tab is ready. |
| 53 this._browser.removeTab(tabToRemove); |
| 54 return tab; |
| 55 }); |
| 47 } | 56 } |
| 48 TabAllocator.prototype = { | 57 TabAllocator.prototype = { |
| 49 /** | 58 /** |
| 50 * Returns a promise that will resolve into a tab once a tab can be allocated. | 59 * Creates a blank tab in this._browser. |
| 60 * |
| 61 * @return {Promise.<tab>} promise which resolves once the tab is fully initia
lized. |
| 62 */ |
| 63 _createTab: function() |
| 64 { |
| 65 this._tabs++; |
| 66 let tab = this._browser.addTab("about:blank"); |
| 67 if (tab.linkedBrowser.outerWindowID) |
| 68 return Promise.resolve(tab); |
| 69 return new Promise((resolve, reject) => |
| 70 { |
| 71 let onBrowserInit = (msg) => |
| 72 { |
| 73 tab.linkedBrowser.messageManager.removeMessageListener("Browser:Init", o
nBrowserInit); |
| 74 resolve(tab); |
| 75 }; |
| 76 // "Browser:Init" message is sent once the browser is ready, see |
| 77 // https://bugzil.la/1256602#c1 |
| 78 tab.linkedBrowser.messageManager.addMessageListener("Browser:Init", onBrow
serInit); |
| 79 }); |
| 80 }, |
| 81 |
| 82 /** |
| 83 * Returns a promise that will resolve into a tab once a tab is allocated. |
| 51 * The tab cannot be used by other tasks until releaseTab() is called. | 84 * The tab cannot be used by other tasks until releaseTab() is called. |
| 52 * | 85 * |
| 53 * @result {Promise} | 86 * @result {Promise.<tab>} |
| 54 */ | 87 */ |
| 55 getTab: function() | 88 getTab: function() |
| 56 { | 89 { |
| 57 if (this._tabs.length) | 90 if (this._tab) |
| 58 return this._tabs.shift(); | |
| 59 else | |
| 60 { | 91 { |
| 61 let deferred = Promise.defer(); | 92 let tab = this._tab; |
| 62 this._deferred.push(deferred); | 93 delete this._tab; |
| 63 return deferred.promise; | 94 return tab; |
| 64 } | 95 } |
| 96 if (this._tabs < this._maxtabs) |
| 97 return this._createTab(); |
| 98 return new Promise((resolve, reject) => this._resolvers.push(resolve)); |
| 65 }, | 99 }, |
| 66 | 100 |
| 67 /** | 101 /** |
| 68 * Adds a tab back to the pool so that it can be used by other tasks. | 102 * Adds a tab back to the pool so that it can be used by other tasks. |
| 69 * | 103 * |
| 70 * @param {tab} tab | 104 * @param {tab} tab |
| 71 */ | 105 */ |
| 72 releaseTab: function(tab) | 106 releaseTab: function(tab) |
| 73 { | 107 { |
| 74 let browser = tab.parentNode.tabbrowser; | 108 // If we are about to close last tab don't close it immediately rather |
| 75 browser.removeTab(tab); | 109 // allocate a new blank tab and close the current one afterwards. |
| 76 tab = browser.addTab("about:blank"); | 110 if (this._tabs == 1) |
| 111 { |
| 112 this._tab = this._createTab().then((resultTab) => |
| 113 { |
| 114 this.releaseTab(tab); |
| 115 return resultTab; |
| 116 }); |
| 117 return; |
| 118 } |
| 77 | 119 |
| 78 if (this._deferred.length) | 120 this._browser.removeTab(tab); |
| 79 this._deferred.shift().resolve(tab); | 121 this._tabs--; |
| 80 else | 122 if (this._resolvers.length) |
| 81 this._tabs.push(tab); | 123 { |
| 82 } | 124 if (this._tab) |
| 125 { |
| 126 this._resolvers.shift()(this._tab); |
| 127 delete this._tab; |
| 128 } |
| 129 else if (this._tabs < this._maxtabs) |
| 130 { |
| 131 this._resolvers.shift()(this._createTab()); |
| 132 } |
| 133 } |
| 134 }, |
| 83 }; | 135 }; |
| 84 | 136 |
| 85 /** | 137 /** |
| 86 * Observes page loads in a particular tabbed browser. | 138 * Observes page loads in a particular tabbed browser. |
| 87 * | 139 * |
| 88 * @param {tabbrowser} browser | 140 * @param {tabbrowser} browser |
| 89 * The tabbed browser to be observed | 141 * The tabbed browser to be observed |
| 90 * @param {int} timeout | 142 * @param {int} timeout |
| 91 * Load timeout in milliseconds | 143 * Load timeout in milliseconds |
| 92 * @constructor | 144 * @constructor |
| (...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 224 resolve(); | 276 resolve(); |
| 225 } | 277 } |
| 226 }; | 278 }; |
| 227 FilterNotifier.addListener(onFiltersLoaded); | 279 FilterNotifier.addListener(onFiltersLoaded); |
| 228 }).then(() => crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)) | 280 }).then(() => crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)) |
| 229 .catch(reportException); | 281 .catch(reportException); |
| 230 } | 282 } |
| 231 exports.run = run; | 283 exports.run = run; |
| 232 | 284 |
| 233 /** | 285 /** |
| 234 * Spawns a {Task} task to crawl each url from `urls` argument and calls | 286 * Spawns a {Task} task to crawl each url from urls argument and calls |
| 235 * `onDone` when all tasks are finished. | 287 * onDone when all tasks are finished. |
| 236 * @param {Window} window | 288 * @param {Window} window |
| 237 * The browser window we're operating in | 289 * The browser window we're operating in |
| 238 * @param {String[]} urls | 290 * @param {String[]} urls |
| 239 * URLs to be crawled | 291 * URLs to be crawled |
| 240 * @param {int} timeout | 292 * @param {int} timeout |
| 241 * Load timeout in milliseconds | 293 * Load timeout in milliseconds |
| 242 * @param {int} maxtabs | 294 * @param {int} maxtabs |
| 243 * Maximum number of tabs to be opened | 295 * Maximum number of tabs to be opened |
| 244 * @param {String} targetURL | 296 * @param {String} targetURL |
| 245 * URL that should receive the results | 297 * URL that should receive the results |
| (...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 356 | 408 |
| 357 function reportException(e) | 409 function reportException(e) |
| 358 { | 410 { |
| 359 let stack = ""; | 411 let stack = ""; |
| 360 if (e && typeof e == "object" && "stack" in e) | 412 if (e && typeof e == "object" && "stack" in e) |
| 361 stack = e.stack + "\n"; | 413 stack = e.stack + "\n"; |
| 362 | 414 |
| 363 Cu.reportError(e); | 415 Cu.reportError(e); |
| 364 dump(e + "\n" + stack + "\n"); | 416 dump(e + "\n" + stack + "\n"); |
| 365 } | 417 } |
| OLD | NEW |