Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: lib/crawler.js

Issue 29338153: Issue 3780 - wait for the loading of filters and only afterwards start to fetch pages (Closed)
Left Patch Set: move .catch() on the same line with .then() Created March 15, 2016, 2:33 p.m.
Right Patch Set: remove onFiltersLoaded listener and fit 80 chars Created March 15, 2016, 2:57 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « no previous file | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 /* 1 /*
2 * This Source Code is subject to the terms of the Mozilla Public License 2 * This Source Code is subject to the terms of the Mozilla Public License
3 * version 2.0 (the "License"). You can obtain a copy of the License at 3 * version 2.0 (the "License"). You can obtain a copy of the License at
4 * http://mozilla.org/MPL/2.0/. 4 * http://mozilla.org/MPL/2.0/.
5 */ 5 */
6 6
7 /** 7 /**
8 * @module crawler 8 * @module crawler
9 */ 9 */
10 10
11 Cu.import("resource://gre/modules/Services.jsm"); 11 Cu.import("resource://gre/modules/Services.jsm");
12 Cu.import("resource://gre/modules/Task.jsm"); 12 Cu.import("resource://gre/modules/Task.jsm");
13 Cu.import("resource://gre/modules/Promise.jsm"); 13 Cu.import("resource://gre/modules/Promise.jsm");
14 14
15 function abprequire(module) 15 function abprequire(module)
16 { 16 {
17 let result = {}; 17 let result = {};
18 result.wrappedJSObject = result; 18 result.wrappedJSObject = result;
19 Services.obs.notifyObservers(result, "adblockplus-require", module); 19 Services.obs.notifyObservers(result, "adblockplus-require", module);
20 return result.exports; 20 return result.exports;
21 } 21 }
22 22
23 let {RequestNotifier} = abprequire("requestNotifier"); 23 let {RequestNotifier} = abprequire("requestNotifier");
24
25 let {FilterNotifier} = abprequire("filterNotifier"); 24 let {FilterNotifier} = abprequire("filterNotifier");
26 let {FilterStorage} = abprequire("filterStorage"); 25 let {FilterStorage} = abprequire("filterStorage");
27 26
28 /** 27 /**
29 * Creates a pool of tabs and allocates them to tasks on request. 28 * Creates a pool of tabs and allocates them to tasks on request.
30 * 29 *
31 * @param {tabbrowser} browser 30 * @param {tabbrowser} browser
32 * The tabbed browser where tabs should be created 31 * The tabbed browser where tabs should be created
33 * @param {int} maxtabs 32 * @param {int} maxtabs
34 * The maximum number of tabs to be allocated 33 * The maximum number of tabs to be allocated
(...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after
192 }; 191 };
193 192
194 /** 193 /**
195 * Starts the crawling session. The crawler opens each URL in a tab and stores 194 * Starts the crawling session. The crawler opens each URL in a tab and stores
196 * the results. 195 * the results.
197 * 196 *
198 * @param {Window} window 197 * @param {Window} window
199 * The browser window we're operating in 198 * The browser window we're operating in
200 * @param {String[]} urls 199 * @param {String[]} urls
201 * URLs to be crawled 200 * URLs to be crawled
202 * @param {int} number_of_tabs 201 * @param {int} timeout
202 * Load timeout in milliseconds
203 * @param {int} maxtabs
203 * Maximum number of tabs to be opened 204 * Maximum number of tabs to be opened
204 * @param {String} targetURL 205 * @param {String} targetURL
205 * URL that should receive the results 206 * URL that should receive the results
207 * @param {Function} onDone
208 * The callback which is called after finishing of crawling of all URLs.
206 */ 209 */
207 function run(window, urls, timeout, maxtabs, targetURL, onDone) 210 function run(window, urls, timeout, maxtabs, targetURL, onDone)
211 {
212 new Promise((resolve, reject) =>
213 {
214 if (FilterStorage.subscriptions.length > 0)
215 {
216 resolve();
217 return;
218 }
219 let onFiltersLoaded = (action, item, newValue, oldValue) =>
220 {
221 if (action == "load")
222 {
223 FilterNotifier.removeListener(onFiltersLoaded);
224 resolve();
225 }
226 };
227 FilterNotifier.addListener(onFiltersLoaded);
228 }).then(() => crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone))
229 .catch(reportException);
230 }
231 exports.run = run;
232
233 /**
234 * Spawns a {Task} task to crawl each url from `urls` argument and calls
235 * `onDone` when all tasks are finished.
236 * @param {Window} window
237 * The browser window we're operating in
238 * @param {String[]} urls
239 * URLs to be crawled
240 * @param {int} timeout
241 * Load timeout in milliseconds
242 * @param {int} maxtabs
243 * Maximum number of tabs to be opened
244 * @param {String} targetURL
245 * URL that should receive the results
246 * @param {Function} onDone
247 * The callback which is called after finishing of all tasks.
248 */
249 function crawl_urls(window, urls, timeout, maxtabs, targetURL, onDone)
208 { 250 {
209 let tabAllocator = new TabAllocator(window.getBrowser(), maxtabs); 251 let tabAllocator = new TabAllocator(window.getBrowser(), maxtabs);
210 let loadListener = new LoadListener(window.getBrowser(), timeout); 252 let loadListener = new LoadListener(window.getBrowser(), timeout);
211 let running = 0; 253 let running = 0;
212 let windowCloser = new WindowCloser(); 254 let windowCloser = new WindowCloser();
213 let taskDone = function() 255 let taskDone = function()
214 { 256 {
215 running--; 257 running--;
216 if (running <= 0) 258 if (running <= 0)
217 { 259 {
218 loadListener.stop(); 260 loadListener.stop();
219 windowCloser.stop(); 261 windowCloser.stop();
220 onDone(); 262 onDone();
221 } 263 }
222 }; 264 };
223 265
224 let crawl_urls = () => 266 for (let url of urls)
225 { 267 {
226 for (let url of urls) 268 running++;
227 { 269 Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(funct ion(result)
228 running++; 270 {
229 Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(fun ction(result) 271 let request = new XMLHttpRequest();
230 { 272 request.open("POST", targetURL);
231 let request = new XMLHttpRequest(); 273 request.addEventListener("load", taskDone, false);
232 request.open("POST", targetURL); 274 request.addEventListener("error", taskDone, false);
233 request.addEventListener("load", taskDone, false); 275 request.send(JSON.stringify(result));
234 request.addEventListener("error", taskDone, false); 276 }, function(url, exception)
235 request.send(JSON.stringify(result)); 277 {
236 }, function(url, exception) 278 reportException(exception);
237 { 279
238 reportException(exception); 280 let request = new XMLHttpRequest();
239 281 request.open("POST", targetURL);
240 let request = new XMLHttpRequest(); 282 request.addEventListener("load", taskDone, false);
241 request.open("POST", targetURL); 283 request.addEventListener("error", taskDone, false);
242 request.addEventListener("load", taskDone, false); 284 request.send(JSON.stringify({
243 request.addEventListener("error", taskDone, false); 285 url: url,
244 request.send(JSON.stringify({ 286 startTime: Date.now(),
245 url: url, 287 error: String(exception)
246 startTime: Date.now(), 288 }));
247 error: String(exception) 289 }.bind(null, url));
248 })); 290 }
249 }.bind(null, url)); 291 }
250 }
251 };
252 new Promise((resolve, reject) =>
253 {
254 if (FilterStorage.subscriptions.length > 0 && !FilterStorage._loading)
255 {
256 resolve();
257 return;
258 }
259 FilterNotifier.addListener((action, item, newValue, oldValue) =>
260 {
261 if (action == "load")
262 {
263 resolve();
264 }
265 });
266 }).then(crawl_urls).catch(reportException);
267 }
268 exports.run = run;
269 292
270 /** 293 /**
271 * Crawls a URL. This is a generator meant to be used via a Task object. 294 * Crawls a URL. This is a generator meant to be used via a Task object.
272 * 295 *
273 * @param {String} url 296 * @param {String} url
274 * @param {TabAllocator} tabAllocator 297 * @param {TabAllocator} tabAllocator
275 * @param {loadListener} loadListener 298 * @param {loadListener} loadListener
276 * @result {Object} 299 * @result {Object}
277 * Crawling result 300 * Crawling result
278 */ 301 */
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after
333 356
334 function reportException(e) 357 function reportException(e)
335 { 358 {
336 let stack = ""; 359 let stack = "";
337 if (e && typeof e == "object" && "stack" in e) 360 if (e && typeof e == "object" && "stack" in e)
338 stack = e.stack + "\n"; 361 stack = e.stack + "\n";
339 362
340 Cu.reportError(e); 363 Cu.reportError(e);
341 dump(e + "\n" + stack + "\n"); 364 dump(e + "\n" + stack + "\n");
342 } 365 }
LEFTRIGHT
« no previous file | no next file » | Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Toggle Comments ('s')

Powered by Google App Engine
This is Rietveld