Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: lib/crawler.js

Issue 29338153: Issue 3780 - wait for the loading of filters and only afterwards start to fetch pages (Closed)
Patch Set: move .catch() on the same line with .then() Created March 15, 2016, 2:33 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* 1 /*
2 * This Source Code is subject to the terms of the Mozilla Public License 2 * This Source Code is subject to the terms of the Mozilla Public License
3 * version 2.0 (the "License"). You can obtain a copy of the License at 3 * version 2.0 (the "License"). You can obtain a copy of the License at
4 * http://mozilla.org/MPL/2.0/. 4 * http://mozilla.org/MPL/2.0/.
5 */ 5 */
6 6
7 /** 7 /**
8 * @module crawler 8 * @module crawler
9 */ 9 */
10 10
11 Cu.import("resource://gre/modules/Services.jsm"); 11 Cu.import("resource://gre/modules/Services.jsm");
12 Cu.import("resource://gre/modules/Task.jsm"); 12 Cu.import("resource://gre/modules/Task.jsm");
13 Cu.import("resource://gre/modules/Promise.jsm"); 13 Cu.import("resource://gre/modules/Promise.jsm");
14 14
15 function abprequire(module) 15 function abprequire(module)
16 { 16 {
17 let result = {}; 17 let result = {};
18 result.wrappedJSObject = result; 18 result.wrappedJSObject = result;
19 Services.obs.notifyObservers(result, "adblockplus-require", module); 19 Services.obs.notifyObservers(result, "adblockplus-require", module);
20 return result.exports; 20 return result.exports;
21 } 21 }
22 22
23 let {RequestNotifier} = abprequire("requestNotifier"); 23 let {RequestNotifier} = abprequire("requestNotifier");
24 24
25 let {FilterNotifier} = abprequire("filterNotifier");
26 let {FilterStorage} = abprequire("filterStorage");
25 27
26 /** 28 /**
27 * Creates a pool of tabs and allocates them to tasks on request. 29 * Creates a pool of tabs and allocates them to tasks on request.
28 * 30 *
29 * @param {tabbrowser} browser 31 * @param {tabbrowser} browser
30 * The tabbed browser where tabs should be created 32 * The tabbed browser where tabs should be created
31 * @param {int} maxtabs 33 * @param {int} maxtabs
32 * The maximum number of tabs to be allocated 34 * The maximum number of tabs to be allocated
33 * @constructor 35 * @constructor
34 */ 36 */
(...skipping 177 matching lines...) Expand 10 before | Expand all | Expand 10 after
212 { 214 {
213 running--; 215 running--;
214 if (running <= 0) 216 if (running <= 0)
215 { 217 {
216 loadListener.stop(); 218 loadListener.stop();
217 windowCloser.stop(); 219 windowCloser.stop();
218 onDone(); 220 onDone();
219 } 221 }
220 }; 222 };
221 223
222 for (let url of urls) 224 let crawl_urls = () =>
223 { 225 {
224 running++; 226 for (let url of urls)
225 Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(funct ion(result)
226 { 227 {
227 let request = new XMLHttpRequest(); 228 running++;
228 request.open("POST", targetURL); 229 Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(fun ction(result)
229 request.addEventListener("load", taskDone, false); 230 {
230 request.addEventListener("error", taskDone, false); 231 let request = new XMLHttpRequest();
231 request.send(JSON.stringify(result)); 232 request.open("POST", targetURL);
232 }, function(url, exception) 233 request.addEventListener("load", taskDone, false);
234 request.addEventListener("error", taskDone, false);
235 request.send(JSON.stringify(result));
236 }, function(url, exception)
237 {
238 reportException(exception);
239
240 let request = new XMLHttpRequest();
241 request.open("POST", targetURL);
242 request.addEventListener("load", taskDone, false);
243 request.addEventListener("error", taskDone, false);
244 request.send(JSON.stringify({
245 url: url,
246 startTime: Date.now(),
247 error: String(exception)
248 }));
249 }.bind(null, url));
250 }
251 };
252 new Promise((resolve, reject) =>
253 {
254 if (FilterStorage.subscriptions.length > 0 && !FilterStorage._loading)
233 { 255 {
234 reportException(exception); 256 resolve();
235 257 return;
236 let request = new XMLHttpRequest(); 258 }
237 request.open("POST", targetURL); 259 FilterNotifier.addListener((action, item, newValue, oldValue) =>
238 request.addEventListener("load", taskDone, false); 260 {
239 request.addEventListener("error", taskDone, false); 261 if (action == "load")
240 request.send(JSON.stringify({ 262 {
241 url: url, 263 resolve();
242 startTime: Date.now(), 264 }
243 error: String(exception) 265 });
244 })); 266 }).then(crawl_urls).catch(reportException);
245 }.bind(null, url));
246 }
247 } 267 }
248 exports.run = run; 268 exports.run = run;
249 269
250 /** 270 /**
251 * Crawls a URL. This is a generator meant to be used via a Task object. 271 * Crawls a URL. This is a generator meant to be used via a Task object.
252 * 272 *
253 * @param {String} url 273 * @param {String} url
254 * @param {TabAllocator} tabAllocator 274 * @param {TabAllocator} tabAllocator
255 * @param {loadListener} loadListener 275 * @param {loadListener} loadListener
256 * @result {Object} 276 * @result {Object}
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after
313 333
314 function reportException(e) 334 function reportException(e)
315 { 335 {
316 let stack = ""; 336 let stack = "";
317 if (e && typeof e == "object" && "stack" in e) 337 if (e && typeof e == "object" && "stack" in e)
318 stack = e.stack + "\n"; 338 stack = e.stack + "\n";
319 339
320 Cu.reportError(e); 340 Cu.reportError(e);
321 dump(e + "\n" + stack + "\n"); 341 dump(e + "\n" + stack + "\n");
322 } 342 }
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld