Issue 3780 - wait for the loading of filters and only afterwards start to fetch pages

lib/crawler.js

Index: lib/crawler.js
diff --git a/lib/crawler.js b/lib/crawler.js
index 3445b2f5424fd846e71c3ffe3cb6fe964fc78329..a087fb484e3591ed553a35cf7662d48e0c08b694 100644
--- a/lib/crawler.js
+++ b/lib/crawler.js
@@ -22,6 +22,8 @@ function abprequire(module)
 
 let {RequestNotifier} = abprequire("requestNotifier");
 

[Wladimir Palant, 2016/03/15 13:44:22]

Nit: Why the empty line here?

[sergei, 2016/03/15 14:44:25]

Done.

+let {FilterNotifier} = abprequire("filterNotifier");
+let {FilterStorage} = abprequire("filterStorage");
 
 /**
  * Creates a pool of tabs and allocates them to tasks on request.
@@ -219,31 +221,50 @@ function run(window, urls, timeout, maxtabs, targetURL, onDone)
     }
   };
 
-  for (let url of urls)
+  let crawl_urls = () =>

[Wladimir Palant, 2016/03/15 13:44:22]

I really meant having crawl_urls as a regular function - |function
clawl_urls(urls)|, just above |function* crawl_url|.

[sergei, 2016/03/15 14:44:25]

Done.

   {
-    running++;
-    Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(function(result)
+    for (let url of urls)
     {
-      let request = new XMLHttpRequest();
-      request.open("POST", targetURL);
-      request.addEventListener("load", taskDone, false);
-      request.addEventListener("error", taskDone, false);
-      request.send(JSON.stringify(result));
-    }, function(url, exception)
-    {
-      reportException(exception);
+      running++;
+      Task.spawn(crawl_url.bind(null, url, tabAllocator, loadListener)).then(function(result)
+      {
+        let request = new XMLHttpRequest();
+        request.open("POST", targetURL);
+        request.addEventListener("load", taskDone, false);
+        request.addEventListener("error", taskDone, false);
+        request.send(JSON.stringify(result));
+      }, function(url, exception)
+      {
+        reportException(exception);
 
-      let request = new XMLHttpRequest();
-      request.open("POST", targetURL);
-      request.addEventListener("load", taskDone, false);
-      request.addEventListener("error", taskDone, false);
-      request.send(JSON.stringify({
-        url: url,
-        startTime: Date.now(),
-        error: String(exception)
-      }));
-    }.bind(null, url));
-  }
+        let request = new XMLHttpRequest();
+        request.open("POST", targetURL);
+        request.addEventListener("load", taskDone, false);
+        request.addEventListener("error", taskDone, false);
+        request.send(JSON.stringify({
+          url: url,
+          startTime: Date.now(),
+          error: String(exception)
+        }));
+      }.bind(null, url));
+    }
+  };
+  new Promise((resolve, reject) =>
+  {
+    if (FilterStorage.subscriptions.length > 0 && !FilterStorage._loading)
+    {
+      resolve();
+      return;
+    }
+    FilterNotifier.addListener((action, item, newValue, oldValue) =>
+    {
+      if (action == "load")
+      {
+        resolve();
+      }
+    });
+  }).then(crawl_urls)
+  .catch(reportException);

[Wladimir Palant, 2016/03/15 13:44:22]

Nit: put catch() on the same line as then()?

[sergei, 2016/03/15 14:44:25]

Done.

 }
 exports.run = run;