abpcrawler: Collect unmatched filters

lib/crawler.js

 Cu.import("resource://gre/modules/Services.jsm");
 
 function abprequire(module)
 {
   let result = {};
   result.wrappedJSObject = result;
   Services.obs.notifyObservers(result, "adblockplus-require", module);
   return result.exports;
 }
 
 let {Storage} = require("storage");
 let {Client} = require("client");
 
 let {Policy} = abprequire("contentPolicy");
+let {RequestNotifier} = abprequire("requestNotifier");
 let {Filter} = abprequire("filterClasses");
 let {Utils} = abprequire("utils");
 
 let origProcessNode = Policy.processNode;
 
+let requestNotifier;
 let siteTabs;
 let currentTabs;
+let currentFilter;

[Wladimir Palant, 2012/12/13 16:14:01]

Initialize with null?

[Felix Dahlke, 2012/12/14 10:11:58]

I'd rather keep it undefined, like currentTabs etc. It's fals-y.

 
 function processNode(wnd, node, contentType, location, collapse)
 {
   let result = origProcessNode.apply(this, arguments);
   let url = (contentType === Policy.type.ELEMHIDE) ? location.text :
       location.spec;
 
   let topWindow = wnd.top;
   if (!topWindow.document)
   {
@@ skipping 10 matching lines @@
 
   let browser = tabbrowser.getBrowserForDocument(topWindow.document);
   if (!browser)
   {
     Cu.reportError("Unable to get browser for the tab");
     return result;
   }
 
   let site = siteTabs.get(browser);
   let filtered = !result;
-  Storage.write([url, site, filtered]);
+  let data = [url, site, filtered];
+  if (currentFilter) {

[Wladimir Palant, 2012/12/14 16:43:01]

Missed on the first pass - bracket should be on next line.

+    data.push(currentFilter);
+    currentFilter = null;
+  }
+  Storage.write(data);
   return result;
 }
 
+function handleFilterHit(wnd, node, data)
+{
+  if (data.filter)
+    currentFilter = data.filter.text;
+}
+
+function prepare()
+{
+  if (Policy.processNode != origProcessNode)
+    return false;
+
+  Policy.processNode = processNode;
+
+  requestNotifier = new RequestNotifier(null, handleFilterHit);
+  siteTabs = new WeakMap();
+  currentTabs = 0;
+
+  Storage.init();
+
+  return true;
+}
+
 function loadSite(site, window, callback)
 {
   if (!site)
     return;
 
   let tabbrowser = window.gBrowser;
   let tab = tabbrowser.addTab(site);
   let browser = tabbrowser.getBrowserForTab(tab);
 
   siteTabs.set(browser, site);
@@ skipping 20 matching lines @@
   while (currentTabs < parallelTabs && sites.length)
   {
     currentTabs++;
     let site = sites.shift();
     loadSite(site, window, function()
     {
       currentTabs--;
       if (!sites.length && !currentTabs)
       {
         Storage.finish();
-        let dataFilePath = Storage.dataFile.path;
-        Client.sendCrawlerDataFile(backendUrl, dataFilePath, function()
+        let requestsFilePath = Storage.requestsFile.path;
+        Client.sendRequestsFile(backendUrl, requestsFilePath, function()
         {
           Storage.destroy();
           callback();
         });
       }
       else
         loadSites(backendUrl, parallelTabs, window, sites, callback);
     });
   }
 }
 
+function cleanUp()
+{
+  Policy.processNode = origProcessNode;
+  siteTabs = null;
+}
+
 let Crawler = exports.Crawler = {};
 
 Crawler.crawl = function(backendUrl, parallelTabs, window, callback)
 {
-  if (Policy.processNode != origProcessNode)
+  if (!prepare())
     return;
 
-  Policy.processNode = processNode;
-
-  siteTabs = new WeakMap();
-  currentTabs = 0;
-
-  Storage.init();
-
   Client.fetchCrawlableSites(backendUrl, function(sites)
   {
     loadSites(backendUrl, parallelTabs, window, sites, function()
     {
-      Policy.processNode = origProcessNode;
-      siteTabs = null;
+      cleanUp();
       callback();
     });
   });
 };