Issue 3775 - fix saving of requests

lib/crawler.js

 /*
  * This Source Code is subject to the terms of the Mozilla Public License
  * version 2.0 (the "License"). You can obtain a copy of the License at
  * http://mozilla.org/MPL/2.0/.
  */
 
 /**
  * @module crawler
  */
 
@@ skipping 242 matching lines @@
  * @param {String} url
  * @param {TabAllocator} tabAllocator
  * @param {loadListener} loadListener
  * @result {Object}
  *    Crawling result
  */
 function* crawl_url(url, tabAllocator, loadListener)
 {
   let tab = yield tabAllocator.getTab();
   let result = {url, requests: []};
-
+  let requestNotifier;
   try
   {
     result.startTime = Date.now();
-    let requestNotifier = new RequestNotifier(tab.linkedBrowser.outerWindowID, function({type, location, filter}, scanComplete)

[Wladimir Palant, 2016/03/14 19:50:43]

When the scan is completed the listener is called with null as first parameter -
this will produce an error right now. You need to check the parameter first,
before decomposing.

Style nit: while we didn't strictly enforce this rule here, please keep the code
reasonably within 80 characters.

[sergei, 2016/03/15 10:59:56]

Fixed, is the indentation correct now?

[sergei, 2016/03/15 10:59:56]

Fixed.

However, I guess, it should be documented in `RequestNotifier` constructor and I
actually cannot reproduce it, how to reproduce it in crawler?

[Wladimir Palant, 2016/03/15 11:05:44]

Yes, it should be documented. As to reproducing - I don't know, but
RequestNotifier.onComplete() will trigger the listener with null parameter.

-    {
-      result.requests.push({location, contentType: type, filter});
+    requestNotifier = new RequestNotifier(tab.linkedBrowser.outerWindowID,
+      function(entry, scanComplete)
+    {
+      if (!entry)
+        return;
+      let {type: contentType, location, filter} = entry;
+      result.requests.push({location, contentType, filter});
     });

[Wladimir Palant, 2016/03/14 19:50:43]

You need to shut down this notifier when you are done, just like the tab is
being released. Otherwise the content side won't be shut down - you'll leak
memory and produce unnecessary IPC messages.

[sergei, 2016/03/15 10:59:56]

Done. Thanks, overlooked it.

 
     tab.linkedBrowser.loadURI(url, null, null);
     [result.channelStatus, result.headers] = yield loadListener.waitForLoad(tab);
     result.endTime = Date.now();
     result.finalUrl = tab.linkedBrowser.currentURI.spec;
 
     let document = tab.linkedBrowser.contentDocument;
     if (document.documentElement)
     {
       try
@@ skipping 12 matching lines @@
         result.error = "Capturing screenshot failed: " + e;
       }
 
       // TODO: Capture frames as well?
       let serializer = new tab.ownerDocument.defaultView.XMLSerializer();
       result.source = serializer.serializeToString(document.documentElement);
     }
   }
   finally
   {
+    if (requestNotifier)
+      requestNotifier.shutdown();
     tabAllocator.releaseTab(tab);
   }
   return result;
 }
 
 function reportException(e)
 {
   let stack = "";
   if (e && typeof e == "object" && "stack" in e)
     stack = e.stack + "\n";
 
   Cu.reportError(e);
   dump(e + "\n" + stack + "\n");
 }