Improved patterns.ini reading to be more memory-efficient

lib/io.js

Index: lib/io.js
===================================================================
--- a/lib/io.js
+++ b/lib/io.js
@@ -54,96 +54,92 @@ let IO = exports.IO =
    * Reads strings from a file asynchronously, calls listener.process() with
    * each line read and with a null parameter once the read operation is done.
    * The callback will be called when the operation is done.
    */
   readFromFile: function(/**nsIFile|nsIURI*/ file, /**Boolean*/ decode, /**Object*/ listener, /**Function*/ callback, /**String*/ timeLineID)
   {
     try
     {
+      let buffer = "";
       let uri = file instanceof Ci.nsIFile ? Services.io.newFileURI(file) : file;
-      let channel = Services.io.newChannelFromURI(uri);
-      channel.contentType = "text/plain";
-      let converter = null;
-      if (decode)
+      let request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"].createInstance(Ci.nsIXMLHttpRequest);
+      request.open("GET", uri.spec);
+      request.mozBackgroundRequest = true;
+      request.responseType = "moz-chunked-text";
+      request.overrideMimeType("text/plain" + (decode ? "? charset=utf-8" : ""));
+
+      request.addEventListener("progress", function(event)
       {
-        converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].createInstance(Ci.nsIScriptableUnicodeConverter);
-        converter.charset = "utf-8";
-      }
+        if (timeLineID)
+        {
+          TimeLine.asyncStart(timeLineID);
+        }
 
-      channel.asyncOpen({
-        buffer: "",
-        QueryInterface: XPCOMUtils.generateQI([Ci.nsIRequestObserver, Ci.nsIStreamListener]),
-        onStartRequest: function(request, context) {},
-        onDataAvailable: function(request, context, stream, offset, count)
+        let data = event.target.response;
+        let index = Math.max(data.lastIndexOf("\n"), data.lastIndexOf("\r"));
+        if (index >= 0)
         {
-          if (timeLineID)
-          {
-            TimeLine.asyncStart(timeLineID);
-          }
+          let oldBuffer = buffer;
+          buffer = data.substr(index + 1);
+          data = data.substr(0, index + 1);

[Felix Dahlke, 2012/10/30 17:40:06]

Nit: You're doing "index + 1" twice, might be worth a variable.

+          let lines = data.split(/[\r\n]+/);
+          lines.pop();
+          lines[0] = oldBuffer + lines[0];
+          for (let i = 0; i < lines.length; i++)
+            listener.process(lines[i]);
+        }
+        else
+          buffer += data;
 
-          let inputStream = Cc["@mozilla.org/scriptableinputstream;1"].createInstance(Ci.nsIScriptableInputStream);
-          inputStream.init(stream);
+        if (timeLineID)
+        {
+          TimeLine.asyncEnd(timeLineID);
+        }
+      }, false);
 
-          let data = this.buffer + inputStream.readBytes(count);
-          let index = Math.max(data.lastIndexOf("\n"), data.lastIndexOf("\r"));
-          if (index >= 0)
-          {
-            this.buffer = data.substr(index + 1);
-            data = data.substr(0, index + 1);
-            if (converter)
-              data = converter.ConvertToUnicode(data);
+      request.addEventListener("load", function(event)
+      {
+        if (timeLineID)
+        {
+          TimeLine.asyncStart(timeLineID);
+        }
 
-            let lines = data.split(/[\r\n]+/);
-            lines.pop();
-            for (let i = 0; i < lines.length; i++)
-              listener.process(lines[i]);
-          }
-          else
-            this.buffer = data;
+        if (buffer !== "")
+          listener.process(buffer);
+        listener.process(null);
 
-          if (timeLineID)
-          {
-            TimeLine.asyncEnd(timeLineID);
-          }
-        },
-        onStopRequest: function(request, context, result)
+        if (timeLineID)
         {
-          if (timeLineID)
-          {
-            TimeLine.asyncStart(timeLineID);
-          }
+          TimeLine.asyncEnd(timeLineID);
+          TimeLine.asyncDone(timeLineID);
+        }
 
-          if (Components.isSuccessCode(result) && this.buffer.length)
-            listener.process(this.buffer);
-          listener.process(null);
+        callback(null);
+      }, false);
 
-          if (timeLineID)
-          {
-            TimeLine.asyncEnd(timeLineID);
-            TimeLine.asyncDone(timeLineID);
-          }
+      request.addEventListener("error", function()
+      {
+        let e = Cc["@mozilla.org/js/xpc/Exception;1"].createInstance(Ci.nsIXPCException);
+        e.initialize("File read operation failed", result, null, Components.stack, file, null);
+        callback(e);
 
-          if (!Components.isSuccessCode(result))
-          {
-            let e = Cc["@mozilla.org/js/xpc/Exception;1"].createInstance(Ci.nsIXPCException);
-            e.initialize("File read operation failed", result, null, Components.stack, file, null);
-            callback(e);
-          }
-          else
-            callback(null);
+        if (timeLineID)
+        {
+          TimeLine.asyncDone(timeLineID);
         }
-      }, null);
+      }, false);
+
+      request.send(null);
     }
     catch (e)
     {
       callback(e);
     }
   },
-
   /**
    * Writes string data to a file asynchronously, optionally encodes it into
    * UTF-8 first. The callback will be called when the write operation is done.
    */
   writeToFile: function(/**nsIFile*/ file, /**Boolean*/ encode, /**Iterator*/ data, /**Function*/ callback, /**String*/ timeLineID)
   {
     try
     {