Issue 3716 - Split up files stored in storage.local

lib/io.js

Index: lib/io.js
===================================================================
--- a/lib/io.js
+++ b/lib/io.js
@@ -37,16 +37,46 @@
   });

[Sebastian Noack, 2016/03/31 12:36:54]

While changing most of the code in here, this is a good opportunity to enable
strict mode for this file.

 }
 
+// Save the file, splitting it into chunks, if required.
 function saveFile(file, data, callback)

[Sebastian Noack, 2016/03/31 12:36:53]

Since this function is only used by writeToFile() now, mind moving the code
there, and getting rid of saveFile()?

 {
-  ext.storage.set(
-    fileToKey(file),
+  let key = fileToKey(file);
+  let items = {};
+  items[key] = {lastModified: Date.now()};
+
+  if (typeof browser == "object")

[Sebastian Noack, 2016/03/31 12:36:54]

A comment that this accounts for Edge and how the quota was determined would
make sense here.

[Sebastian Noack, 2016/03/31 14:08:53]

I just noticed that we potentially leak chunks when overriding the file with a
smaller one. So we should remove previous chunks that doesn't get overridden:

  let items = {};
  let entry = items[fileToKey(file)] = {lastModified: Date.now()};

  if (typeof browser == "object")
  {
    loadFile(file).catch(() => null)
                  .then(oldEntry =>
    {
      let quota = 1024 * 1024 / 2 - 1000;
      let chunks = [];
      let chunk = [];
      let chunkSize = 0;

      for (let line of data)
      {
        if (line.length + chunkSize > quota)
        {
          chunks.push(chunk);
          chunk = [];
          chunkSize = 0;
        }

        chunk.push(line);
        chunkSize += line.length;
      }
      chunks.push(chunk);

      for (let i = 0; i < chunks.length; i++)
        items[fileToKey(file, i)] = chunks[i];
      entry.chunks = chunks.length;
      ext.storage.set(items, callback);

      if (oldEntry && "chunks" in oldEntry)
        for (let i = entry.chunks; i < oldEntry.chunks; i++)
          ext.storage.remove(fileToKey(file, i));
    });
  }
  else
  {
    entry.content = data;
    ext.storage.set(items, callback); 
  }

Note that in order to simplify the code above, I implied to make loadFile()
return a Promise:

  function loadFile(file)
  {
    return new Promise((resolve, reject) =>
    {
      let key = fileToKey(file);
      ext.storage.get([key], items =>
      {
        let entry = items[key];
        if (entry)
          resolve(entry);
        else
          reject(new Error("File doesn't exist"));
      });
    });
  }

+  {
+    let quota = 1024 * 1024 / 2 - 1000;
+    let chunks = [];
+    let chunk = [];
+    let chunkSize = 0;
+    
+    for (let line of data)
     {
-      content: data,
-      lastModified: Date.now()
-    },
-    callback
-  );
+      if (line.length + chunkSize > quota)
+      {
+        chunks.push(chunk);
+        chunk = [];
+        chunkSize = 0;
+      }
+
+      chunk.push(line);
+      chunkSize += line.length;
+    }
+    chunks.push(chunk);
+
+    if (chunks.length > 1)
+    {
+      for (let i = 0; i < chunks.length; i++)
+        items[key + ":" + i] = chunks[i];
+      items[key].chunks = chunks.length;
+      ext.storage.set(items, callback);
+      return;
+    }
+  }
+
+  items[key].content = data;
+  ext.storage.set(items, callback);  
 }
 
 exports.IO =
@@ -60,11 +90,30 @@
   {
     function onLoaded(entry)
     {
-      for (let line of entry.content)
-        listener.process(line);
+      if ("content" in entry)
+      {
+        for (let line of entry.content)
+          listener.process(line);
 
-      listener.process(null);
-      callback(null);
+        listener.process(null);
+        callback(null);
+      }
+      else
+      {
+        let keys = [];
+        for (let i = 0; i < entry.chunks; i++)
+          keys.push(fileToKey(file) + ":" + i);

[Sebastian Noack, 2016/03/31 12:36:54]

How about moving the logic to create those keys into fileToKey()?

  function fileToKey(file, chunk)
  {
    let key = keyPrefix;

    if (file instanceof FakeFile)
      key += file.path;
    else
      key += file.spec;

    if (typeof chunk != "undefined")
      key += ":" + chunk;

    return key;
  }

+
+        ext.storage.get(keys, items =>
+        {
+           for (let key of keys)
+             for (let line of items[key])
+               listener.process(line);
+
+           listener.process(null);
+           callback(null);
+        });
+      }
     }
 
     loadFile(file, onLoaded, callback);
@@ -75,34 +124,6 @@
     saveFile(file, data, callback);
   },
 
-  copyFile: function(fromFile, toFile, callback)
-  {
-    function onLoaded(entry)
-    {
-      saveFile(toFile, entry.content, callback);
-    }
-
-    loadFile(fromFile, onLoaded, callback);
-  },
-
-  renameFile: function(fromFile, newName, callback)
-  {
-    function onLoaded()
-    {
-      ext.storage.remove(fileToKey(fromFile), function()
-      {
-        ext.storage.set(keyPrefix + newName, entry, callback);
-      });
-    }
-
-    loadFile(fromFile, onLoaded, callback);
-  },
-
-  removeFile: function(file, callback)
-  {
-    ext.storage.remove(fileToKey(file), callback);
-  },
-
   statFile: function(file, callback)
   {
     function onLoaded(entry)