Issue 3716 - Split up files stored in storage.local

lib/io.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-2016 Eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ skipping 16 matching lines @@
   let key = fileToKey(file);
 
   ext.storage.get([key], function(items)
   {
     let entry = items[key];
 
     if (entry)
       successCallback(entry);
     else
       errorCallback(new Error("File doesn't exist"));
   });

[Sebastian Noack, 2016/03/31 12:36:54]

While changing most of the code in here, this is a good opportunity to enable
strict mode for this file.

 }
 
+// Save the file, splitting it into chunks, if required.
 function saveFile(file, data, callback)

[Sebastian Noack, 2016/03/31 12:36:53]

Since this function is only used by writeToFile() now, mind moving the code
there, and getting rid of saveFile()?

 {
-  ext.storage.set(
-    fileToKey(file),
+  let key = fileToKey(file);
+  let items = {};
+  items[key] = {lastModified: Date.now()};
+
+  if (typeof browser == "object")

[Sebastian Noack, 2016/03/31 12:36:54]

A comment that this accounts for Edge and how the quota was determined would
make sense here.

[Sebastian Noack, 2016/03/31 14:08:53]

I just noticed that we potentially leak chunks when overriding the file with a
smaller one. So we should remove previous chunks that doesn't get overridden:

  let items = {};
  let entry = items[fileToKey(file)] = {lastModified: Date.now()};

  if (typeof browser == "object")
  {
    loadFile(file).catch(() => null)
                  .then(oldEntry =>
    {
      let quota = 1024 * 1024 / 2 - 1000;
      let chunks = [];
      let chunk = [];
      let chunkSize = 0;

      for (let line of data)
      {
        if (line.length + chunkSize > quota)
        {
          chunks.push(chunk);
          chunk = [];
          chunkSize = 0;
        }

        chunk.push(line);
        chunkSize += line.length;
      }
      chunks.push(chunk);

      for (let i = 0; i < chunks.length; i++)
        items[fileToKey(file, i)] = chunks[i];
      entry.chunks = chunks.length;
      ext.storage.set(items, callback);

      if (oldEntry && "chunks" in oldEntry)
        for (let i = entry.chunks; i < oldEntry.chunks; i++)
          ext.storage.remove(fileToKey(file, i));
    });
  }
  else
  {
    entry.content = data;
    ext.storage.set(items, callback); 
  }

Note that in order to simplify the code above, I implied to make loadFile()
return a Promise:

  function loadFile(file)
  {
    return new Promise((resolve, reject) =>
    {
      let key = fileToKey(file);
      ext.storage.get([key], items =>
      {
        let entry = items[key];
        if (entry)
          resolve(entry);
        else
          reject(new Error("File doesn't exist"));
      });
    });
  }

+  {
+    let quota = 1024 * 1024 / 2 - 1000;
+    let chunks = [];
+    let chunk = [];
+    let chunkSize = 0;
+    
+    for (let line of data)
     {
-      content: data,
-      lastModified: Date.now()
-    },
-    callback
-  );
+      if (line.length + chunkSize > quota)
+      {
+        chunks.push(chunk);
+        chunk = [];
+        chunkSize = 0;
+      }
+
+      chunk.push(line);
+      chunkSize += line.length;
+    }
+    chunks.push(chunk);
+
+    if (chunks.length > 1)
+    {
+      for (let i = 0; i < chunks.length; i++)
+        items[key + ":" + i] = chunks[i];
+      items[key].chunks = chunks.length;
+      ext.storage.set(items, callback);
+      return;
+    }
+  }
+
+  items[key].content = data;
+  ext.storage.set(items, callback);  
 }
 
 exports.IO =
 {
   resolveFilePath: function(path)
   {
     return new FakeFile(path);
   },
 
   readFromFile: function(file, listener, callback)
   {
     function onLoaded(entry)
     {
-      for (let line of entry.content)
-        listener.process(line);
+      if ("content" in entry)
+      {
+        for (let line of entry.content)
+          listener.process(line);
 
-      listener.process(null);
-      callback(null);
+        listener.process(null);
+        callback(null);
+      }
+      else
+      {
+        let keys = [];
+        for (let i = 0; i < entry.chunks; i++)
+          keys.push(fileToKey(file) + ":" + i);

[Sebastian Noack, 2016/03/31 12:36:54]

How about moving the logic to create those keys into fileToKey()?

  function fileToKey(file, chunk)
  {
    let key = keyPrefix;

    if (file instanceof FakeFile)
      key += file.path;
    else
      key += file.spec;

    if (typeof chunk != "undefined")
      key += ":" + chunk;

    return key;
  }

+
+        ext.storage.get(keys, items =>
+        {
+           for (let key of keys)
+             for (let line of items[key])
+               listener.process(line);
+
+           listener.process(null);
+           callback(null);
+        });
+      }
     }
 
     loadFile(file, onLoaded, callback);
   },
 
   writeToFile: function(file, data, callback)
   {
     saveFile(file, data, callback);
   },
 
-  copyFile: function(fromFile, toFile, callback)
-  {
-    function onLoaded(entry)
-    {
-      saveFile(toFile, entry.content, callback);
-    }
-
-    loadFile(fromFile, onLoaded, callback);
-  },
-
-  renameFile: function(fromFile, newName, callback)
-  {
-    function onLoaded()
-    {
-      ext.storage.remove(fileToKey(fromFile), function()
-      {
-        ext.storage.set(keyPrefix + newName, entry, callback);
-      });
-    }
-
-    loadFile(fromFile, onLoaded, callback);
-  },
-
-  removeFile: function(file, callback)
-  {
-    ext.storage.remove(fileToKey(file), callback);
-  },
-
   statFile: function(file, callback)
   {
     function onLoaded(entry)
     {
       callback(null, {
         exists: true,
         lastModified: entry.lastModified
       });
     }
 
     loadFile(file, onLoaded, callback);
   }
 };