Issue 3716 - Split up files stored in storage.local

lib/io.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-2016 Eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+"use strict";
+
 const keyPrefix = "file:";
 
-function fileToKey(file)
+function fileToKey(file, chunk)
 {
-  return keyPrefix + (file instanceof FakeFile ? file.path : file.spec);
+  let key = keyPrefix;
+
+  if (file instanceof FakeFile)
+    key += file.path;
+  else
+    key += file.spec;
+
+  if (typeof chunk != "undefined")
+    key += ":" + chunk;
+
+  return key;
 }
 
-function loadFile(file, successCallback, errorCallback)
+function loadFile(file)
 {
-  let key = fileToKey(file);
-
-  ext.storage.get([key], function(items)
+  return new Promise((resolve, reject) =>
   {
-    let entry = items[key];
-
-    if (entry)
-      successCallback(entry);
-    else
-      errorCallback(new Error("File doesn't exist"));
+    let key = fileToKey(file);
+    ext.storage.get([key], items =>
+    {
+      let entry = items[key];
+      if (entry)
+        resolve(entry);
+      else
+        reject(new Error("File doesn't exist"));
+    });
   });
-}
-
-// Save the file, splitting it into chunks, if required.
-function saveFile(file, data, callback)
-{
-  let key = fileToKey(file);
-  let items = {};
-  items[key] = {lastModified: Date.now()};
-
-  if (typeof browser == "object")

[Sebastian Noack, 2016/03/31 12:36:54]

A comment that this accounts for Edge and how the quota was determined would
make sense here.

[Sebastian Noack, 2016/03/31 14:08:53]

I just noticed that we potentially leak chunks when overriding the file with a
smaller one. So we should remove previous chunks that doesn't get overridden:

  let items = {};
  let entry = items[fileToKey(file)] = {lastModified: Date.now()};

  if (typeof browser == "object")
  {
    loadFile(file).catch(() => null)
                  .then(oldEntry =>
    {
      let quota = 1024 * 1024 / 2 - 1000;
      let chunks = [];
      let chunk = [];
      let chunkSize = 0;

      for (let line of data)
      {
        if (line.length + chunkSize > quota)
        {
          chunks.push(chunk);
          chunk = [];
          chunkSize = 0;
        }

        chunk.push(line);
        chunkSize += line.length;
      }
      chunks.push(chunk);

      for (let i = 0; i < chunks.length; i++)
        items[fileToKey(file, i)] = chunks[i];
      entry.chunks = chunks.length;
      ext.storage.set(items, callback);

      if (oldEntry && "chunks" in oldEntry)
        for (let i = entry.chunks; i < oldEntry.chunks; i++)
          ext.storage.remove(fileToKey(file, i));
    });
  }
  else
  {
    entry.content = data;
    ext.storage.set(items, callback); 
  }

Note that in order to simplify the code above, I implied to make loadFile()
return a Promise:

  function loadFile(file)
  {
    return new Promise((resolve, reject) =>
    {
      let key = fileToKey(file);
      ext.storage.get([key], items =>
      {
        let entry = items[key];
        if (entry)
          resolve(entry);
        else
          reject(new Error("File doesn't exist"));
      });
    });
  }

-  {
-    let quota = 1024 * 1024 / 2 - 1000;
-    let chunks = [];
-    let chunk = [];
-    let chunkSize = 0;
-    
-    for (let line of data)
-    {
-      if (line.length + chunkSize > quota)
-      {
-        chunks.push(chunk);
-        chunk = [];
-        chunkSize = 0;
-      }
-
-      chunk.push(line);
-      chunkSize += line.length;
-    }
-    chunks.push(chunk);
-
-    if (chunks.length > 1)
-    {
-      for (let i = 0; i < chunks.length; i++)
-        items[key + ":" + i] = chunks[i];
-      items[key].chunks = chunks.length;
-      ext.storage.set(items, callback);
-      return;
-    }
-  }
-
-  items[key].content = data;
-  ext.storage.set(items, callback);  
 }
 
 exports.IO =
 {
   resolveFilePath: function(path)
   {
     return new FakeFile(path);
   },
 
   readFromFile: function(file, listener, callback)
   {
     function onLoaded(entry)
     {
       if ("content" in entry)
       {
         for (let line of entry.content)
           listener.process(line);
 
         listener.process(null);
         callback(null);
       }
       else
       {
         let keys = [];
         for (let i = 0; i < entry.chunks; i++)
-          keys.push(fileToKey(file) + ":" + i);

[Sebastian Noack, 2016/03/31 12:36:54]

How about moving the logic to create those keys into fileToKey()?

  function fileToKey(file, chunk)
  {
    let key = keyPrefix;

    if (file instanceof FakeFile)
      key += file.path;
    else
      key += file.spec;

    if (typeof chunk != "undefined")
      key += ":" + chunk;

    return key;
  }

+          keys.push(fileToKey(file, i));
 
         ext.storage.get(keys, items =>
         {
            for (let key of keys)
              for (let line of items[key])
                listener.process(line);
 
            listener.process(null);
            callback(null);
         });
       }
     }
 
-    loadFile(file, onLoaded, callback);
+    loadFile(file).then(onLoaded, callback);
   },
 
   writeToFile: function(file, data, callback)
   {
-    saveFile(file, data, callback);
+    let items = {};
+    let entry = items[fileToKey(file)] = {lastModified: Date.now()};
+
+    if (typeof browser == "object")
+    {
+      loadFile(file).catch(() => null)
+                    .then(oldEntry =>
+      {
+        let quota = 1024 * 1024 / 2 - 1000;
+        let chunks = [];
+        let chunk = [];
+        let chunkSize = 0;
+
+        for (let line of data)
+        {
+          if (line.length + chunkSize > quota)
+          {
+            chunks.push(chunk);
+            chunk = [];
+            chunkSize = 0;
+          }
+
+          chunk.push(line);
+          chunkSize += line.length;
+        }
+        chunks.push(chunk);
+
+        for (let i = 0; i < chunks.length; i++)
+          items[fileToKey(file, i)] = chunks[i];
+        entry.chunks = chunks.length;
+        ext.storage.set(items, callback);
+
+        if (oldEntry && "chunks" in oldEntry)
+          for (let i = entry.chunks; i < oldEntry.chunks; i++)
+            ext.storage.remove(fileToKey(file, i));
+      });
+    }
+    else
+    {
+      entry.content = data;
+      ext.storage.set(items, callback);
+    }
   },
 
   statFile: function(file, callback)
   {
     function onLoaded(entry)
     {
       callback(null, {
         exists: true,
         lastModified: entry.lastModified
       });
     }
 
-    loadFile(file, onLoaded, callback);
+    loadFile(file).then(onLoaded, callback);
   }
 };