Issue 3716 - Split up files stored in storage.local

lib/io.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-2016 Eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+"use strict";
+
 const keyPrefix = "file:";
 
-function fileToKey(file)
+function fileToKey(file, chunk)
 {
-  return keyPrefix + (file instanceof FakeFile ? file.path : file.spec);
+  let key = keyPrefix;
+
+  if (file instanceof FakeFile)
+    key += file.path;
+  else
+    key += file.spec;
+
+  if (typeof chunk != "undefined")
+    key += ":" + chunk;
+
+  return key;
 }
 
-function loadFile(file, successCallback, errorCallback)
+function loadFile(file)
 {
-  let key = fileToKey(file);
-
-  ext.storage.get([key], function(items)
+  return new Promise((resolve, reject) =>
   {
-    let entry = items[key];
-
-    if (entry)
+    let key = fileToKey(file);
+    ext.storage.get([key], items =>
     {
-      if (typeof entry.content.multipartKeys != "undefined")

[Sebastian Noack, 2016/03/30 12:06:38]

Please leave loadFile() untouched. There is no reason to load the chunks if
statFile() was called for example. Instead, please adapt readFile() like below,
and simply remove copyFile(), renameFile() and removeFile() these aren't used
anymore.

  readFromFile: function(file, listener, callback)
  {
    function onLoaded(entry)
    {
      if ("content" in entry)
      {
        for (let line of entry.content)
          listener.process(line);

        listener.process(null);
        callback(null);
      }
      else
      {
        let keys = [];
        for (let i = 0; i < entry.chunks; i++)
          keys.push(fileToKey(file) + ":" + i);

        ext.storage.get(keys, items =>
        {
           for (let key of keys)
             for (let line of items[key])
               listener.process(line);

           listener.process(null);
           callback(null);
        });
      }
    }

    loadFile(file, onLoaded, callback);
  }

-      {
-        var multipartData = [];
-        ext.storage.get(entry.content.multipartKeys, function(parts) 
-        {       
-          for (var part in parts)
-          {
-            multipartData = multipartData.concat(parts[part]);
-          }
-          entry.content = multipartData;
-          successCallback(entry);
-        }, errorCallback);
-      } 
+      let entry = items[key];
+      if (entry)
+        resolve(entry);
       else
-      {
-        successCallback(entry); 
-      }      
-    }
-    else
-    {
-      errorCallback(new Error("File doesn't exist"));
-    }
+        reject(new Error("File doesn't exist"));
+    });
   });
-}
-// Save the file, splitting it into chunks, if required.
-function saveFile(file, data, callback)
-{
-  var storageList = {};
-  // Note: In future this should probably be something like 
-  // browser.storage.local.QUOTA_BYTES_PER_ITEM  
-  var chunkSize = 104856;

[Sebastian Noack, 2016/03/30 12:06:37]

They told me that each character is two bytes. Moreover we need to reserve some
addional bytes for the keys and overhead of the array. So I guess the quota
should rather be like that:

1024 * 1024 / 2 - 1000 = 523288

[Oleksandr, 2016/03/31 09:59:54]

Yes, strings are UTF-16 in Edge, unlike in Chrome where they are UTF-8, it
seems. I was trying to be more generic here and foresee storing Arrays that are
not strings. I don't think there's a need for that now, you're right.

-  if ((data.length > chunkSize) && (typeof browser == "object"))
-  {
-    for (var dataSize = 0; dataSize < data.length; dataSize += chunkSize)
-    {
-      var key = fileToKey(file) + "_" + dataSize;
-      storageList[key] = data.slice(dataSize, dataSize + chunkSize);

[Sebastian Noack, 2016/03/30 12:06:38]

Note that data is an array of strings, where each item represents a line in the
filter list. So the logic here has to be a little more complex:

  let key = fileToKey(file);
  let items = {};
  items[key] = {lastModified: Date.now()};

  if (typeof browser == "object")
  {
    let quota = 1024 * 1024 / 2 - 1000;
    let chunks = [];
    let chunk = [];
    let chunkSize = 0;
    
    for (let line of data)
    {
      if (line.length + chunkSize > quota)
      {
        chunks.push(chunk);
        chunk = [];
        chunkSize = 0;
      }

      chunk.push(line);
      chunkSize += line.length;
    }
    chunks.push(chunk);

    if (chunks.length > 1)
    {
      for (let i = 0; i < chunks.length; i++)
        items[key + ":" + i] = chunks[i];
      items[key].chunks = chunks.length;
      ext.storage.set(items, callback);
      return;
    }
  }

  items[key].content = data;
  ext.storage.set(items, callback);

[Oleksandr, 2016/03/31 09:59:54]

Yep. For some reason I did not notice the data comes in lines, thanks!

-    }
-    storageList[fileToKey(file)] = {
-      content: { multipartKeys: Object.keys(storageList) },

[Sebastian Noack, 2016/03/30 12:06:38]

We can easily reconstruct the keys for the chunks. So instead of a list of keys,
we can just store the number of chunks, as in the suggested code above.

-      lastModified: Date.now()
-    };
-  } 
-  else
-  {
-    storageList[fileToKey(file)] = {
-        content: data,
-        lastModified: Date.now()
-      };    
-  }  
-  ext.storage.setMultiple(storageList, callback);
 }
 
 exports.IO =
 {
   resolveFilePath: function(path)
   {
     return new FakeFile(path);
   },
 
   readFromFile: function(file, listener, callback)
   {
     function onLoaded(entry)
     {
-      for (let line of entry.content)
-        listener.process(line);
+      if ("content" in entry)
+      {
+        for (let line of entry.content)
+          listener.process(line);
 
-      listener.process(null);
-      callback(null);
+        listener.process(null);
+        callback(null);
+      }
+      else
+      {
+        let keys = [];
+        for (let i = 0; i < entry.chunks; i++)
+          keys.push(fileToKey(file, i));
+
+        ext.storage.get(keys, items =>
+        {
+           for (let key of keys)
+             for (let line of items[key])
+               listener.process(line);
+
+           listener.process(null);
+           callback(null);
+        });
+      }
     }
 
-    loadFile(file, onLoaded, callback);
+    loadFile(file).then(onLoaded, callback);
   },
 
   writeToFile: function(file, data, callback)
   {
-    saveFile(file, data, callback);
-  },
+    let items = {};
+    let entry = items[fileToKey(file)] = {lastModified: Date.now()};
 
-  copyFile: function(fromFile, toFile, callback)
-  {
-    function onLoaded(entry)
+    if (typeof browser == "object")
     {
-      saveFile(toFile, entry.content, callback);
-    }
+      loadFile(file).catch(() => null)
+                    .then(oldEntry =>
+      {
+        let quota = 1024 * 1024 / 2 - 1000;
+        let chunks = [];
+        let chunk = [];
+        let chunkSize = 0;
 
-    loadFile(fromFile, onLoaded, callback);
-  },
+        for (let line of data)
+        {
+          if (line.length + chunkSize > quota)
+          {
+            chunks.push(chunk);
+            chunk = [];
+            chunkSize = 0;
+          }
 
-  renameFile: function(fromFile, newName, callback)
-  {
-    function onLoaded()
-    {
-      ext.storage.remove(fileToKey(fromFile), function()
-      {
-        ext.storage.set(keyPrefix + newName, entry, callback);
+          chunk.push(line);
+          chunkSize += line.length;
+        }
+        chunks.push(chunk);
+
+        for (let i = 0; i < chunks.length; i++)
+          items[fileToKey(file, i)] = chunks[i];
+        entry.chunks = chunks.length;
+        ext.storage.set(items, callback);
+
+        if (oldEntry && "chunks" in oldEntry)
+          for (let i = entry.chunks; i < oldEntry.chunks; i++)
+            ext.storage.remove(fileToKey(file, i));
       });
     }
-
-    loadFile(fromFile, onLoaded, callback);
-  },
-
-  removeFile: function(file, callback)
-  {
-    ext.storage.remove(fileToKey(file), callback);
+    else
+    {
+      entry.content = data;
+      ext.storage.set(items, callback);
+    }
   },
 
   statFile: function(file, callback)
   {
     function onLoaded(entry)
     {
       callback(null, {
         exists: true,
         lastModified: entry.lastModified
       });
     }
 
-    loadFile(file, onLoaded, callback);
+    loadFile(file).then(onLoaded, callback);
   }
 };