Issue 3716 - Split up files stored in storage.local

lib/io.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-2016 Eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ skipping 13 matching lines @@
 
 function loadFile(file, successCallback, errorCallback)
 {
   let key = fileToKey(file);
 
   ext.storage.get([key], function(items)
   {
     let entry = items[key];
 
     if (entry)
-      successCallback(entry);
+    {
+      if (typeof entry.content.multipartKeys != "undefined")

[Sebastian Noack, 2016/03/30 12:06:38]

Please leave loadFile() untouched. There is no reason to load the chunks if
statFile() was called for example. Instead, please adapt readFile() like below,
and simply remove copyFile(), renameFile() and removeFile() these aren't used
anymore.

  readFromFile: function(file, listener, callback)
  {
    function onLoaded(entry)
    {
      if ("content" in entry)
      {
        for (let line of entry.content)
          listener.process(line);

        listener.process(null);
        callback(null);
      }
      else
      {
        let keys = [];
        for (let i = 0; i < entry.chunks; i++)
          keys.push(fileToKey(file) + ":" + i);

        ext.storage.get(keys, items =>
        {
           for (let key of keys)
             for (let line of items[key])
               listener.process(line);

           listener.process(null);
           callback(null);
        });
      }
    }

    loadFile(file, onLoaded, callback);
  }

+      {
+        var multipartData = [];
+        ext.storage.get(entry.content.multipartKeys, function(parts) 
+        {       
+          for (var part in parts)
+          {
+            multipartData = multipartData.concat(parts[part]);
+          }
+          entry.content = multipartData;
+          successCallback(entry);
+        }, errorCallback);
+      } 
+      else
+      {
+        successCallback(entry); 
+      }      
+    }
     else
+    {
       errorCallback(new Error("File doesn't exist"));
+    }
   });
 }
-
+// Save the file, splitting it into chunks, if required.
 function saveFile(file, data, callback)
 {
-  ext.storage.set(
-    fileToKey(file),
+  var storageList = {};
+  // Note: In future this should probably be something like 
+  // browser.storage.local.QUOTA_BYTES_PER_ITEM  
+  var chunkSize = 104856;

[Sebastian Noack, 2016/03/30 12:06:37]

They told me that each character is two bytes. Moreover we need to reserve some
addional bytes for the keys and overhead of the array. So I guess the quota
should rather be like that:

1024 * 1024 / 2 - 1000 = 523288

[Oleksandr, 2016/03/31 09:59:54]

Yes, strings are UTF-16 in Edge, unlike in Chrome where they are UTF-8, it
seems. I was trying to be more generic here and foresee storing Arrays that are
not strings. I don't think there's a need for that now, you're right.

+  if ((data.length > chunkSize) && (typeof browser == "object"))
+  {
+    for (var dataSize = 0; dataSize < data.length; dataSize += chunkSize)
     {
-      content: data,
+      var key = fileToKey(file) + "_" + dataSize;
+      storageList[key] = data.slice(dataSize, dataSize + chunkSize);

[Sebastian Noack, 2016/03/30 12:06:38]

Note that data is an array of strings, where each item represents a line in the
filter list. So the logic here has to be a little more complex:

  let key = fileToKey(file);
  let items = {};
  items[key] = {lastModified: Date.now()};

  if (typeof browser == "object")
  {
    let quota = 1024 * 1024 / 2 - 1000;
    let chunks = [];
    let chunk = [];
    let chunkSize = 0;
    
    for (let line of data)
    {
      if (line.length + chunkSize > quota)
      {
        chunks.push(chunk);
        chunk = [];
        chunkSize = 0;
      }

      chunk.push(line);
      chunkSize += line.length;
    }
    chunks.push(chunk);

    if (chunks.length > 1)
    {
      for (let i = 0; i < chunks.length; i++)
        items[key + ":" + i] = chunks[i];
      items[key].chunks = chunks.length;
      ext.storage.set(items, callback);
      return;
    }
  }

  items[key].content = data;
  ext.storage.set(items, callback);

[Oleksandr, 2016/03/31 09:59:54]

Yep. For some reason I did not notice the data comes in lines, thanks!

+    }
+    storageList[fileToKey(file)] = {
+      content: { multipartKeys: Object.keys(storageList) },

[Sebastian Noack, 2016/03/30 12:06:38]

We can easily reconstruct the keys for the chunks. So instead of a list of keys,
we can just store the number of chunks, as in the suggested code above.

       lastModified: Date.now()
-    },
-    callback
-  );
+    };
+  } 
+  else
+  {
+    storageList[fileToKey(file)] = {
+        content: data,
+        lastModified: Date.now()
+      };    
+  }  
+  ext.storage.setMultiple(storageList, callback);
 }
 
 exports.IO =
 {
   resolveFilePath: function(path)
   {
     return new FakeFile(path);
   },
 
   readFromFile: function(file, listener, callback)
@@ skipping 49 matching lines @@
     {
       callback(null, {
         exists: true,
         lastModified: entry.lastModified
       });
     }
 
     loadFile(file, onLoaded, callback);
   }
 };