Issue 3716 - Split up files stored in storage.local

lib/io.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-2016 Eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+"use strict";
+
 const keyPrefix = "file:";
 
-function fileToKey(file)
+function fileToKey(file, chunk)
 {
-  return keyPrefix + (file instanceof FakeFile ? file.path : file.spec);
+  let key = keyPrefix;
+
+  if (file instanceof FakeFile)
+    key += file.path;
+  else
+    key += file.spec;
+
+  if (typeof chunk != "undefined")
+    key += ":" + chunk;
+
+  return key;
 }
 
 function loadFile(file, successCallback, errorCallback)
 {
   let key = fileToKey(file);
 
   ext.storage.get([key], function(items)
   {
     let entry = items[key];
 
     if (entry)
       successCallback(entry);
     else
       errorCallback(new Error("File doesn't exist"));
   });
 }
 
-function saveFile(file, data, callback)
-{
-  ext.storage.set(
-    fileToKey(file),
-    {
-      content: data,
-      lastModified: Date.now()
-    },
-    callback
-  );
-}
-
 exports.IO =
 {
   resolveFilePath: function(path)
   {
     return new FakeFile(path);
   },
 
   readFromFile: function(file, listener, callback)
   {
     function onLoaded(entry)
     {
-      for (let line of entry.content)
-        listener.process(line);
+      if ("content" in entry)
+      {
+        for (let line of entry.content)
+          listener.process(line);
 
-      listener.process(null);
-      callback(null);
+        listener.process(null);
+        callback(null);
+      }
+      else
+      {
+        let keys = [];
+        for (let i = 0; i < entry.chunks; i++)
+          keys.push(fileToKey(file, i));
+
+        ext.storage.get(keys, items =>
+        {
+           for (let key of keys)
+             for (let line of items[key])
+               listener.process(line);
+
+           listener.process(null);
+           callback(null);
+        });
+      }
     }
 
     loadFile(file, onLoaded, callback);
   },
-
+  
+  // Write to file, splitting into chunks, if required.

[Sebastian Noack, 2016/03/31 14:08:53]

Nit: This comment is kinda redundant with the comment below.

   writeToFile: function(file, data, callback)
   {
-    saveFile(file, data, callback);
-  },
+    let key = fileToKey(file);
+    let items = {};
+    items[key] = {lastModified: Date.now()};
 
-  copyFile: function(fromFile, toFile, callback)
-  {
-    function onLoaded(entry)
+    // Edge currently has a limitation of how much data can be stored
+    // using chrome.storage.local.set. The current limit is 1Mb per operation.
+    // Edge uses 'browser' namespace instead of 'chrome'.
+    if (typeof browser == "object")
     {
-      saveFile(toFile, entry.content, callback);
+      // In Edge strings are stored in UTF-16 format, so 2 bytes per char

[Sebastian Noack, 2016/03/31 14:08:53]

How about mentioning that we reserve another 1000 bytes for keys and the array
overhead?

+      let quota = 1024 * 1024 / 2 - 1000;
+      let chunks = [];
+      let chunk = [];
+      let chunkSize = 0;
+      
+      for (let line of data)
+      {
+        if (line.length + chunkSize > quota)
+        {
+          chunks.push(chunk);
+          chunk = [];
+          chunkSize = 0;
+        }
+
+        chunk.push(line);
+        chunkSize += line.length;
+      }
+      chunks.push(chunk);
+
+      if (chunks.length > 1)
+      {
+        for (let i = 0; i < chunks.length; i++)
+          items[fileToKey(file, i)] = chunks[i];
+        items[key].chunks = chunks.length;
+        ext.storage.set(items, callback);
+        return;
+      }
     }
 
-    loadFile(fromFile, onLoaded, callback);
-  },
-
-  renameFile: function(fromFile, newName, callback)
-  {
-    function onLoaded()
-    {
-      ext.storage.remove(fileToKey(fromFile), function()
-      {
-        ext.storage.set(keyPrefix + newName, entry, callback);
-      });
-    }
-
-    loadFile(fromFile, onLoaded, callback);
-  },
-
-  removeFile: function(file, callback)
-  {
-    ext.storage.remove(fileToKey(file), callback);
+    items[key].content = data;
+    ext.storage.set(items, callback);  
   },
 
   statFile: function(file, callback)
   {
     function onLoaded(entry)
     {
       callback(null, {
         exists: true,
         lastModified: entry.lastModified
       });
     }
 
     loadFile(file, onLoaded, callback);
   }
 };