Issue 5137 - [emscripten] Added basic filter storage implementation

lib/filterStorage.js

Index: lib/filterStorage.js
===================================================================
--- a/lib/filterStorage.js
+++ b/lib/filterStorage.js
@@ -12,775 +12,87 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 "use strict";
 
-/**
- * @fileOverview FilterStorage class responsible for managing user's
- *               subscriptions and filters.
- */
-
-const {IO} = require("io");
+const {FilterStorage} = require("compiled");
+const {Subscription, SpecialSubscription} = require("subscriptionClasses");
+const {ActiveFilter} = require("filterClasses");
 const {Prefs} = require("prefs");
-const {Filter, ActiveFilter} = require("filterClasses");
-const {Subscription, SpecialSubscription,
-       ExternalSubscription} = require("subscriptionClasses");
-const {FilterNotifier} = require("filterNotifier");
 
-/**
- * Version number of the filter storage file format.
- * @type {number}
- */
-let formatVersion = 5;
+// Backwards compatibility
+FilterStorage.getGroupForFilter = FilterStorage.getSubscriptionForFilter;
 
 /**
- * This class reads user's filters from disk, manages them in memory
- * and writes them back.
- * @class
+ * This property allows iterating over the list of subscriptions. It will delete
+ * references automatically at the end of the current loop iteration. If you
+ * need persistent references or element access by position you should use
+ * FilterStorage.subscriptionAt() instead.
+ * @type {Iterable}
  */
-let FilterStorage = exports.FilterStorage =
-{
-  /**
-   * Will be set to true after the initial loadFromDisk() call completes.
-   * @type {boolean}
-   */
-  initialized: false,
-
-  /**
-   * Version number of the patterns.ini format used.
-   * @type {number}
-   */
-  get formatVersion()
-  {
-    return formatVersion;
-  },
-
-  /**
-   * File containing the filter list
-   * @type {string}
-   */
-  get sourceFile()
+FilterStorage.subscriptions = {
+  [Symbol.iterator]: function*()
   {
-    return "patterns.ini";
-  },
-
-  /**
-   * Will be set to true if no patterns.ini file exists.
-   * @type {boolean}
-   */
-  firstRun: false,
-
-  /**
-   * Map of properties listed in the filter storage file before the sections
-   * start. Right now this should be only the format version.
-   */
-  fileProperties: Object.create(null),
-
-  /**
-   * List of filter subscriptions containing all filters
-   * @type {Subscription[]}
-   */
-  subscriptions: [],
-
-  /**
-   * Map of subscriptions already on the list, by their URL/identifier
-   * @type {Object}
-   */
-  knownSubscriptions: Object.create(null),
-
-  /**
-   * Finds the filter group that a filter should be added to by default. Will
-   * return null if this group doesn't exist yet.
-   * @param {Filter} filter
-   * @return {?SpecialSubscription}
-   */
-  getGroupForFilter(filter)
-  {
-    let generalSubscription = null;
-    for (let subscription of FilterStorage.subscriptions)
+    for (let i = 0, l = FilterStorage.subscriptionCount; i < l; i++)
     {
-      if (subscription instanceof SpecialSubscription && !subscription.disabled)
+      let subscription = FilterStorage.subscriptionAt(i);
+      try
       {
-        // Always prefer specialized subscriptions
-        if (subscription.isDefaultFor(filter))
-          return subscription;
-
-        // If this is a general subscription - store it as fallback
-        if (!generalSubscription &&
-            (!subscription.defaults || !subscription.defaults.length))
-        {
-          generalSubscription = subscription;
-        }
-      }
-    }
-    return generalSubscription;
-  },
-
-  /**
-   * Adds a filter subscription to the list
-   * @param {Subscription} subscription filter subscription to be added
-   */
-  addSubscription(subscription)
-  {
-    if (subscription.url in FilterStorage.knownSubscriptions)
-      return;
-
-    FilterStorage.subscriptions.push(subscription);
-    FilterStorage.knownSubscriptions[subscription.url] = subscription;
-    addSubscriptionFilters(subscription);
-
-    FilterNotifier.triggerListeners("subscription.added", subscription);
-  },
-
-  /**
-   * Removes a filter subscription from the list
-   * @param {Subscription} subscription filter subscription to be removed
-   */
-  removeSubscription(subscription)
-  {
-    for (let i = 0; i < FilterStorage.subscriptions.length; i++)
-    {
-      if (FilterStorage.subscriptions[i].url == subscription.url)
-      {
-        removeSubscriptionFilters(subscription);
-
-        FilterStorage.subscriptions.splice(i--, 1);
-        delete FilterStorage.knownSubscriptions[subscription.url];
-        FilterNotifier.triggerListeners("subscription.removed", subscription);
-        return;
+        yield subscription;
       }
-    }
-  },
-
-  /**
-   * Moves a subscription in the list to a new position.
-   * @param {Subscription} subscription filter subscription to be moved
-   * @param {Subscription} [insertBefore] filter subscription to insert before
-   *        (if omitted the subscription will be put at the end of the list)
-   */
-  moveSubscription(subscription, insertBefore)
-  {
-    let currentPos = FilterStorage.subscriptions.indexOf(subscription);
-    if (currentPos < 0)
-      return;
-
-    let newPos = -1;
-    if (insertBefore)
-      newPos = FilterStorage.subscriptions.indexOf(insertBefore);
-
-    if (newPos < 0)
-      newPos = FilterStorage.subscriptions.length;
-
-    if (currentPos < newPos)
-      newPos--;
-    if (currentPos == newPos)
-      return;
-
-    FilterStorage.subscriptions.splice(currentPos, 1);
-    FilterStorage.subscriptions.splice(newPos, 0, subscription);
-    FilterNotifier.triggerListeners("subscription.moved", subscription);
-  },
-
-  /**
-   * Replaces the list of filters in a subscription by a new list
-   * @param {Subscription} subscription filter subscription to be updated
-   * @param {Filter[]} filters new filter list
-   */
-  updateSubscriptionFilters(subscription, filters)
-  {
-    removeSubscriptionFilters(subscription);
-    subscription.oldFilters = subscription.filters;
-    subscription.filters = filters;
-    addSubscriptionFilters(subscription);
-    FilterNotifier.triggerListeners("subscription.updated", subscription);
-    delete subscription.oldFilters;
-  },
-
-  /**
-   * Adds a user-defined filter to the list
-   * @param {Filter} filter
-   * @param {SpecialSubscription} [subscription]
-   *   particular group that the filter should be added to
-   * @param {number} [position]
-   *   position within the subscription at which the filter should be added
-   */
-  addFilter(filter, subscription, position)
-  {
-    if (!subscription)
-    {
-      if (filter.subscriptions.some(s => s instanceof SpecialSubscription &&
-                                         !s.disabled))
+      finally
       {
-        return;   // No need to add
-      }
-      subscription = FilterStorage.getGroupForFilter(filter);
-    }
-    if (!subscription)
-    {
-      // No group for this filter exists, create one
-      subscription = SpecialSubscription.createForFilter(filter);
-      this.addSubscription(subscription);
-      return;
-    }
-
-    if (typeof position == "undefined")
-      position = subscription.filters.length;
-
-    if (filter.subscriptions.indexOf(subscription) < 0)
-      filter.subscriptions.push(subscription);
-    subscription.filters.splice(position, 0, filter);
-    FilterNotifier.triggerListeners("filter.added", filter, subscription,
-                                    position);
-  },
-
-  /**
-   * Removes a user-defined filter from the list
-   * @param {Filter} filter
-   * @param {SpecialSubscription} [subscription] a particular filter group that
-   *      the filter should be removed from (if ommited will be removed from all
-   *      subscriptions)
-   * @param {number} [position]  position inside the filter group at which the
-   *      filter should be removed (if ommited all instances will be removed)
-   */
-  removeFilter(filter, subscription, position)
-  {
-    let subscriptions = (
-      subscription ? [subscription] : filter.subscriptions.slice()
-    );
-    for (let i = 0; i < subscriptions.length; i++)
-    {
-      let currentSubscription = subscriptions[i];
-      if (currentSubscription instanceof SpecialSubscription)
-      {
-        let positions = [];
-        if (typeof position == "undefined")
-        {
-          let index = -1;
-          do
-          {
-            index = currentSubscription.filters.indexOf(filter, index + 1);
-            if (index >= 0)
-              positions.push(index);
-          } while (index >= 0);
-        }
-        else
-          positions.push(position);
-
-        for (let j = positions.length - 1; j >= 0; j--)
-        {
-          let currentPosition = positions[j];
-          if (currentSubscription.filters[currentPosition] == filter)
-          {
-            currentSubscription.filters.splice(currentPosition, 1);
-            if (currentSubscription.filters.indexOf(filter) < 0)
-            {
-              let index = filter.subscriptions.indexOf(currentSubscription);
-              if (index >= 0)
-                filter.subscriptions.splice(index, 1);
-            }
-            FilterNotifier.triggerListeners(
-              "filter.removed", filter, currentSubscription, currentPosition
-            );
-          }
-        }
+        subscription.delete();
       }
     }
-  },
-
-  /**
-   * Moves a user-defined filter to a new position
-   * @param {Filter} filter
-   * @param {SpecialSubscription} subscription filter group where the filter is
-   *                                           located
-   * @param {number} oldPosition current position of the filter
-   * @param {number} newPosition new position of the filter
-   */
-  moveFilter(filter, subscription, oldPosition, newPosition)
-  {
-    if (!(subscription instanceof SpecialSubscription) ||
-        subscription.filters[oldPosition] != filter)
-    {
-      return;
-    }
-
-    newPosition = Math.min(Math.max(newPosition, 0),
-                           subscription.filters.length - 1);
-    if (oldPosition == newPosition)
-      return;
-
-    subscription.filters.splice(oldPosition, 1);
-    subscription.filters.splice(newPosition, 0, filter);
-    FilterNotifier.triggerListeners("filter.moved", filter, subscription,
-                                    oldPosition, newPosition);
-  },
-
-  /**
-   * Increases the hit count for a filter by one
-   * @param {Filter} filter
-   */
-  increaseHitCount(filter)
-  {
-    if (!Prefs.savestats || !(filter instanceof ActiveFilter))
-      return;
-
-    filter.hitCount++;
-    filter.lastHit = Date.now();
-  },
-
-  /**
-   * Resets hit count for some filters
-   * @param {Filter[]} filters  filters to be reset, if null all filters will
-   *                            be reset
-   */
-  resetHitCounts(filters)
-  {
-    if (!filters)
-    {
-      filters = [];
-      for (let text in Filter.knownFilters)
-        filters.push(Filter.knownFilters[text]);
-    }
-    for (let filter of filters)
-    {
-      filter.hitCount = 0;
-      filter.lastHit = 0;
-    }
-  },
-
-  /**
-   * @callback TextSink
-   * @param {string?} line
-   */
-
-  /**
-   * Allows importing previously serialized filter data.
-   * @param {boolean} silent
-   *    If true, no "load" notification will be sent out.
-   * @return {TextSink}
-   *    Function to be called for each line of data. Calling it with null as
-   *    parameter finalizes the import and replaces existing data. No changes
-   *    will be applied before finalization, so import can be "aborted" by
-   *    forgetting this callback.
-   */
-  importData(silent)
-  {
-    let parser = new INIParser();
-    return line =>
-    {
-      parser.process(line);
-      if (line === null)
-      {
-        let knownSubscriptions = Object.create(null);
-        for (let subscription of parser.subscriptions)
-          knownSubscriptions[subscription.url] = subscription;
-
-        this.fileProperties = parser.fileProperties;
-        this.subscriptions = parser.subscriptions;
-        this.knownSubscriptions = knownSubscriptions;
-        Filter.knownFilters = parser.knownFilters;
-        Subscription.knownSubscriptions = parser.knownSubscriptions;
-
-        if (!silent)
-          FilterNotifier.triggerListeners("load");
-      }
-    };
-  },
-
-  /**
-   * Loads all subscriptions from the disk.
-   * @return {Promise} promise resolved or rejected when loading is complete
-   */
-  loadFromDisk()
-  {
-    let tryBackup = backupIndex =>
-    {
-      return this.restoreBackup(backupIndex, true).then(() =>
-      {
-        if (this.subscriptions.length == 0)
-          return tryBackup(backupIndex + 1);
-      }).catch(error =>
-      {
-        // Give up
-      });
-    };
-
-    return IO.statFile(this.sourceFile).then(statData =>
-    {
-      if (!statData.exists)
-      {
-        this.firstRun = true;
-        return;
-      }
-
-      let parser = this.importData(true);
-      return IO.readFromFile(this.sourceFile, parser).then(() =>
-      {
-        parser(null);
-        if (this.subscriptions.length == 0)
-        {
-          // No filter subscriptions in the file, this isn't right.
-          throw new Error("No data in the file");
-        }
-      });
-    }).catch(error =>
-    {
-      Cu.reportError(error);
-      return tryBackup(1);
-    }).then(() =>
-    {
-      this.initialized = true;
-      FilterNotifier.triggerListeners("load");
-    });
-  },
-
-  /**
-   * Constructs the file name for a patterns.ini backup.
-   * @param {number} backupIndex
-   *    number of the backup file (1 being the most recent)
-   * @return {string} backup file name
-   */
-  getBackupName(backupIndex)
-  {
-    let [name, extension] = this.sourceFile.split(".", 2);
-    return (name + "-backup" + backupIndex + "." + extension);
-  },
-
-  /**
-   * Restores an automatically created backup.
-   * @param {number} backupIndex
-   *    number of the backup to restore (1 being the most recent)
-   * @param {boolean} silent
-   *    If true, no "load" notification will be sent out.
-   * @return {Promise} promise resolved or rejected when restoring is complete
-   */
-  restoreBackup(backupIndex, silent)
-  {
-    let backupFile = this.getBackupName(backupIndex);
-    let parser = this.importData(silent);
-    return IO.readFromFile(backupFile, parser).then(() =>
-    {
-      parser(null);
-      return this.saveToDisk();
-    });
-  },
-
-  /**
-   * Generator serializing filter data and yielding it line by line.
-   */
-  *exportData()
-  {
-    // Do not persist external subscriptions
-    let subscriptions = this.subscriptions.filter(
-      s => !(s instanceof ExternalSubscription)
-    );
-
-    yield "# Adblock Plus preferences";
-    yield "version=" + formatVersion;
-
-    let saved = new Set();
-    let buf = [];
-
-    // Save subscriptions
-    for (let subscription of subscriptions)
-    {
-      yield "";
-
-      subscription.serialize(buf);
-      if (subscription.filters.length)
-      {
-        buf.push("", "[Subscription filters]");
-        subscription.serializeFilters(buf);
-      }
-      for (let line of buf)
-        yield line;
-      buf.splice(0);
-    }
-
-    // Save filter data
-    for (let subscription of subscriptions)
-    {
-      for (let filter of subscription.filters)
-      {
-        if (!saved.has(filter.text))
-        {
-          filter.serialize(buf);
-          saved.add(filter.text);
-          for (let line of buf)
-            yield line;
-          buf.splice(0);
-        }
-      }
-    }
-  },
-
-  /**
-   * Will be set to true if saveToDisk() is running (reentrance protection).
-   * @type {boolean}
-   */
-  _saving: false,
-
-  /**
-   * Will be set to true if a saveToDisk() call arrives while saveToDisk() is
-   * already running (delayed execution).
-   * @type {boolean}
-   */
-  _needsSave: false,
-
-  /**
-   * Saves all subscriptions back to disk
-   * @return {Promise} promise resolved or rejected when saving is complete
-   */
-  saveToDisk()
-  {
-    if (this._saving)
-    {
-      this._needsSave = true;
-      return;
-    }
-
-    this._saving = true;
-
-    return Promise.resolve().then(() =>
-    {
-      // First check whether we need to create a backup
-      if (Prefs.patternsbackups <= 0)
-        return false;
-
-      return IO.statFile(this.sourceFile).then(statData =>
-      {
-        if (!statData.exists)
-          return false;
-
-        return IO.statFile(this.getBackupName(1)).then(backupStatData =>
-        {
-          if (backupStatData.exists &&
-              (Date.now() - backupStatData.lastModified) / 3600000 <
-                Prefs.patternsbackupinterval)
-          {
-            return false;
-          }
-          return true;
-        });
-      });
-    }).then(backupRequired =>
-    {
-      if (!backupRequired)
-        return;
-
-      let ignoreErrors = error =>
-      {
-        // Expected error, backup file doesn't exist.
-      };
-
-      let renameBackup = index =>
-      {
-        if (index > 0)
-        {
-          return IO.renameFile(this.getBackupName(index),
-                               this.getBackupName(index + 1))
-                   .catch(ignoreErrors)
-                   .then(() => renameBackup(index - 1));
-        }
-
-        return IO.renameFile(this.sourceFile, this.getBackupName(1))
-                 .catch(ignoreErrors);
-      };
-
-      // Rename existing files
-      return renameBackup(Prefs.patternsbackups - 1);
-    }).catch(error =>
-    {
-      // Errors during backup creation shouldn't prevent writing filters.
-      Cu.reportError(error);
-    }).then(() =>
-    {
-      return IO.writeToFile(this.sourceFile, this.exportData());
-    }).then(() =>
-    {
-      FilterNotifier.triggerListeners("save");
-    }).catch(error =>
-    {
-      // If saving failed, report error but continue - we still have to process
-      // flags.
-      Cu.reportError(error);
-    }).then(() =>
-    {
-      this._saving = false;
-      if (this._needsSave)
-      {
-        this._needsSave = false;
-        this.saveToDisk();
-      }
-    });
-  },
-
-  /**
-   * @typedef FileInfo
-   * @type {object}
-   * @property {nsIFile} file
-   * @property {number} lastModified
-   */
-
-  /**
-   * Returns a promise resolving in a list of existing backup files.
-   * @return {Promise.<FileInfo[]>}
-   */
-  getBackupFiles()
-  {
-    let backups = [];
-
-    let checkBackupFile = index =>
-    {
-      return IO.statFile(this.getBackupName(index)).then(statData =>
-      {
-        if (!statData.exists)
-          return backups;
-
-        backups.push({
-          index,
-          lastModified: statData.lastModified
-        });
-        return checkBackupFile(index + 1);
-      }).catch(error =>
-      {
-        // Something went wrong, return whatever data we got so far.
-        Cu.reportError(error);
-        return backups;
-      });
-    };
-
-    return checkBackupFile(1);
   }
 };
 
 /**
- * Joins subscription's filters to the subscription without any notifications.
- * @param {Subscription} subscription
- *   filter subscription that should be connected to its filters
+ * Adds a user-defined filter to the most suitable subscription in the list,
+ * creates one if none found.
+ * @param {Filter} filter
  */
-function addSubscriptionFilters(subscription)
+FilterStorage.addFilter = function(filter)
 {
-  if (!(subscription.url in FilterStorage.knownSubscriptions))
-    return;
-
-  for (let filter of subscription.filters)
-    filter.subscriptions.push(subscription);
-}
+  for (let subscription of this.subscriptions)
+    if (!subscription.disabled && subscription.indexOfFilter(filter) >= 0)
+      return;
 
-/**
- * Removes subscription's filters from the subscription without any
- * notifications.
- * @param {Subscription} subscription filter subscription to be removed
- */
-function removeSubscriptionFilters(subscription)
-{
-  if (!(subscription.url in FilterStorage.knownSubscriptions))
-    return;
-
-  for (let filter of subscription.filters)
+  let subscription = this.getSubscriptionForFilter(filter);
+  if (!subscription)
   {
-    let i = filter.subscriptions.indexOf(subscription);
-    if (i >= 0)
-      filter.subscriptions.splice(i, 1);
+    subscription = Subscription.fromURL(null);
+    subscription.makeDefaultFor(filter);
+    this.addSubscription(subscription);
   }
-}
+  subscription.insertFilterAt(filter, subscription.filterCount);
+  subscription.delete();

[sergei, 2017/05/08 10:54:39]

strictly speaking it should be in finally block, but I'm not sure that we follow
that rule.

[Wladimir Palant, 2017/05/08 12:55:41]

You are correct. I tend to be too comfortable for unit test code, but for
production code we should get used to that pattern.

+  return true;
+};
 
 /**
- * Listener returned by FilterStorage.importData(), parses filter data.
- * @constructor
+ * Removes a user-defined filter from the list
+ * @param {Filter} filter
  */
-function INIParser()
-{
-  this.fileProperties = this.curObj = {};
-  this.subscriptions = [];
-  this.knownFilters = Object.create(null);
-  this.knownSubscriptions = Object.create(null);
-}
-INIParser.prototype =
+FilterStorage.removeFilter = function(filter)
 {
-  linesProcessed: 0,
-  subscriptions: null,
-  knownFilters: null,
-  knownSubscriptions: null,
-  wantObj: true,
-  fileProperties: null,
-  curObj: null,
-  curSection: null,
-
-  process(val)
+  for (let subscription of this.subscriptions)
   {
-    let origKnownFilters = Filter.knownFilters;
-    Filter.knownFilters = this.knownFilters;
-    let origKnownSubscriptions = Subscription.knownSubscriptions;
-    Subscription.knownSubscriptions = this.knownSubscriptions;
-    let match;
-    try
+    if (subscription instanceof SpecialSubscription)
     {
-      if (this.wantObj === true && (match = /^(\w+)=(.*)$/.exec(val)))
-        this.curObj[match[1]] = match[2];
-      else if (val === null || (match = /^\s*\[(.+)\]\s*$/.exec(val)))
+      while (true)

[sergei, 2017/05/08 10:54:39]

Is this loop to remove duplicates of the filter in the same subscription? I
guess that a subscription is a set of unique filters, isn't it?

[Wladimir Palant, 2017/05/08 12:55:41]

No, a subscription can contain the same filter multiple times, we don't impose
any such restrictions (any more, since around Adblock Plus 0.7). This is
particularly important for comments, filter list authors tend to use comments as
separators - and these are identical then.

       {
-        if (this.curObj)
-        {
-          // Process current object before going to next section
-          switch (this.curSection)
-          {
-            case "filter":
-              if ("text" in this.curObj)
-                Filter.fromObject(this.curObj);
-              break;
-            case "subscription": {
-              let subscription = Subscription.fromObject(this.curObj);
-              if (subscription)
-                this.subscriptions.push(subscription);
-              break;
-            }
-            case "subscription filters":
-              if (this.subscriptions.length)
-              {
-                let subscription = this.subscriptions[
-                  this.subscriptions.length - 1
-                ];
-                for (let text of this.curObj)
-                {
-                  let filter = Filter.fromText(text);
-                  subscription.filters.push(filter);
-                  filter.subscriptions.push(subscription);
-                }
-              }
-              break;
-          }
-        }
-
-        if (val === null)
-          return;
-
-        this.curSection = match[1].toLowerCase();
-        switch (this.curSection)
-        {
-          case "filter":
-          case "subscription":
-            this.wantObj = true;
-            this.curObj = {};
-            break;
-          case "subscription filters":
-            this.wantObj = false;
-            this.curObj = [];
-            break;
-          default:
-            this.wantObj = undefined;
-            this.curObj = null;
-        }
+        let index = subscription.indexOfFilter(filter);
+        if (index >= 0)
+          subscription.removeFilterAt(index);
+        else
+          break;
       }
-      else if (this.wantObj === false && val)
-        this.curObj.push(val.replace(/\\\[/g, "["));
-    }
-    finally
-    {
-      Filter.knownFilters = origKnownFilters;
-      Subscription.knownSubscriptions = origKnownSubscriptions;
     }
   }
 };
+
+exports.FilterStorage = FilterStorage;