Issue 5137 - [emscripten] Added basic filter storage implementation

lib/filterStorage.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-present eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 "use strict";
 
-/**
- * @fileOverview FilterStorage class responsible for managing user's
- *               subscriptions and filters.
- */
+const {FilterStorage} = require("compiled");
+const {Subscription, SpecialSubscription} = require("subscriptionClasses");
+const {ActiveFilter} = require("filterClasses");
 
-const {IO} = require("io");
-const {Prefs} = require("prefs");
-const {Filter, ActiveFilter} = require("filterClasses");
-const {Subscription, SpecialSubscription,
-       ExternalSubscription} = require("subscriptionClasses");
-const {FilterNotifier} = require("filterNotifier");
+// Backwards compatibility
+FilterStorage.getGroupForFilter = FilterStorage.getSubscriptionForFilter;
 
 /**
- * Version number of the filter storage file format.
- * @type {number}
+ * This property allows iterating over the list of subscriptions. It will delete
+ * references automatically at the end of the current loop iteration. If you
+ * need persistent references or element access by position you should use
+ * FilterStorage.subscriptionAt() instead.
+ * @type {Iterable}
  */
-let formatVersion = 5;

[sergei, 2017/08/24 13:32:07]

I'm not sure that it's a good idea to already remove the parsing code.

[Wladimir Palant, 2017/08/31 11:32:36]

I can leave it in of course, but that would be merely a whole lot of dead code.
And I would have to mess with that dead code because the FilterStorage object is
now being defined elsewhere. Are you sure that you want it?

-
-/**
- * This class reads user's filters from disk, manages them in memory
- * and writes them back.
- * @class
- */
-let FilterStorage = exports.FilterStorage =
-{
-  /**
-   * Will be set to true after the initial loadFromDisk() call completes.
-   * @type {boolean}
-   */
-  initialized: false,
-
-  /**
-   * Version number of the patterns.ini format used.
-   * @type {number}
-   */
-  get formatVersion()
+FilterStorage.subscriptions = {
+  [Symbol.iterator]: function*()
   {
-    return formatVersion;
-  },
-
-  /**
-   * File containing the filter list
-   * @type {string}
-   */
-  get sourceFile()
-  {
-    return "patterns.ini";
-  },
-
-  /**
-   * Will be set to true if no patterns.ini file exists.
-   * @type {boolean}
-   */
-  firstRun: false,
-
-  /**
-   * Map of properties listed in the filter storage file before the sections
-   * start. Right now this should be only the format version.
-   */
-  fileProperties: Object.create(null),
-
-  /**
-   * List of filter subscriptions containing all filters
-   * @type {Subscription[]}
-   */
-  subscriptions: [],
-
-  /**
-   * Map of subscriptions already on the list, by their URL/identifier
-   * @type {Object}
-   */
-  knownSubscriptions: Object.create(null),
-
-  /**
-   * Finds the filter group that a filter should be added to by default. Will
-   * return null if this group doesn't exist yet.
-   * @param {Filter} filter
-   * @return {?SpecialSubscription}
-   */
-  getGroupForFilter(filter)
-  {
-    let generalSubscription = null;
-    for (let subscription of FilterStorage.subscriptions)
+    for (let i = 0, l = FilterStorage.subscriptionCount; i < l; i++)
     {
-      if (subscription instanceof SpecialSubscription && !subscription.disabled)
+      let subscription = FilterStorage.subscriptionAt(i);
+      try
       {
-        // Always prefer specialized subscriptions
-        if (subscription.isDefaultFor(filter))
-          return subscription;
-
-        // If this is a general subscription - store it as fallback
-        if (!generalSubscription &&
-            (!subscription.defaults || !subscription.defaults.length))
-        {
-          generalSubscription = subscription;
-        }
+        yield subscription;
+      }
+      finally
+      {
+        subscription.delete();
       }
     }
-    return generalSubscription;
-  },
-
-  /**
-   * Adds a filter subscription to the list
-   * @param {Subscription} subscription filter subscription to be added
-   */
-  addSubscription(subscription)
-  {
-    if (subscription.url in FilterStorage.knownSubscriptions)
-      return;
-
-    FilterStorage.subscriptions.push(subscription);
-    FilterStorage.knownSubscriptions[subscription.url] = subscription;
-    addSubscriptionFilters(subscription);
-
-    FilterNotifier.triggerListeners("subscription.added", subscription);
-  },
-
-  /**
-   * Removes a filter subscription from the list
-   * @param {Subscription} subscription filter subscription to be removed
-   */
-  removeSubscription(subscription)
-  {
-    for (let i = 0; i < FilterStorage.subscriptions.length; i++)
-    {
-      if (FilterStorage.subscriptions[i].url == subscription.url)
-      {
-        removeSubscriptionFilters(subscription);
-
-        FilterStorage.subscriptions.splice(i--, 1);
-        delete FilterStorage.knownSubscriptions[subscription.url];
-        FilterNotifier.triggerListeners("subscription.removed", subscription);
-        return;
-      }
-    }
-  },
-
-  /**
-   * Moves a subscription in the list to a new position.
-   * @param {Subscription} subscription filter subscription to be moved
-   * @param {Subscription} [insertBefore] filter subscription to insert before
-   *        (if omitted the subscription will be put at the end of the list)
-   */
-  moveSubscription(subscription, insertBefore)
-  {
-    let currentPos = FilterStorage.subscriptions.indexOf(subscription);
-    if (currentPos < 0)
-      return;
-
-    let newPos = -1;
-    if (insertBefore)
-      newPos = FilterStorage.subscriptions.indexOf(insertBefore);
-
-    if (newPos < 0)
-      newPos = FilterStorage.subscriptions.length;
-
-    if (currentPos < newPos)
-      newPos--;
-    if (currentPos == newPos)
-      return;
-
-    FilterStorage.subscriptions.splice(currentPos, 1);
-    FilterStorage.subscriptions.splice(newPos, 0, subscription);
-    FilterNotifier.triggerListeners("subscription.moved", subscription);
-  },
-
-  /**
-   * Replaces the list of filters in a subscription by a new list
-   * @param {Subscription} subscription filter subscription to be updated
-   * @param {Filter[]} filters new filter list
-   */
-  updateSubscriptionFilters(subscription, filters)
-  {
-    removeSubscriptionFilters(subscription);
-    subscription.oldFilters = subscription.filters;
-    subscription.filters = filters;
-    addSubscriptionFilters(subscription);
-    FilterNotifier.triggerListeners("subscription.updated", subscription);
-    delete subscription.oldFilters;
-  },
-
-  /**
-   * Adds a user-defined filter to the list
-   * @param {Filter} filter
-   * @param {SpecialSubscription} [subscription]
-   *   particular group that the filter should be added to
-   * @param {number} [position]
-   *   position within the subscription at which the filter should be added
-   */
-  addFilter(filter, subscription, position)
-  {
-    if (!subscription)
-    {
-      if (filter.subscriptions.some(s => s instanceof SpecialSubscription &&
-                                         !s.disabled))
-      {
-        return;   // No need to add
-      }
-      subscription = FilterStorage.getGroupForFilter(filter);
-    }
-    if (!subscription)
-    {
-      // No group for this filter exists, create one
-      subscription = SpecialSubscription.createForFilter(filter);
-      this.addSubscription(subscription);
-      return;
-    }
-
-    if (typeof position == "undefined")
-      position = subscription.filters.length;
-
-    if (filter.subscriptions.indexOf(subscription) < 0)
-      filter.subscriptions.push(subscription);
-    subscription.filters.splice(position, 0, filter);
-    FilterNotifier.triggerListeners("filter.added", filter, subscription,
-                                    position);
-  },
-
-  /**
-   * Removes a user-defined filter from the list
-   * @param {Filter} filter
-   * @param {SpecialSubscription} [subscription] a particular filter group that
-   *      the filter should be removed from (if ommited will be removed from all
-   *      subscriptions)
-   * @param {number} [position]  position inside the filter group at which the
-   *      filter should be removed (if ommited all instances will be removed)
-   */
-  removeFilter(filter, subscription, position)
-  {
-    let subscriptions = (
-      subscription ? [subscription] : filter.subscriptions.slice()
-    );
-    for (let i = 0; i < subscriptions.length; i++)
-    {
-      let currentSubscription = subscriptions[i];
-      if (currentSubscription instanceof SpecialSubscription)
-      {
-        let positions = [];
-        if (typeof position == "undefined")
-        {
-          let index = -1;
-          do
-          {
-            index = currentSubscription.filters.indexOf(filter, index + 1);
-            if (index >= 0)
-              positions.push(index);
-          } while (index >= 0);
-        }
-        else
-          positions.push(position);
-
-        for (let j = positions.length - 1; j >= 0; j--)
-        {
-          let currentPosition = positions[j];
-          if (currentSubscription.filters[currentPosition] == filter)
-          {
-            currentSubscription.filters.splice(currentPosition, 1);
-            if (currentSubscription.filters.indexOf(filter) < 0)
-            {
-              let index = filter.subscriptions.indexOf(currentSubscription);
-              if (index >= 0)
-                filter.subscriptions.splice(index, 1);
-            }
-            FilterNotifier.triggerListeners(
-              "filter.removed", filter, currentSubscription, currentPosition
-            );
-          }
-        }
-      }
-    }
-  },
-
-  /**
-   * Moves a user-defined filter to a new position
-   * @param {Filter} filter
-   * @param {SpecialSubscription} subscription filter group where the filter is
-   *                                           located
-   * @param {number} oldPosition current position of the filter
-   * @param {number} newPosition new position of the filter
-   */
-  moveFilter(filter, subscription, oldPosition, newPosition)
-  {
-    if (!(subscription instanceof SpecialSubscription) ||
-        subscription.filters[oldPosition] != filter)
-    {
-      return;
-    }
-
-    newPosition = Math.min(Math.max(newPosition, 0),
-                           subscription.filters.length - 1);
-    if (oldPosition == newPosition)
-      return;
-
-    subscription.filters.splice(oldPosition, 1);
-    subscription.filters.splice(newPosition, 0, filter);
-    FilterNotifier.triggerListeners("filter.moved", filter, subscription,
-                                    oldPosition, newPosition);
-  },
-
-  /**
-   * Increases the hit count for a filter by one
-   * @param {Filter} filter
-   */
-  increaseHitCount(filter)
-  {
-    if (!Prefs.savestats || !(filter instanceof ActiveFilter))
-      return;
-
-    filter.hitCount++;
-    filter.lastHit = Date.now();
-  },
-
-  /**
-   * Resets hit count for some filters
-   * @param {Filter[]} filters  filters to be reset, if null all filters will
-   *                            be reset
-   */
-  resetHitCounts(filters)
-  {
-    if (!filters)
-    {
-      filters = [];
-      for (let text in Filter.knownFilters)
-        filters.push(Filter.knownFilters[text]);
-    }
-    for (let filter of filters)
-    {
-      filter.hitCount = 0;
-      filter.lastHit = 0;
-    }
-  },
-
-  /**
-   * @callback TextSink
-   * @param {string?} line
-   */
-
-  /**
-   * Allows importing previously serialized filter data.
-   * @param {boolean} silent
-   *    If true, no "load" notification will be sent out.
-   * @return {TextSink}
-   *    Function to be called for each line of data. Calling it with null as
-   *    parameter finalizes the import and replaces existing data. No changes
-   *    will be applied before finalization, so import can be "aborted" by
-   *    forgetting this callback.
-   */
-  importData(silent)
-  {
-    let parser = new INIParser();
-    return line =>
-    {
-      parser.process(line);
-      if (line === null)
-      {
-        let knownSubscriptions = Object.create(null);
-        for (let subscription of parser.subscriptions)
-          knownSubscriptions[subscription.url] = subscription;
-
-        this.fileProperties = parser.fileProperties;
-        this.subscriptions = parser.subscriptions;
-        this.knownSubscriptions = knownSubscriptions;
-        Filter.knownFilters = parser.knownFilters;
-        Subscription.knownSubscriptions = parser.knownSubscriptions;
-
-        if (!silent)
-          FilterNotifier.triggerListeners("load");
-      }
-    };
-  },
-
-  /**
-   * Loads all subscriptions from the disk.
-   * @return {Promise} promise resolved or rejected when loading is complete
-   */
-  loadFromDisk()
-  {
-    let tryBackup = backupIndex =>
-    {
-      return this.restoreBackup(backupIndex, true).then(() =>
-      {
-        if (this.subscriptions.length == 0)
-          return tryBackup(backupIndex + 1);
-      }).catch(error =>
-      {
-        // Give up
-      });
-    };
-
-    return IO.statFile(this.sourceFile).then(statData =>
-    {
-      if (!statData.exists)
-      {
-        this.firstRun = true;
-        return;
-      }
-
-      let parser = this.importData(true);
-      return IO.readFromFile(this.sourceFile, parser).then(() =>
-      {
-        parser(null);
-        if (this.subscriptions.length == 0)
-        {
-          // No filter subscriptions in the file, this isn't right.
-          throw new Error("No data in the file");
-        }
-      });
-    }).catch(error =>
-    {
-      Cu.reportError(error);
-      return tryBackup(1);
-    }).then(() =>
-    {
-      this.initialized = true;
-      FilterNotifier.triggerListeners("load");
-    });
-  },
-
-  /**
-   * Constructs the file name for a patterns.ini backup.
-   * @param {number} backupIndex
-   *    number of the backup file (1 being the most recent)
-   * @return {string} backup file name
-   */
-  getBackupName(backupIndex)
-  {
-    let [name, extension] = this.sourceFile.split(".", 2);
-    return (name + "-backup" + backupIndex + "." + extension);
-  },
-
-  /**
-   * Restores an automatically created backup.
-   * @param {number} backupIndex
-   *    number of the backup to restore (1 being the most recent)
-   * @param {boolean} silent
-   *    If true, no "load" notification will be sent out.
-   * @return {Promise} promise resolved or rejected when restoring is complete
-   */
-  restoreBackup(backupIndex, silent)
-  {
-    let backupFile = this.getBackupName(backupIndex);
-    let parser = this.importData(silent);
-    return IO.readFromFile(backupFile, parser).then(() =>
-    {
-      parser(null);
-      return this.saveToDisk();
-    });
-  },
-
-  /**
-   * Generator serializing filter data and yielding it line by line.
-   */
-  *exportData()
-  {
-    // Do not persist external subscriptions
-    let subscriptions = this.subscriptions.filter(
-      s => !(s instanceof ExternalSubscription)
-    );
-
-    yield "# Adblock Plus preferences";
-    yield "version=" + formatVersion;
-
-    let saved = new Set();
-    let buf = [];
-
-    // Save subscriptions
-    for (let subscription of subscriptions)
-    {
-      yield "";
-
-      subscription.serialize(buf);
-      if (subscription.filters.length)
-      {
-        buf.push("", "[Subscription filters]");
-        subscription.serializeFilters(buf);
-      }
-      for (let line of buf)
-        yield line;
-      buf.splice(0);
-    }
-
-    // Save filter data
-    for (let subscription of subscriptions)
-    {
-      for (let filter of subscription.filters)
-      {
-        if (!saved.has(filter.text))
-        {
-          filter.serialize(buf);
-          saved.add(filter.text);
-          for (let line of buf)
-            yield line;
-          buf.splice(0);
-        }
-      }
-    }
-  },
-
-  /**
-   * Will be set to true if saveToDisk() is running (reentrance protection).
-   * @type {boolean}
-   */
-  _saving: false,
-
-  /**
-   * Will be set to true if a saveToDisk() call arrives while saveToDisk() is
-   * already running (delayed execution).
-   * @type {boolean}
-   */
-  _needsSave: false,
-
-  /**
-   * Saves all subscriptions back to disk
-   * @return {Promise} promise resolved or rejected when saving is complete
-   */
-  saveToDisk()
-  {
-    if (this._saving)
-    {
-      this._needsSave = true;
-      return;
-    }
-
-    this._saving = true;
-
-    return Promise.resolve().then(() =>
-    {
-      // First check whether we need to create a backup
-      if (Prefs.patternsbackups <= 0)
-        return false;
-
-      return IO.statFile(this.sourceFile).then(statData =>
-      {
-        if (!statData.exists)
-          return false;
-
-        return IO.statFile(this.getBackupName(1)).then(backupStatData =>
-        {
-          if (backupStatData.exists &&
-              (Date.now() - backupStatData.lastModified) / 3600000 <
-                Prefs.patternsbackupinterval)
-          {
-            return false;
-          }
-          return true;
-        });
-      });
-    }).then(backupRequired =>
-    {
-      if (!backupRequired)
-        return;
-
-      let ignoreErrors = error =>
-      {
-        // Expected error, backup file doesn't exist.
-      };
-
-      let renameBackup = index =>
-      {
-        if (index > 0)
-        {
-          return IO.renameFile(this.getBackupName(index),
-                               this.getBackupName(index + 1))
-                   .catch(ignoreErrors)
-                   .then(() => renameBackup(index - 1));
-        }
-
-        return IO.renameFile(this.sourceFile, this.getBackupName(1))
-                 .catch(ignoreErrors);
-      };
-
-      // Rename existing files
-      return renameBackup(Prefs.patternsbackups - 1);
-    }).catch(error =>
-    {
-      // Errors during backup creation shouldn't prevent writing filters.
-      Cu.reportError(error);
-    }).then(() =>
-    {
-      return IO.writeToFile(this.sourceFile, this.exportData());
-    }).then(() =>
-    {
-      FilterNotifier.triggerListeners("save");
-    }).catch(error =>
-    {
-      // If saving failed, report error but continue - we still have to process
-      // flags.
-      Cu.reportError(error);
-    }).then(() =>
-    {
-      this._saving = false;
-      if (this._needsSave)
-      {
-        this._needsSave = false;
-        this.saveToDisk();
-      }
-    });
-  },
-
-  /**
-   * @typedef FileInfo
-   * @type {object}
-   * @property {nsIFile} file
-   * @property {number} lastModified
-   */
-
-  /**
-   * Returns a promise resolving in a list of existing backup files.
-   * @return {Promise.<FileInfo[]>}
-   */
-  getBackupFiles()
-  {
-    let backups = [];
-
-    let checkBackupFile = index =>
-    {
-      return IO.statFile(this.getBackupName(index)).then(statData =>
-      {
-        if (!statData.exists)
-          return backups;
-
-        backups.push({
-          index,
-          lastModified: statData.lastModified
-        });
-        return checkBackupFile(index + 1);
-      }).catch(error =>
-      {
-        // Something went wrong, return whatever data we got so far.
-        Cu.reportError(error);
-        return backups;
-      });
-    };
-
-    return checkBackupFile(1);
   }
 };
 
 /**
- * Joins subscription's filters to the subscription without any notifications.
- * @param {Subscription} subscription
- *   filter subscription that should be connected to its filters
+ * Adds a user-defined filter to the most suitable subscription in the list,
+ * creates one if none found.
+ * @param {Filter} filter
  */
-function addSubscriptionFilters(subscription)
+FilterStorage.addFilter = function(filter)
 {
-  if (!(subscription.url in FilterStorage.knownSubscriptions))
-    return;
+  for (let subscription of this.subscriptions)
+    if (!subscription.disabled && subscription.indexOfFilter(filter) >= 0)
+      return;
 
-  for (let filter of subscription.filters)
-    filter.subscriptions.push(subscription);
-}
+  let subscription = this.getSubscriptionForFilter(filter);
+  try
+  {
+    if (!subscription)
+    {
+      subscription = Subscription.fromURL(null);
+      subscription.makeDefaultFor(filter);
+      this.addSubscription(subscription);
+    }
+    subscription.insertFilterAt(filter, subscription.filterCount);
+  }
+  finally
+  {
+    if (subscription)
+      subscription.delete();
+  }
+  return true;
+};
 
 /**
- * Removes subscription's filters from the subscription without any
- * notifications.
- * @param {Subscription} subscription filter subscription to be removed
+ * Removes a user-defined filter from the list
+ * @param {Filter} filter
  */
-function removeSubscriptionFilters(subscription)
+FilterStorage.removeFilter = function(filter)
 {
-  if (!(subscription.url in FilterStorage.knownSubscriptions))
-    return;
-
-  for (let filter of subscription.filters)
+  for (let subscription of this.subscriptions)
   {
-    let i = filter.subscriptions.indexOf(subscription);
-    if (i >= 0)
-      filter.subscriptions.splice(i, 1);
-  }
-}
-
-/**
- * Listener returned by FilterStorage.importData(), parses filter data.
- * @constructor
- */
-function INIParser()
-{
-  this.fileProperties = this.curObj = {};
-  this.subscriptions = [];
-  this.knownFilters = Object.create(null);
-  this.knownSubscriptions = Object.create(null);
-}
-INIParser.prototype =
-{
-  linesProcessed: 0,
-  subscriptions: null,
-  knownFilters: null,
-  knownSubscriptions: null,
-  wantObj: true,
-  fileProperties: null,
-  curObj: null,
-  curSection: null,
-
-  process(val)
-  {
-    let origKnownFilters = Filter.knownFilters;
-    Filter.knownFilters = this.knownFilters;
-    let origKnownSubscriptions = Subscription.knownSubscriptions;
-    Subscription.knownSubscriptions = this.knownSubscriptions;
-    let match;
-    try
+    if (subscription instanceof SpecialSubscription)
     {
-      if (this.wantObj === true && (match = /^(\w+)=(.*)$/.exec(val)))
-        this.curObj[match[1]] = match[2];
-      else if (val === null || (match = /^\s*\[(.+)\]\s*$/.exec(val)))
+      while (true)
       {
-        if (this.curObj)
-        {
-          // Process current object before going to next section
-          switch (this.curSection)
-          {
-            case "filter":
-              if ("text" in this.curObj)
-                Filter.fromObject(this.curObj);
-              break;
-            case "subscription": {
-              let subscription = Subscription.fromObject(this.curObj);
-              if (subscription)
-                this.subscriptions.push(subscription);
-              break;
-            }
-            case "subscription filters":
-              if (this.subscriptions.length)
-              {
-                let subscription = this.subscriptions[
-                  this.subscriptions.length - 1
-                ];
-                for (let text of this.curObj)
-                {
-                  let filter = Filter.fromText(text);
-                  subscription.filters.push(filter);
-                  filter.subscriptions.push(subscription);
-                }
-              }
-              break;
-          }
-        }
-
-        if (val === null)
-          return;
-
-        this.curSection = match[1].toLowerCase();
-        switch (this.curSection)
-        {
-          case "filter":
-          case "subscription":
-            this.wantObj = true;
-            this.curObj = {};
-            break;
-          case "subscription filters":
-            this.wantObj = false;
-            this.curObj = [];
-            break;
-          default:
-            this.wantObj = undefined;
-            this.curObj = null;
-        }
+        let index = subscription.indexOfFilter(filter);
+        if (index >= 0)
+          subscription.removeFilterAt(index);
+        else
+          break;
       }
-      else if (this.wantObj === false && val)
-        this.curObj.push(val.replace(/\\\[/g, "["));
-    }
-    finally
-    {
-      Filter.knownFilters = origKnownFilters;
-      Subscription.knownSubscriptions = origKnownSubscriptions;
     }
   }
 };
+
+exports.FilterStorage = FilterStorage;