Issue 394 - hit statistics tool data collection

lib/filterHits.js

Index: lib/filterHits.js
===================================================================
new file mode 100644
--- /dev/null
+++ b/lib/filterHits.js
@@ -0,0 +1,323 @@
+/*
+ * This file is part of Adblock Plus <https://adblockplus.org/>,
+ * Copyright (C) 2006-2016 Eyeo GmbH
+ *
+ * Adblock Plus is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * Adblock Plus is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+let {Services} = Cu.import("resource://gre/modules/Services.jsm", null);
+let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null);
+let {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm", null);
+
+let {IO} = require("io");
+let {Prefs} = require("prefs");
+let {Downloader, Downloadable, MILLIS_IN_SECOND, MILLIS_IN_DAY, MILLIS_IN_HOUR} = require("downloader");
+let {FilterNotifier} = require("filterNotifier");
+let {DownloadableSubscription} = require("subscriptionClasses");
+
+const CHECK_INTERVAL = MILLIS_IN_DAY;
+const INITIAL_DELAY = MILLIS_IN_SECOND * 5;
+const PUSH_INTERVAL = MILLIS_IN_DAY * 7;
+
+/**
+ * The value of storage statement normal execution constant 
+ * @type Number
+ */
+const REASON_FINISHED = Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED;
+
+/**
+ * This class collects filter hits statistics in a SQLite database
+ * and sends them to the server when user opt-in for that
+ * @class
+ */
+let FilterHits = exports.FilterHits =
+{
+  /**
+   * Used to prevent timers running in parallel
+   * @type Number
+   */
+  _prefToggleTimeoutId: 0,
+
+  /**
+   * Indicates whether the data is being sent to the server 
+   * @type Boolean
+   */
+  _sending: false,
+
+  get connection()
+  {
+    let connection = Services.storage.openDatabase(this.getStorageFile);
+    if (!connection.tableExists("filtersubscriptions"))
+    {
+      connection.executeSimpleSQL("CREATE TABLE filtersubscriptions " +
+        "(id INTEGER, subscriptions TEXT, PRIMARY KEY" +
+        "(id), UNIQUE(subscriptions))");
+    }
+    if (!connection.tableExists("filterhits"))
+    {
+      connection.executeSimpleSQL("CREATE TABLE filterhits (filter TEXT, " +
+        "host TEXT, thirdParty INTEGER, hitCount INTEGER, lastHit INTEGER, " +
+        "subscriptions INTEGER NOT NULL, PRIMARY KEY(filter, host, " +
+        "thirdParty), FOREIGN KEY(subscriptions) REFERENCES "+
+        "filtersubscriptions(id))");
+      Prefs.sendstats_last_push = Date.now();
+    }
+    Object.defineProperty(this, "connection",
+    {
+      value: connection
+    });
+    return connection;
+  },
+
+  /**
+   * @return nsIFile SQLite database file with filter hits data
+   */
+  get getStorageFile()
+  {
+    let file = IO.resolveFilePath(Prefs.data_directory);
+    if (file)
+      file.append("adblockplus.sqlite");
+
+    Object.defineProperty(this, "getStorageFile", 
+    {
+      value: file
+    });
+    return file;
+  },
+
+  /**
+   * Called on module startup.
+   */
+  init: function()
+  {
+    Prefs.addListener(name =>
+    {
+      if (name == "sendstats")
+      {
+        if (this._prefToggleTimeout)
+          clearTimeout(this._prefToggleTimeout);
+        this._prefToggleTimeout = setTimeout(() =>
+        {
+          if (!Prefs.sendstats)
+            this.resetFilterHits();
+        }, 1000);
+      }
+    });
+
+    let downloader = new Downloader(this._getDownloadables.bind(this), 
+      INITIAL_DELAY, CHECK_INTERVAL, this.sendFilterHitsToServer);
+    downloader.onExpirationChange = this._onExpirationChange.bind(this);
+    onShutdown.add(() => downloader.cancel());
+  },
+
+  /**
+   * Yields a Downloadable instances for the notifications download.
+   */
+  _getDownloadables: function*()
+  {
+    let downloadable = new Downloadable();
+    if (typeof Prefs.sendstats_status.lastError === "number")
+      downloadable.lastError = Prefs.sendstats_status.lastError;
+    if (typeof Prefs.sendstats_status.lastCheck === "number")
+      downloadable.lastCheck = Prefs.sendstats_status.lastCheck;
+    if (typeof Prefs.sendstats_status.softExpiration === "number")
+      downloadable.softExpiration = Prefs.sendstats_status.softExpiration;
+    if (typeof Prefs.sendstats_status.hardExpiration === "number")
+      downloadable.hardExpiration = Prefs.sendstats_status.hardExpiration;
+    yield downloadable;
+  },
+
+  _onExpirationChange: function(downloadable)
+  {
+    Prefs.sendstats_status.lastCheck = downloadable.lastCheck;
+    Prefs.sendstats_status.softExpiration = downloadable.softExpiration;
+    Prefs.sendstats_status.hardExpiration = downloadable.hardExpiration;
+  },
+
+  /**
+   * Increases the filter hit count
+   * @param {Filter} filter
+   * @param {String} host 
+   */
+  increaseFilterHits: function(filter, host, thirdParty)
+  {
+    let subscriptions = filter.subscriptions;
+    let downloadableSubscriptions = [];
+    for (let i = 0; i < subscriptions.length; i++)
+    {
+      if (subscriptions[i] instanceof DownloadableSubscription)
+        downloadableSubscriptions.push(subscriptions[i].url);
+    }
+
+    if (downloadableSubscriptions.length == 0)
+      return;
+
+    if (!this.getStorageFile)
+      return;
+
+    let statementsArray = [];
+    let filtersubscriptions = JSON.stringify(downloadableSubscriptions);
+    let subscriptionStatement = this.connection.createStatement("INSERT OR " +
+      "IGNORE INTO filtersubscriptions (subscriptions) VALUES " +
+      "(:subscriptions)");
+    subscriptionStatement.params.subscriptions = filtersubscriptions;
+    statementsArray.push(subscriptionStatement);
+    let filterHitstatement = this.connection.createStatement("INSERT OR " +
+      "REPLACE INTO filterhits (filter, host, thirdParty, hitCount, lastHit, " +
+      "subscriptions) VALUES (:filter, :host, :thirdParty, coalesce ((SELECT " +
+      "hitCount  FROM filterhits WHERE filter=:filter AND host=:host AND " +
+      "thirdParty=:thirdParty), 0) + 1, :lastHit, (SELECT id " +
+      "FROM filtersubscriptions WHERE subscriptions=:subscriptions))");
+    filterHitstatement.params.filter = filter.text;
+    filterHitstatement.params.host = host;
+    filterHitstatement.params.lastHit = roundToHours(filter.lastHit);
+    filterHitstatement.params.thirdParty = thirdParty;
+    filterHitstatement.params.subscriptions = filtersubscriptions;
+    statementsArray.push(filterHitstatement);
+
+    this.connection.executeAsync(statementsArray, statementsArray.length,
+    {
+      handleError: function(error)
+      {
+        Cu.reportError("Error during writing of filter hits: " + reason);
+      },
+      handleCompletion: function(reason)
+      {
+        if (reason != REASON_FINISHED)
+          Cu.reportError("Writing of filter hits canceled or aborted: " + reason);
+      }
+    });
+  },
+
+  /**
+   * Remove all local collected filter hits data
+   */
+  resetFilterHits: function()
+  {
+    if (!this.getStorageFile)
+      return;
+
+    this.connection.executeSimpleSQL("DELETE FROM filterhits");
+    this.connection.executeSimpleSQL("DELETE FROM filtersubscriptions");
+  },
+
+  /**
+   * Send local stored filter hit data to the 
+   * service specified in Prefs.sendstats_url
+   */
+  sendFilterHitsToServer: function()
+  {
+    let now = Date.now();
+    if (Prefs.sendstats_last_push > now)
+      Prefs.sendstats_last_push = now;
+
+    if ((now - Prefs.sendstats_last_push < PUSH_INTERVAL) || !Prefs.sendstats 
+        || !this.getStorageFile)
+      return;
+
+    let statement = this.connection.createStatement("SELECT filterhits.*, " +
+      "filtersubscriptions.subscriptions FROM filterhits LEFT JOIN " +
+      "filtersubscriptions filtersubscriptions ON " +
+      "filterhits.subscriptions=filtersubscriptions.id");
+    statement.executeAsync(
+    {
+      handleResult: (result) =>
+      {
+        let filters = Object.create(null);
+        for (let row = result.getNextRow(); row;
+             row = result.getNextRow())
+        {
+          let filterText = row.getResultByName("filter");
+          let host = row.getResultByName("host");
+          let matchType = (row.getResultByName("thirdParty") ? 
+                            "thirdParty" : "firstParty");
+          let hitCount = row.getResultByName("hitCount");
+          let lastHit = row.getResultByName("lastHit");
+          let subscriptions = row.getResultByName("subscriptions");
+          
+          if (!(filterText in filters))
+           filters[filterText] = Object.create(null);
+          
+          let filter = filters[filterText];
+          filter.subscriptions = subscriptions;
+          filter[matchType] = Object.create(null);
+          filter[matchType][host] = {"hits": hitCount, latest: lastHit};
+        }
+
+        let request = new XMLHttpRequest();
+        request.open("POST", Prefs.sendstats_url);
+        request.setRequestHeader("Content-Type", "application/json");
+        request.addEventListener("load", () =>
+        {
+          this._sending = false;
+          if (request.status >= 200 && request.status < 300)
+          {
+            Prefs.sendstats_status.lastError = 0;
+            Prefs.sendstats_last_push = Date.now();
+            FilterHits.resetFilterHits();
+          }
+          else
+          {
+            Prefs.sendstats_status.lastError = Date.now();
+            this._sending = false;
+            Cu.reportError("Couldn't send filter hit statistics to server. Status code: " + request.status);
+          }
+        }, false);
+        request.addEventListener("error", () =>
+        {
+          this._sending = false;
+          Cu.reportError("Error occured while sending filter hit statistics to server.");
+        }, false);
+
+        let {addonName, addonVersion, application, 
+            applicationVersion, platform, platformVersion} = require("info");
+        let data = {
+          version: 1,
+          timeSincePush: roundToHours(Date.now() - Prefs.sendstats_last_push),
+          addonName: addonName,
+          addonVersion: addonVersion,
+          application: application,
+          applicationVersion: applicationVersion,
+          platform: platform,
+          platformVersion: platformVersion,
+          filters: filters
+        };
+        if (!this._sending)
+        {
+          this._sending = true;
+          request.send(JSON.stringify(data));
+        }
+      },
+      handleError: function(error)
+      {
+        Prefs.sendstats_status.lastError = Date.now();
+        Cu.reportError(error.message);
+      },
+      handleCompletion: function(reason)
+      {
+        if (reason != REASON_FINISHED)
+        {
+          Prefs.sendstats_status.lastError = Date.now();
+          Cu.reportError("Loading of filter hits canceled or aborted: " + reason);
+        }
+      }
+    });
+  },
+};
+
+function roundToHours(timestamp)
+{
+  return Math.round(timestamp / MILLIS_IN_HOUR) * 3600;
+}
+
+FilterHits.init();