Issue 394 - hit statistics tool data collection

lib/filterHits.js

+/*
+ * This file is part of Adblock Plus <https://adblockplus.org/>,
+ * Copyright (C) 2006-2015 Eyeo GmbH
+ *
+ * Adblock Plus is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * Adblock Plus is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+let {Services} = Cu.import("resource://gre/modules/Services.jsm", null);
+let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null);
+
+let {Prefs} = require("prefs");
+let {Utils} = require("utils");
+let {MILLIS_IN_DAY} = require("downloader");
+let {FilterNotifier} = require("filterNotifier");
+let {DownloadableSubscription} = require("subscriptionClasses");
+
+/**
+ * This class reads filter hits statistics from SQLite database,
+ * manages them in memory and writes them back.
+ * @class
+ */
+let FilterHits = exports.FilterHits =
+{
+  /**
+   * Data that shoud be sent to the server
+   * @type Object
+   */
+  filters: Object.create(null),
+
+  /**
+   * Time since last push
+   * @type Number
+   */
+  _lastPush: 0,
+
+  /**
+   * Indicates the timeframe between pushes
+   * @type Number
+   */
+  _pushInterval: MILLIS_IN_DAY * 7,
+
+  /**
+   * Indicates whether the data is being loaded from storage
+   * @type Boolean
+   */
+  _loading: false,
+
+  /**
+   * Indicates whether the data is being saved to storage
+   * @type Boolean
+   */
+  _saving: false,
+
+  /**
+   * Indicates whether the data is being sent to the server
+   * @type Boolean
+   */
+  _sending: false,
+
+  /**
+   * Increases the filter hit count
+   * @param {Filter} filter
+   * @param {String} host 
+   */
+  increaseFilterHits: function(filter, host)
+  {
+    let subscriptions = filter.subscriptions;
+    let inDownloadableSubscription = false;
+    for (let i = 0; i < subscriptions.length; i++)
+    {
+      if (subscriptions[i] instanceof DownloadableSubscription)
+      {
+        inDownloadableSubscription = true;
+        break;
+      }
+    }
+
+    if (!inDownloadableSubscription)
+      return;
+
+    if (!(filter.text in this.filters))
+      this.filters[filter.text] = Object.create(null);
+
+    let statFilter = this.filters[filter.text];
+    let filterType = filter.thirdParty ? "thirdParty" : "firstParty";
+
+    if (!(filterType in statFilter))
+      statFilter[filterType] = Object.create(null);
+
+    if (!("subscriptions" in statFilter))
+      statFilter.subscriptions = [];
+
+    for (let i = 0; i < subscriptions.length; i++)
+    {
+      if (subscriptions[i] instanceof DownloadableSubscription
+          && statFilter.subscriptions.indexOf(subscriptions[i].url) == -1)
+        statFilter.subscriptions.push(subscriptions[i].url);
+    }
+
+    if (!(host in statFilter[filterType]))
+    {
+      statFilter[filterType][host] = {hits: 1, latest: filter.lastHit};
+    }
+    else
+    {
+      statFilter[filterType][host].hits++;
+      statFilter[filterType][host].latest = filter.lastHit;
+    }
+  },
+
+  resetFilterHits: function()
+  {
+    this.filters = Object.create(null);
+    this.saveFilterHitsToDatabase();
+  },
+
+  sendFilterHitsToServer: function()
+  {
+    if (!Prefs.sendstats)
+      return;
+
+    let request = new XMLHttpRequest();
+    request.open("POST", Prefs.sendstats_url);
+    request.setRequestHeader("Content-Type", "application/json");
+    request.addEventListener("load", function(event)
+    {
+      FilterHits._sending = false;
+      if (request.status >= 200 && request.status < 300)
+      {
+        FilterHits._lastPush = new Date().getTime();
+        FilterHits.resetFilterHits();
+      }
+      else
+      {
+        Cu.reportError("Could not send filter hit statistics to Adblock Plus server.");
+      }
+    }, false);
+
+    let {addonName, addonVersion, application, 
+        applicationVersion, platform, platformVersion} = require("info");
+    let data = {
+      version: 1,
+      timeSincePush: this._lastPush,
+      addonName: addonName,
+      addonVersion: addonVersion,
+      application: application,
+      applicationVersion: applicationVersion,
+      platform: platform,
+      platformVersion: platformVersion,
+      filters: this.filters
+    };
+
+    this._sending = true;
+    request.send(JSON.stringify(data));
+  },
+
+  getStorageFile: function()
+  {
+    return FileUtils.getFile("ProfD", ["adblockplus.sqlite"]);
+  },
+
+  checkCreateTable: function(connection)
+  {
+    if (!connection.tableExists("filterhits"))
+      connection.executeSimpleSQL("CREATE TABLE filterhits (id INTEGER PRIMARY KEY, filters TEXT, date INTEGER)");
+  },
+
+  /**
+   * Load Filter hits from database
+   */
+  loadFilterHitsFromDatabase: function()
+  {
+    let storageFile = this.getStorageFile();
+    if (!storageFile)
+      return;
+
+    let connection = Services.storage.openDatabase(storageFile);
+    this.checkCreateTable(connection);
+
+    let statement = connection.createStatement("SELECT * FROM filterhits");

[Wladimir Palant, 2016/02/15 12:25:40]

That was not really the idea. We use the database to *avoid* loading all the
data into memory, because we expect lots of data here.

Of course, for that we would need to store the data in a more meaningful format
than a JSON blob. So our table should be something like:

  CREATE TABLE filterhits (filter TEXT, host TEXT, hitCount INTEGER, lastHit
INTEGER, PRIMARY KEY(filter, host))

Note: I'm not sure whether SQLite allows text fields to be keys, if not we might
also have to save the MD5 or SHA1 hash of the filter and host as well and use
those for the key.

And to increase a hit you would run an SQL statement like the following:

  INSERT OR REPLACE INTO filterhits (filter, host, hitCount, lastHit)
    VALUES (?, ?,
      (SELECT hitCount FROM filterhits WHERE filter = ? AND host = ?) + 1,
    ?)

There should be no explicit "save to disk," SQLite should rather take care of
that by itself.

[saroyanm, 2016/02/19 17:38:11]

Fare enough, makes more sense, not sure why initially was implemented like that.

Also It's possible to have TEXT as primary key:
http://www.sqlite.org/faq.html#q4 

+    if (!this._loading)
+    {
+      this._loading = true;
+      statement.executeAsync(
+      {
+        handleResult: function(results)
+        {
+          let row = results.getNextRow();
+          if (row)
+          {
+            let filters = row.getResultByName("filters");
+            let lastDate = row.getResultByName("date");
+            FilterHits.filters = JSON.parse(filters);
+            FilterHits._lastPush = lastDate;
+          }
+        },
+
+        handleError: function(error)
+        {
+          Cu.reportError(error.message);
+        },
+
+        handleCompletion: function(reason)
+        {
+          if (reason != Ci.mozIStorageStatementCallback.REASON_FINISHED)
+            Cu.reportError("Loading of filter hits canceled or aborted.");
+          FilterHits._loading = false;
+        }
+      });
+    }
+
+    connection.asyncClose();
+  },
+
+  /**
+   * Save Filter hits to database
+   */
+  saveFilterHitsToDatabase: function()
+  {
+    let now = new Date().getTime();
+    if (!this._lastPush)
+      this._lastPush = now;
+
+    if (!this._sending && now - this._lastPush > this._pushInterval)
+    {
+      this.sendFilterHitsToServer();
+      return;
+    }
+
+    let storageFile = this.getStorageFile();
+    if (!storageFile)
+      return;
+
+    let connection = Services.storage.openDatabase(storageFile);
+    this.checkCreateTable(connection);
+
+    let statement = connection.createStatement("INSERT OR REPLACE INTO filterhits (id, filters, date) VALUES(0, :filters, :date)");
+    statement.params.filters = JSON.stringify(this.filters);
+    statement.params.date = this._lastPush;
+    if (!this._saving)
+    {
+      this._saving = true;
+      statement.executeAsync(
+      {
+        handleError: function(aError)
+        {
+          Cu.reportError(aError.message);
+        },
+
+        handleCompletion: function(aReason)
+        {
+          if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED)
+            Cu.reportError("Writing of filter hits canceled or aborted.");
+          FilterHits._saving = false;
+        }
+      });
+    }
+
+    connection.asyncClose();
+  }
+};
+
+FilterNotifier.addListener(function(action)
+{
+  if (action == "load" && Prefs.sendstats)
+    FilterHits.loadFilterHitsFromDatabase();
+});