Issue 394 - hit statistics tool data collection

lib/filterHits.js

+/*
+ * This file is part of Adblock Plus <https://adblockplus.org/>,
+ * Copyright (C) 2006-2015 Eyeo GmbH
+ *
+ * Adblock Plus is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * Adblock Plus is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+let {Services} = Cu.import("resource://gre/modules/Services.jsm", null);
+let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null);
+
+let {Prefs} = require("prefs");
+let {Utils} = require("utils");
+let {MILLIS_IN_DAY} = require("downloader");
+let {FilterNotifier} = require("filterNotifier");
+let {DownloadableSubscription} = require("subscriptionClasses");
+
+/**
+ * This class reads filter hits statistics from SQLite database,
+ * manages them in memory and writes them back.
+ * @class
+ */
+let FilterHits = exports.FilterHits =
+{
+  /**
+   * Data that shoud be sent to the server
+   * @type Object
+   */
+  filters: Object.create(null),
+  
+  /**
+   * The Service URL that should handle the data
+   * @type String
+   */
+  _serviceURL: "",
+
+  /**
+   * Time since last push
+   * @type Number
+   */
+  _lastPush: 0,
+
+  /**
+   * Indicates the timeframe between pushes
+   * @type Number
+   */
+  _pushInterval: MILLIS_IN_DAY * 7,
+
+  /**
+   * Indicates whether the data are being loaded from storage
+   * @type Boolean
+   */
+  _loading: false,
+
+  /**
+   * Indicates whether the data are being saved to storage
+   * @type Boolean
+   */
+  _saving: false,
+
+  /**
+   * Indicates whether the data are being sent to the server
+   * @type Boolean
+   */
+  _sending: false,
+
+  /**
+   * Increases the filter hit count
+   * @param {Filter} filter
+   * @param {Window} window  Window that the match originated in (required to get host)
+   */
+  increaseFilterHits: function(filter, wnd)
+  {
+    let subscriptionTitles = [];
+    for (let i = 0; i < filter.subscriptions.length; i++)
+    {
+      if (filter.subscriptions[i] instanceof DownloadableSubscription)
+        subscriptionTitles.push(filter.subscriptions[i]._title)

[Thomas Greiner, 2015/03/06 11:29:19]

See https://issues.adblockplus.org/ticket/394#comment:40

[saroyanm, 2015/03/06 16:54:04]

Ohh, so stupid from my side :/
Done.

+    }
+
+    if (subscriptionTitles.length == 0)
+      return;

[Thomas Greiner, 2015/03/06 11:29:19]

Rather than constructing an array on each function call I'd suggest having a
simpler loop which would also allow you to break early in some cases.

let subscriptions = filter.subscriptions;
let inDownloadableSubscription = false;
for (let i = 0; i < subscriptions.length; i++)
{
  if (subscriptions[i] instanceof DownloadableSubscription)
  {
    inDownloadableSubscription = true;
    break;
  }
}

if (!inDownloadableSubscription)
  return;

The theory behind it is that array modifications are more expensive than looping
through an array and that it would therefore turn out to be more performant. I
was able to confirm that by measuring the performance of both methods within my
local Adblock Plus instance.

[saroyanm, 2015/03/06 16:54:04]

Done.

+    
+    if (!(filter.text in this.filters))
+      this.filters[filter.text] = Object.create(null);
+
+    let statFilter = this.filters[filter.text];
+    let filterType = filter.thirdParty ? "thirdParty" : "firstParty";
+
+    if (!(filterType in statFilter))
+      statFilter[filterType] = Object.create(null);
+
+    if (!("subscriptions" in statFilter))
+      statFilter.subscriptions = [];
+      
+    for (let i = 0; i < subscriptionTitles.length; i++)
+    {
+      if (statFilter.subscriptions.indexOf(subscriptionTitles[i]) == -1)
+        statFilter.subscriptions.push(subscriptionTitles[i]);
+    }
+
+    let wndLocation = Utils.getOriginWindow(wnd).location.href;
+    let host = Utils.unwrapURL(wndLocation).host;
+    
+    if (!(host in statFilter[filterType]))
+      statFilter[filterType][host] = {hits: 1, latest: filter.lastHit};
+    else
+    {
+      statFilter[filterType][host].hits++;
+      statFilter[filterType][host].latest = filter.lastHit;
+    }
+  },
+  
+  resetFilterHits: function()
+  {
+    this.filters = Object.create(null);
+    this.saveFilterHitsToDatabase();
+  },
+  
+  sendFilterHitsToServer: function()
+  {
+    let request = new XMLHttpRequest();
+    request.open("POST", this._serviceURL);
+    request.setRequestHeader("Content-Type", "application/json");
+    request.addEventListener("load", function(event)
+    {
+      FilterHits._sending = false;
+      if (request.status == 200)
+      {
+        FilterHits._lastPush = new Date().getTime();
+        FilterHits.resetFilterHits();
+      }
+      else
+        Cu.reportError("Could not send filter hit statistics to Adblock Plus server.");
+    }, false);
+
+    let {addonName, addonVersion, application, applicationVersion, platform, platformVersion} = require("info");
+    let data = {
+      version: 1,
+      timeSincePush: this._lastPush,
+      addonName: addonName,
+      addonVersion: addonVersion,
+      application: application,
+      applicationVersion: applicationVersion,
+      platform: platform,
+      platformVersion: platformVersion,
+      filters: this.filters
+    };
+
+    this._sending = true;
+    request.send(JSON.stringify(data));
+  },
+  
+  getStorageFile: function()
+  {
+    return FileUtils.getFile("ProfD", ["adblockplus.sqlite"]);
+  },
+  
+  checkCreateTable: function(connection)
+  {
+    if (!connection.tableExists("filterhits"))
+      connection.executeSimpleSQL("CREATE TABLE filterhits (id INTEGER PRIMARY KEY, filters TEXT, date INTEGER)");
+  },
+  
+  /**
+   * Load Filter hits from database
+   */
+  loadFilterHitsFromDatabase: function()
+  {
+    let storageFile = this.getStorageFile();
+    if (!storageFile)
+      return;
+
+    let connection = Services.storage.openDatabase(storageFile);
+    this.checkCreateTable(connection);
+    
+    let statement = connection.createStatement("SELECT * FROM filterhits");
+    if (!this._loading)
+    {
+      this._loading = true;
+      statement.executeAsync(
+      {
+        handleResult: function(results)
+        {
+          let row = results.getNextRow();
+          if (row)
+          {
+            let filters = row.getResultByName("filters");
+            let lastDate = row.getResultByName("date");
+            FilterHits.filters = JSON.parse(filters);
+            FilterHits._lastPush = lastDate;
+          }
+        },
+      
+        handleError: function(error)
+        {
+          Cu.reportError(error.message);
+        },
+      
+        handleCompletion: function(reason)
+        {
+          if (reason != Ci.mozIStorageStatementCallback.REASON_FINISHED)
+            Cu.reportError("Loading of filter hits canceled or aborted.");
+          FilterHits._loading = false;
+        }
+      });
+    }
+    
+    connection.asyncClose();
+  },
+  
+  /**
+   * Save Filter hits to database
+   */
+  saveFilterHitsToDatabase: function()
+  {
+    let now = new Date().getTime();
+    if (!this._lastPush)
+      this._lastPush = now;
+    
+    if (!this._sending && now - this._lastPush > this._pushInterval)
+    {
+      this.sendFilterHitsToServer();
+      return;
+    }
+
+    let storageFile = this.getStorageFile();
+    if (!storageFile)
+      return;
+
+    let connection = Services.storage.openDatabase(storageFile);
+    this.checkCreateTable(connection);
+    
+    let statement = connection.createStatement("INSERT OR REPLACE INTO filterhits (id, filters, date) VALUES(0, :filters, :date)");
+    statement.params.filters = JSON.stringify(this.filters);
+    statement.params.date = this._lastPush;
+    if (!this._saving)
+    {
+      this._saving = true;
+      statement.executeAsync(
+      {
+        handleError: function(aError)
+        {
+          Cu.reportError(aError.message);
+        },
+      
+        handleCompletion: function(aReason) 
+        {
+          if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED)
+            Cu.reportError("Writing of filter hits canceled or aborted.");
+          FilterHits._saving = false;
+        }
+      });
+    }
+    
+    connection.asyncClose();
+  }
+};
+
+FilterNotifier.addListener(function(action)
+{
+  if (action == "load" && Prefs.sendstats)
+    FilterHits.loadFilterHitsFromDatabase();
+});