Issue 395 - Filter hits statistics backend

sitescripts/filterhits/geometrical_mean.py

+# coding: utf-8
+
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2015 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import itertools
+from sitescripts.filterhits import db
+
+def update(interval, data):
+  """
+  Generator that provides all the SQL and parameters needed to update the
+  aggregations for the given data + interval in the database.
+  """
+  for filter, filter_data in data['filters'].iteritems():
+    yield ("""INSERT IGNORE INTO `filters`
+              (filter, sha1) VALUES (%s, UNHEX(SHA1(filter)))""", filter)
+
+    domains = itertools.chain(filter_data.get("thirdParty", {}).iteritems(),
+                              filter_data.get("firstParty", {}).iteritems())
+    for domain, domain_data in domains:
+      yield ("""INSERT INTO `geometrical_mean`

[Wladimir Palant, 2015/03/26 22:56:50]

geometrical_mean is merely the aggregation approach we are using right now. A
table name should describe the meaning of its contents, not the implementation.
Call that table "frequency"?

[kzar, 2015/03/27 11:59:57]

(I chose the plural to keep consistent with the other table name "filters".)
Done.

+                (filter_sha1, domain, hits, timestamp)
+                VALUES (UNHEX(SHA1(%s)), %s, %s, FROM_UNIXTIME(%s))
+                ON DUPLICATE KEY UPDATE
+                hits = (
+                  POW(hits, 1 - (UNIX_TIMESTAMP(VALUES(timestamp)) -
+                                 UNIX_TIMESTAMP(timestamp)) / %s) *
+                  POW(VALUES(hits), (UNIX_TIMESTAMP(VALUES(timestamp)) -
+                                     UNIX_TIMESTAMP(timestamp)) / %s)),
+                timestamp = VALUES(timestamp)""",
+             filter, domain, domain_data["hits"],
+             int(domain_data["latest"] / 1000), interval, interval)