Issue 395 - Filter hits statistics backend

sitescripts/filterhits/geometrical_mean.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import itertools
-import sitescripts.filterhits.db as db
-from functools import partial
 
-def filter_hits(data):
+from sitescripts.filterhits import db
+
+def update(interval, data):
   """
-  Generator that provides all filter hits for the given data,
-  in tuples like (filter, domain, hits, latest).
+  Generator that provides all the SQL and parameters needed to update the
+  aggregations for the given data + interval in the database.
   """
-  for filter, filter_data in data['filters'].iteritems():
+  for filter, filter_data in data["filters"].iteritems():
+    yield ("""INSERT IGNORE INTO `filters`
+              (filter, sha1) VALUES (%s, UNHEX(SHA1(filter)))""", filter)
+
     domains = itertools.chain(filter_data.get("thirdParty", {}).iteritems(),
                               filter_data.get("firstParty", {}).iteritems())
     for domain, domain_data in domains:
-      yield (filter, domain, domain_data["hits"], domain_data["latest"] / 1000)
-
-def update_query(interval, filter, domain, hits, latest):
-  """
-  Function that takes the fields for a filter hit and returns them arranged
-  as the update SQL requires along with the SQL itself.
-  """
-  return (("""INSERT IGNORE INTO `filters`
-              (filter, sha1) VALUES (%s, UNHEX(SHA1(filter)))""",
-           filter),
-          ("""INSERT INTO `geometrical_mean`
-              (filter_sha1, domain, hits, timestamp)
-              VALUES (UNHEX(SHA1(%s)), %s, %s, FROM_UNIXTIME(%s))
-              ON DUPLICATE KEY UPDATE
-              hits = (
-                POW(hits, 1 - (UNIX_TIMESTAMP(VALUES(timestamp)) -
-                               UNIX_TIMESTAMP(timestamp)) / %s) *
-                POW(VALUES(hits), (UNIX_TIMESTAMP(VALUES(timestamp)) -
-                                   UNIX_TIMESTAMP(timestamp)) / %s)),
-              timestamp = VALUES(timestamp)""",
-           filter, domain, hits, int(latest), interval, interval))
-
-def update(interval, data):
-  """
-  Returns an iterator of all the SQL and parameters needed to
-  update the aggregations for the given data + interval in the database.
-  """
-  flatten = itertools.chain.from_iterable
-  return flatten(itertools.imap(lambda fields: update_query(interval, *fields),

[Sebastian Noack, 2015/02/17 14:59:17]

I feel that you are kinda overusing itertools. A generator seems to be more
appropriate and easier to read here: 

for fields in filter_hits(data):
  for query in update_query(interval, *fields):
    yield query

[kzar, 2015/02/24 18:05:11]

Done.

-                                filter_hits(data)))
+      yield ("""INSERT INTO `frequencies`
+                (filter_sha1, domain, frequency, timestamp)
+                VALUES (UNHEX(SHA1(%s)), %s, %s, FROM_UNIXTIME(%s))
+                ON DUPLICATE KEY UPDATE
+                frequency = (
+                  POW(frequency, 1 - (UNIX_TIMESTAMP(VALUES(timestamp)) -
+                                      UNIX_TIMESTAMP(timestamp)) / %s) *
+                  POW(VALUES(frequency), (UNIX_TIMESTAMP(VALUES(timestamp)) -
+                                          UNIX_TIMESTAMP(timestamp)) / %s)),
+                timestamp = VALUES(timestamp)""",
+             filter, domain, domain_data["hits"],
+             int(domain_data["latest"] / 1000), interval, interval)