Issue 395 - Filter hits statistics backend

sitescripts/filterhits/bin/process_logs.py

Index: sitescripts/filterhits/bin/process_logs.py
diff --git a/sitescripts/filterhits/bin/process_logs.py b/sitescripts/filterhits/bin/process_logs.py
new file mode 100755
index 0000000000000000000000000000000000000000..168033588d564ba37047f0aa84a7b22259952965
--- /dev/null
+++ b/sitescripts/filterhits/bin/process_logs.py
@@ -0,0 +1,87 @@
+# coding: utf-8
+
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2015 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import MySQLdb, itertools, json, os, sys
+from sitescripts.utils import get_config
+from sitescripts.filterhits import common, db, geometrical_mean
+
+last_log_file = None
+
+def log_files(dir):
+  """
+  Provides a generator of filter hits log files for the given directory.
+  Works recursively, relative path of log file is returned.
+  """
+  for root, subdirs, files in os.walk(dir):
+    for f in files:
+      if os.path.splitext(f)[1] == ".log" and f[0].isdigit():
+        yield os.path.join(root, f)
+
+def read_data(log_file):
+  """
+  Read, parse and return the JSON data for the given log file name.
+  (As a side effect sets the global last_log_file to the log file name.)
+  """
+  global last_log_file
+  try:
+    with open(log_file, "r") as f:
+      # Skip past the date and GET parameters
+      current = last = ""
+      while last + current != '" ':
+        last, current = current, f.read(1)
+        if not current:
+          sys.exit("Unexpected EOF in log file %s" % log_file)
+
+      # Read the JSON
+      data = json.load(f)
+      # Keep track of the current log file in global variable in case we need to
+      # identify it later if there's a problem. (This works because the files are
+      # processed lazily.)
+      last_log_file = log_file
+  except IOError:
+    sys.exit("Could not read log file %s" % log_file)
+  return data
+
+if __name__ == "__main__":
+  if not len(sys.argv) == 2:
+    print "Usage: python -m sitescripts.filterhits.bin.process_logs /path/to/logs"
+    sys.exit(1)
+
+  interval = get_config().get("filterhitstats", "interval")
+
+  def read_update(f):
+    return geometrical_mean.update(interval, read_data(f))
+
+  if sys.argv[1].endswith(".log"):
+    sql = read_update(sys.argv[1])
+  else:
+    sql = itertools.chain.from_iterable(itertools.imap(read_update,
+                                                       log_files(sys.argv[1])))
+
+  db_connection = db.connect()
+  try:
+    db.write(db_connection, sql)
+  except (KeyError, MySQLdb.Error), e:
+    if isinstance(e, KeyError):
+      message = "KeyError (%s)" % str(e.args[0])
+    elif isinstance(e, MySQLdb.Error):
+      message = "MySQL error (%d) \"%s\"" % (e.args[0], e.args[1])
+    sys.exit("Failed to process file %s, all changes rolled back. %s\n" % (
+      last_log_file, message
+    ))
+  finally:
+    db_connection.close()