Issue 395 - Filter hits statistics backend

sitescripts/filterhits/bin/process_logs.py

Index: sitescripts/filterhits/bin/process_logs.py
diff --git a/sitescripts/filterhits/bin/process_logs.py b/sitescripts/filterhits/bin/process_logs.py
new file mode 100755
index 0000000000000000000000000000000000000000..8f9cdcc31b129b09ed762bfde91c12770365080d
--- /dev/null
+++ b/sitescripts/filterhits/bin/process_logs.py
@@ -0,0 +1,89 @@
+# coding: utf-8
+
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2015 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import MySQLdb, itertools, json, os, sys
+from sitescripts.utils import get_config
+import sitescripts.filterhits.common as common
+import sitescripts.filterhits.db as db
+import sitescripts.filterhits.geometrical_mean as geometrical_mean
+
+last_log_file = None
+
+def log_files(dir):
+  """
+  Provides a generator of filter hits log files for the given directory.
+  Works recursively, relative path of log file is returned.
+  """
+  for root, subdirs, files in os.walk(dir):
+    for f in files:
+      if os.path.splitext(f)[1] == ".log" and f[0].isdigit():
+        yield os.path.join(root, f)
+
+def read_data(log_file):
+  """
+  Read, parse and return the JSON data for the given log file name.
+  (As a side effect sets the global last_log_file to the log file name.)
+  """
+  global last_log_file
+  try:
+    with open(log_file, "r") as f:
+      # Skip past the date and GET parameters
+      current = last = None
+      while not (last == '"' and current == " "):
+        last, current = current, f.read(1)
+        if not current:
+          sys.exit("Unexpected EOF in log file %s" % log_file)
+
+      # Read the JSON
+      data = json.load(f)
+      # Keep track of the current log file in global variable in case we need to
+      # identify it later if there's a problem. (This works because the files are
+      # processed lazily.)
+      last_log_file = log_file
+  except IOError:
+    sys.exit("Could not read log file %s" % log_file)
+  return data
+
+if __name__ == "__main__":
+  if not len(sys.argv) == 2:
+    print "Usage: python -m sitescripts.filterhits.bin.process_logs /path/to/logs"
+    sys.exit(1)
+
+  config = get_config()
+  interval = config.get("filterhitstats", "interval")
+
+  def read_update(f):
+    return geometrical_mean.update(interval, read_data(f))
+
+  if sys.argv[1].endswith(".log"):
+    sql = read_update(sys.argv[1])
+  else:
+    sql = itertools.chain.from_iterable(itertools.imap(read_update,
+                                                       log_files(sys.argv[1])))
+
+  try:
+    db_connection = db.connect(config.get("filterhitstats", "dbuser"),
+                               config.get("filterhitstats", "dbpassword"),
+                               config.get("filterhitstats", "database"))
+    db.write(db_connection, sql)
+  except MySQLdb.Error, e:
+    sys.exit("Failed to process file %s, all changes rolled back. MySQl error (%d): \"%s\"\n" % (
+      last_log_file, e.args[0], e.args[1]
+    ))
+  finally:
+    if db_connection:
+      db_connection.close()