Issue 395 - Filter hits statistics backend

sitescripts/filterhits/bin/process_logs.py

Index: sitescripts/filterhits/bin/process_logs.py
diff --git a/sitescripts/filterhits/bin/process_logs.py b/sitescripts/filterhits/bin/process_logs.py
new file mode 100755
index 0000000000000000000000000000000000000000..745524e0f76521770e54188edbbbc7d491ee3fd8
--- /dev/null
+++ b/sitescripts/filterhits/bin/process_logs.py
@@ -0,0 +1,87 @@
+# coding: utf-8
+
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2014 Eyeo GmbH

[Sebastian Noack, 2015/02/11 16:00:12]

Nit: This line needs to be updated now. In other files as well.

[kzar, 2015/02/17 10:52:24]

Done.

+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import MySQLdb, itertools, json, os, sys
+from sitescripts.utils import get_config
+import sitescripts.filterhits.common as common
+import sitescripts.filterhits.db as db
+import sitescripts.filterhits.geometrical_mean as geometrical_mean
+
+last_log_file = None
+
+def log_files(dir):
+  """
+   Provides a generator of filter hits log files for the given directory.

[Sebastian Noack, 2015/02/11 16:00:12]

Nit: Why do you indent the docstring one more space than the surrounding quotes?

[kzar, 2015/02/17 10:52:24]

Done.

+   Works recursively, relative path of log file is returned.
+  """
+  for root, subdirs, files in os.walk(dir):
+    for f in files:
+      if f.endswith(".log") and f[0].isdigit():

[Sebastian Noack, 2015/02/11 16:29:32]

Nit: Please use os.path.splitext()

[kzar, 2015/02/17 10:52:24]

Done.

+        yield os.path.join(root, f)
+
+def read_data(log_file):
+  """
+   Read, parse and return the JSON data for the given log file name.
+   (As a side effect sets the global last_log_file to the log file name.)
+  """
+  global last_log_file
+  try:
+    with open(log_file, "r") as f:
+      # Skip past the date and GET parameters
+      s = ""
+      while s != "\" ":

[Sebastian Noack, 2015/02/11 16:29:32]

It took me a while to figure this code out. Maybe a proper state machine would
be more readable:

last = None
while True:
  current = f.read(1)
  if not current:
    sys.exit("Unexpected EOF")
  if last == '\' and current == " ":
    break
  last = current

Also note that EOF is currently ignored by your implementation resulting in an
infinite loop.

[kzar, 2015/02/17 10:52:24]

Done.

+        s = s[-1:] + f.read(1)
+      # Read the JSON
+      data = json.load(f)
+      # Keep track of the current log file in global variable in case we need to
+      # identify it later if there's a problem. (This works because the files are
+      # processed lazily.)
+      last_log_file = log_file
+  except IOError:
+    sys.exit("Could not read log file %s" % log_file)
+  if not common.valid_log_data(data):
+    sys.exit("Invalid data in log file %s." % log_file)
+  return data
+
+if __name__ == "__main__":
+  if not len(sys.argv) == 2:
+    print "Usage: python -m sitescripts.filterhits.bin.process_logs /path/to/logs"
+    sys.exit(1)
+
+  config = get_config()
+  interval = config.get("filterhitstats", "interval")
+
+  def read_update(f):
+    return geometrical_mean.update(interval, read_data(f))
+
+  if sys.argv[1].endswith(".log"):
+    sql = read_update(sys.argv[1])
+  else:
+    sql = itertools.chain.from_iterable(itertools.imap(read_update,
+                                                       log_files(sys.argv[1])))
+
+  try:
+    db.connect(config.get("filterhitstats", "dbuser"),
+               config.get("filterhitstats", "dbpassword"),
+               config.get("filterhitstats", "database"))
+    db.write(sql)
+  except MySQLdb.Error, e:
+    sys.exit("Failed to process file %s, all changes rolled back. MySQl error (%d): \"%s\"\n" % (
+      last_log_file, e.args[0], e.args[1]
+    ))
+  finally:
+    db.disconnect()