Issue 395 - Filter hits statistics backend

sitescripts/filterhits/web/submit.py

Index: sitescripts/filterhits/web/submit.py
diff --git a/sitescripts/filterhits/web/submit.py b/sitescripts/filterhits/web/submit.py
index 1354966acc6880edb706ebae85c3282efa14bbd5..673170f8f3d75972a5325041ecf773ed2651650c 100644
--- a/sitescripts/filterhits/web/submit.py
+++ b/sitescripts/filterhits/web/submit.py
@@ -1,7 +1,7 @@
 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2014 Eyeo GmbH
+# Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -15,64 +15,67 @@
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
-import MySQLdb, json
-from sitescripts.web import url_handler, showError
+import MySQLdb, json, os
+from datetime import datetime
+from sitescripts.web import url_handler
 from sitescripts.utils import get_config, setupStderr
-import sitescripts.filterhits.common as common
 
-config = get_config()
+import sitescripts.filterhits.common as common

[Sebastian Noack, 2015/03/02 10:04:01]

Nit: from ... import ...

[kzar, 2015/03/02 10:39:03]

Done.

+import sitescripts.filterhits.db as db
+import sitescripts.filterhits.geometrical_mean as geometrical_mean
 
 @url_handler("/submit")
 def submit(environ, start_response):
-  global config
   setupStderr(environ["wsgi.errors"])
+  config = get_config()
 
   # Check that this is a POST request
-  if environ["REQUEST_METHOD"].upper() != "POST":
-    return showError("Unsupported request method", start_response)
+  if environ["REQUEST_METHOD"] != "POST":
+    return common.showError("Unsupported request method", start_response)
 
   # Parse the submitted JSON
-  data = "{}"
-  try:
-    data_length = int(environ.get("CONTENT_LENGTH", "0"))
-  except ValueError:
-    data_length = 0
-  if data_length != 0:
-    data = environ["wsgi.input"].read(data_length)
   try:
-    data = json.loads(data)
-  except json.decoder.JSONDecodeError:
-    return showError("Error while parsing JSON data.", start_response)
+    data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])))
+  except (KeyError, IOError, ValueError):
+    return common.showError("Error while parsing JSON data.", start_response)
 
-  # Make sure it looks roughly valid
-  if not common.valid_log_data(data):
-    return showError("Data looks invalid.", start_response)
+  # Make sure the submitted data was contained within an object at least
+  if not isinstance(data, dict):
+    return common.showError("Error, data must be contained within an object.", start_response)
 
   # Log the data to a file
-  log_dir = config.get("filterhitstats", "log_dir")
+  if not db.testing:
+    log_dir = config.get("filterhitstats", "log_dir")
+    try:
+      log_file = common.log_filterhits(data, log_dir,
+                                       environ.get("QUERY_STRING", ""))
+    except (OSError, IOError):
+      return common.showError("Failed to write data to log file!", start_response,
+                              "500 Logging error")
 
+  # Update the geometrical_mean aggregations in the database
+  interval = config.get("filterhitstats", "interval")
   try:
-    common.log_filterhits(data, log_dir)
-  except OSError, IOError:
-    return showError("Failed to save data!", start_response)
-
-  # # TODO calculate the geometrical mean and update the database
-  # # Open DB connection set up cursor
-  # db = _get_db(config)
-  # cursor = db.cursor()
-  # for filter, filter_data in data['filters'].iteritems():
-  #   for domain, domain_hits in itertools.chain(filter_data['thirdParty'].iteritems(),
-  #                                              filter_data['firstParty'].iteritems()):
-  #     1 # UPDATE geometrical_mean SET hits=(??) timestamp=new_timestamp WHERE filter=filter AND domain=domain;
-  #       #                                   ^ SELECT current hits + timestamp + perform geometrical mean on combo
-  # # Commit / rollback and close
-  # # db.commit()
-  # db.close()
+    db_connection = db.connect()
+    try:
+      db.write(db_connection, geometrical_mean.update(interval, data))
+    finally:
+      db_connection.close()
+  except (KeyError, MySQLdb.Error), e:
+    # Updating the aggregations in the database failed for whatever reason,
+    # log the details but continue to return 200 OK to the client to avoid
+    # re-transmission of the data.
+    if not db.testing:
+      processing_error_log = os.path.join(config.get("filterhitstats", "log_dir"),
+                                     "processing-errors.log")
+      with open(processing_error_log, "a+") as f:
+        if isinstance(e, KeyError):
+          message = "KeyError (%s) when processing data file %s\n" % (str(e.args[0]), log_file)
+        elif isinstance(e, MySQLdb.Error):
+          message = "MySQL error (%d) when processing data file %s: \"%s\"\n" % (e.args[0], log_file, e.args[1])
+        f.write("[%s] %s" % (datetime.now().strftime('%d/%b/%Y:%H:%M:%S %z'), message))
 
+  # Send back a 200 OK response
   response_headers = [("Content-type", "text/plain")]
   start_response("200 OK", response_headers)
   return []
-
-# def _increment_entry(db, domain_id, status):
-#   cursor = db.cursor(MySQLdb.cursors.DictCursor)
-#   cursor.execute("INSERT INTO corrections(domain, status, curr_month, prev_month, curr_year, prev_year) VALUES (%s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE curr_month = curr_month + 1, curr_year = curr_year + 1", (domain_id, status, 1, 0, 1, 0))