Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: sitescripts/filterhits/web/submit.py

Issue 4615801646612480: Issue 395 - Filter hits statistics backend (Closed)
Left Patch Set: Simplified geometrical_mean code and reduced filter inserts. Created March 16, 2015, 4:24 p.m.
Right Patch Set: Addressed further comments from Sebastian. Created April 2, 2015, 10:13 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « sitescripts/filterhits/web/query.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH 4 # Copyright (C) 2006-2015 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 import MySQLdb, json, os 18 import json
19 import os
20 import tempfile
21 import time
22 import traceback
19 from datetime import datetime 23 from datetime import datetime
24 from errno import EEXIST
25
26 import MySQLdb
27
20 from sitescripts.web import url_handler 28 from sitescripts.web import url_handler
21 from sitescripts.utils import get_config, setupStderr 29 from sitescripts.utils import get_config, setupStderr
22 from sitescripts.filterhits import common, db, geometrical_mean 30 from sitescripts.filterhits import db, geometrical_mean
31 from sitescripts.filterhits.web import common
32
33 def log_filterhits(data, basepath, query_string):
34 """
35 This logs the provided filterhits data as JSON to a file named after
36 the current timestamp in a directory named after the current date.
37 """
38 now = time.gmtime()
39
40 dir_name = time.strftime("%Y-%m-%d", now)
41 path = os.path.join(basepath, dir_name)
42 try:
43 os.makedirs(path)
44 except OSError as e:
45 if e.errno != EEXIST:
46 raise
47
48 with tempfile.NamedTemporaryFile(
49 prefix = str(int(time.mktime(now))) + "-",
50 suffix = ".log",
51 dir = path,
52 delete = False
53 ) as f:
54 print >> f, "[%s] %s" % (time.strftime("%d/%b/%Y:%H:%M:%S", now), query_stri ng)
55 json.dump(data, f)
56 return f.name
23 57
24 @url_handler("/submit") 58 @url_handler("/submit")
25 def submit(environ, start_response): 59 def submit(environ, start_response):
26 setupStderr(environ["wsgi.errors"]) 60 setupStderr(environ["wsgi.errors"])
27 config = get_config() 61 config = get_config()
28 62
29 # Check that this is a POST request 63 # Check that this is a POST request
30 if environ["REQUEST_METHOD"] != "POST": 64 if environ["REQUEST_METHOD"] != "POST":
31 return common.show_error("Unsupported request method", start_response) 65 return common.show_error("Unsupported request method", start_response)
32 66
33 # Parse the submitted JSON 67 # Parse the submitted JSON
34 try: 68 try:
35 data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])) ) 69 data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])) )
36 except (KeyError, IOError, ValueError): 70 except (KeyError, IOError, ValueError):
37 return common.show_error("Error while parsing JSON data.", start_response) 71 return common.show_error("Error while parsing JSON data.", start_response)
38 72
39 # Make sure the submitted data was contained within an object at least 73 # Make sure the submitted data was contained within an object at least
40 if not isinstance(data, dict): 74 if not isinstance(data, dict):
41 return common.show_error("Error, data must be contained within an object.", start_response) 75 return common.show_error("Error, data must be contained within an object.", start_response)
42 76
43 # Log the data to a file 77 # Log the data to a file
44 if not db.testing: 78 log_dir = config.get("filterhitstats", "log_dir")
45 log_dir = config.get("filterhitstats", "log_dir") 79 try:
46 try: 80 log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", ""))
47 log_file = common.log_filterhits(data, log_dir, 81 except (OSError, IOError):
48 environ.get("QUERY_STRING", "")) 82 traceback.print_exc()
49 except (OSError, IOError): 83 return common.show_error("Failed to write data to log file!", start_response ,
50 return common.show_error("Failed to write data to log file!", start_respon se, 84 "500 Logging error")
51 "500 Logging error")
52 85
53 # Update the geometrical_mean aggregations in the database 86 # Update the geometrical_mean aggregations in the database
54 interval = config.get("filterhitstats", "interval") 87 interval = config.get("filterhitstats", "interval")
55 try: 88 try:
56 db_connection = db.connect() 89 db_connection = db.connect()
57 try: 90 try:
58 db.write(db_connection, geometrical_mean.update(interval, data)) 91 db.write(db_connection, geometrical_mean.update(interval, data))
59 finally: 92 finally:
60 db_connection.close() 93 db_connection.close()
61 except (KeyError, MySQLdb.Error), e: 94 except:
62 # Updating the aggregations in the database failed for whatever reason, 95 # Updating the aggregations in the database failed for whatever reason,
63 # log the details but continue to return 200 OK to the client to avoid 96 # log the details but continue to return 204 to the client to avoid the
64 # re-transmission of the data. 97 # re-transmission of data.
65 if not db.testing: 98 processing_error_log = os.path.join(config.get("filterhitstats", "log_dir"),
66 processing_error_log = os.path.join(config.get("filterhitstats", "log_dir" ), 99 "processing-errors.log")
67 "processing-errors.log") 100 with open(processing_error_log, "a+") as f:
68 with open(processing_error_log, "a+") as f: 101 message = "Problem processing data file %s:\n%s" % (
69 if isinstance(e, KeyError): 102 log_file, traceback.format_exc()
70 message = "KeyError (%s) when processing data file %s\n" % (str(e.args [0]), log_file) 103 )
71 elif isinstance(e, MySQLdb.Error): 104 print >> f, "[%s] %s" % (datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z"), message)
72 message = "MySQL error (%d) when processing data file %s: \"%s\"\n" % (e.args[0], log_file, e.args[1])
73 f.write("[%s] %s" % (datetime.now().strftime('%d/%b/%Y:%H:%M:%S %z'), me ssage))
74 105
75 # Send back a 200 OK response 106 # Send back a 204 No Content
76 response_headers = [("Content-type", "text/plain")] 107 start_response("204 No Content", [])
77 start_response("200 OK", response_headers)
78 return [] 108 return []
LEFTRIGHT

Powered by Google App Engine
This is Rietveld