Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: sitescripts/filterhits/web/submit.py

Issue 4615801646612480: Issue 395 - Filter hits statistics backend (Closed)
Left Patch Set: Addressed Sebastian's and Wladimir's comments. Created March 27, 2015, 3:08 p.m.
Right Patch Set: Addressed further comments from Sebastian. Created April 2, 2015, 10:13 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « sitescripts/filterhits/web/query.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH 4 # Copyright (C) 2006-2015 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 import json 18 import json
19 import MySQLdb 19 import os
20 import tempfile 20 import tempfile
21 import time 21 import time
22 import os 22 import traceback
23 from datetime import datetime 23 from datetime import datetime
24 from errno import EEXIST 24 from errno import EEXIST
25 25
26 import MySQLdb
27
26 from sitescripts.web import url_handler 28 from sitescripts.web import url_handler
27 from sitescripts.utils import get_config, setupStderr 29 from sitescripts.utils import get_config, setupStderr
28 from sitescripts.filterhits import common, db, geometrical_mean 30 from sitescripts.filterhits import db, geometrical_mean
31 from sitescripts.filterhits.web import common
29 32
30 def log_filterhits(data, basepath, query_string): 33 def log_filterhits(data, basepath, query_string):
31 """ 34 """
32 This logs the provided filterhits data as JSON to a file named after 35 This logs the provided filterhits data as JSON to a file named after
33 the current timestamp in a directory named after the current date. 36 the current timestamp in a directory named after the current date.
34 """ 37 """
35 now = time.gmtime() 38 now = time.gmtime()
36 39
37 dir_name = time.strftime("%Y-%m-%d", now) 40 dir_name = time.strftime("%Y-%m-%d", now)
38 path = os.path.join(basepath, dir_name) 41 path = os.path.join(basepath, dir_name)
39 try: 42 try:
40 os.makedirs(path) 43 os.makedirs(path)
41 except OSError as e: 44 except OSError as e:
42 if e.errno != EEXIST: 45 if e.errno != EEXIST:
43 raise 46 raise
44 47
45 with tempfile.NamedTemporaryFile( 48 with tempfile.NamedTemporaryFile(
46 prefix = str(int(time.mktime(now))) + "-", 49 prefix = str(int(time.mktime(now))) + "-",
47 suffix = ".log", 50 suffix = ".log",
48 dir = path, 51 dir = path,
49 delete = False 52 delete = False
50 ) as f: 53 ) as f:
51 print >> f, "[%s] \"%s\"" % (time.strftime('%d/%b/%Y:%H:%M:%S', now), 54 print >> f, "[%s] %s" % (time.strftime("%d/%b/%Y:%H:%M:%S", now), query_stri ng)
52 query_string) 55 json.dump(data, f)
Wladimir Palant 2015/03/27 16:29:06 I wonder whether quotation marks in the query stri
kzar 2015/03/27 22:15:00 (Give me a little credit, I would not assume data
53 print >> f, json.dumps(data)
Sebastian Noack 2015/03/27 16:31:04 json.dump(data, f)
kzar 2015/03/27 22:15:00 Done.
54 return f.name 56 return f.name
55 57
56 @url_handler("/submit") 58 @url_handler("/submit")
57 def submit(environ, start_response): 59 def submit(environ, start_response):
58 setupStderr(environ["wsgi.errors"]) 60 setupStderr(environ["wsgi.errors"])
59 config = get_config() 61 config = get_config()
60 62
61 # Check that this is a POST request 63 # Check that this is a POST request
62 if environ["REQUEST_METHOD"] != "POST": 64 if environ["REQUEST_METHOD"] != "POST":
63 return common.show_error("Unsupported request method", start_response) 65 return common.show_error("Unsupported request method", start_response)
64 66
65 # Parse the submitted JSON 67 # Parse the submitted JSON
66 try: 68 try:
67 data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])) ) 69 data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])) )
68 except (KeyError, IOError, ValueError): 70 except (KeyError, IOError, ValueError):
69 return common.show_error("Error while parsing JSON data.", start_response) 71 return common.show_error("Error while parsing JSON data.", start_response)
70 72
71 # Make sure the submitted data was contained within an object at least 73 # Make sure the submitted data was contained within an object at least
72 if not isinstance(data, dict): 74 if not isinstance(data, dict):
73 return common.show_error("Error, data must be contained within an object.", start_response) 75 return common.show_error("Error, data must be contained within an object.", start_response)
74 76
75 # Log the data to a file 77 # Log the data to a file
76 if not db.testing: 78 log_dir = config.get("filterhitstats", "log_dir")
77 log_dir = config.get("filterhitstats", "log_dir") 79 try:
78 try: 80 log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", ""))
79 log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", "")) 81 except (OSError, IOError):
80 except (OSError, IOError): 82 traceback.print_exc()
81 return common.show_error("Failed to write data to log file!", start_respon se, 83 return common.show_error("Failed to write data to log file!", start_response ,
82 "500 Logging error") 84 "500 Logging error")
Wladimir Palant 2015/03/27 16:29:06 This seems to be an unexpected error, one that is
kzar 2015/03/27 22:15:00 Done.
83 85
84 # Update the geometrical_mean aggregations in the database 86 # Update the geometrical_mean aggregations in the database
85 interval = config.get("filterhitstats", "interval") 87 interval = config.get("filterhitstats", "interval")
86 try: 88 try:
87 db_connection = db.connect() 89 db_connection = db.connect()
88 try: 90 try:
89 db.write(db_connection, geometrical_mean.update(interval, data)) 91 db.write(db_connection, geometrical_mean.update(interval, data))
90 finally: 92 finally:
91 db_connection.close() 93 db_connection.close()
92 except (KeyError, MySQLdb.Error), e: 94 except:
93 # Updating the aggregations in the database failed for whatever reason, 95 # Updating the aggregations in the database failed for whatever reason,
94 # log the details but continue to return 200 OK to the client to avoid 96 # log the details but continue to return 204 to the client to avoid the
95 # re-transmission of the data. 97 # re-transmission of data.
96 if not db.testing: 98 processing_error_log = os.path.join(config.get("filterhitstats", "log_dir"),
97 processing_error_log = os.path.join(config.get("filterhitstats", "log_dir" ), 99 "processing-errors.log")
98 "processing-errors.log") 100 with open(processing_error_log, "a+") as f:
99 with open(processing_error_log, "a+") as f: 101 message = "Problem processing data file %s:\n%s" % (
100 if isinstance(e, KeyError): 102 log_file, traceback.format_exc()
101 message = "KeyError (%s) when processing data file %s" % (str(e.args[0 ]), log_file) 103 )
102 elif isinstance(e, MySQLdb.Error): 104 print >> f, "[%s] %s" % (datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z"), message)
103 message = "MySQL error (%d) when processing data file %s: \"%s\"" % (e .args[0], log_file, e.args[1])
104 print >> f, "[%s] %s" % (datetime.now().strftime('%d/%b/%Y:%H:%M:%S %z') , message)
Wladimir Palant 2015/03/27 16:29:06 I'd suggest using traceback module here instead of
kzar 2015/03/27 22:15:00 Done.
105 105
106 # Send back a 200 OK response 106 # Send back a 204 No Content
107 response_headers = [("Content-type", "text/plain")] 107 start_response("204 No Content", [])
108 start_response("200 OK", response_headers)
109 return [] 108 return []
LEFTRIGHT

Powered by Google App Engine
This is Rietveld