Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: sitescripts/filterhits/web/submit.py

Issue 4615801646612480: Issue 395 - Filter hits statistics backend (Closed)
Left Patch Set: Rebased. Created March 28, 2015, 2:10 p.m.
Right Patch Set: Addressed further comments from Sebastian. Created April 2, 2015, 10:13 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « sitescripts/filterhits/web/query.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH 4 # Copyright (C) 2006-2015 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 import json 18 import json
19 import MySQLdb 19 import os
20 import tempfile 20 import tempfile
21 import time 21 import time
22 import traceback 22 import traceback
23 import os
24 from datetime import datetime 23 from datetime import datetime
25 from errno import EEXIST 24 from errno import EEXIST
25
26 import MySQLdb
26 27
27 from sitescripts.web import url_handler 28 from sitescripts.web import url_handler
28 from sitescripts.utils import get_config, setupStderr 29 from sitescripts.utils import get_config, setupStderr
29 from sitescripts.filterhits import db, geometrical_mean 30 from sitescripts.filterhits import db, geometrical_mean
30 from sitescripts.filterhits.web import common 31 from sitescripts.filterhits.web import common
31 32
32 def log_filterhits(data, basepath, query_string): 33 def log_filterhits(data, basepath, query_string):
33 """ 34 """
34 This logs the provided filterhits data as JSON to a file named after 35 This logs the provided filterhits data as JSON to a file named after
35 the current timestamp in a directory named after the current date. 36 the current timestamp in a directory named after the current date.
36 """ 37 """
37 now = time.gmtime() 38 now = time.gmtime()
38 39
39 dir_name = time.strftime("%Y-%m-%d", now) 40 dir_name = time.strftime("%Y-%m-%d", now)
40 path = os.path.join(basepath, dir_name) 41 path = os.path.join(basepath, dir_name)
41 try: 42 try:
42 os.makedirs(path) 43 os.makedirs(path)
43 except OSError as e: 44 except OSError as e:
44 if e.errno != EEXIST: 45 if e.errno != EEXIST:
45 raise 46 raise
46 47
47 with tempfile.NamedTemporaryFile( 48 with tempfile.NamedTemporaryFile(
48 prefix = str(int(time.mktime(now))) + "-", 49 prefix = str(int(time.mktime(now))) + "-",
49 suffix = ".log", 50 suffix = ".log",
50 dir = path, 51 dir = path,
51 delete = False 52 delete = False
52 ) as f: 53 ) as f:
53 print >> f, "[%s] %s" % (time.strftime('%d/%b/%Y:%H:%M:%S', now), query_stri ng) 54 print >> f, "[%s] %s" % (time.strftime("%d/%b/%Y:%H:%M:%S", now), query_stri ng)
54 json.dump(data, f) 55 json.dump(data, f)
55 return f.name 56 return f.name
56 57
57 @url_handler("/submit") 58 @url_handler("/submit")
58 def submit(environ, start_response): 59 def submit(environ, start_response):
59 setupStderr(environ["wsgi.errors"]) 60 setupStderr(environ["wsgi.errors"])
60 config = get_config() 61 config = get_config()
61 62
62 # Check that this is a POST request 63 # Check that this is a POST request
63 if environ["REQUEST_METHOD"] != "POST": 64 if environ["REQUEST_METHOD"] != "POST":
64 return common.show_error("Unsupported request method", start_response) 65 return common.show_error("Unsupported request method", start_response)
65 66
66 # Parse the submitted JSON 67 # Parse the submitted JSON
67 try: 68 try:
68 data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])) ) 69 data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])) )
69 except (KeyError, IOError, ValueError): 70 except (KeyError, IOError, ValueError):
70 return common.show_error("Error while parsing JSON data.", start_response) 71 return common.show_error("Error while parsing JSON data.", start_response)
71 72
72 # Make sure the submitted data was contained within an object at least 73 # Make sure the submitted data was contained within an object at least
73 if not isinstance(data, dict): 74 if not isinstance(data, dict):
74 return common.show_error("Error, data must be contained within an object.", start_response) 75 return common.show_error("Error, data must be contained within an object.", start_response)
75 76
76 # Log the data to a file 77 # Log the data to a file
77 if not db.testing: 78 log_dir = config.get("filterhitstats", "log_dir")
78 log_dir = config.get("filterhitstats", "log_dir") 79 try:
79 try: 80 log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", ""))
80 log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", "")) 81 except (OSError, IOError):
81 except (OSError, IOError): 82 traceback.print_exc()
82 traceback.print_exc() 83 return common.show_error("Failed to write data to log file!", start_response ,
83 return common.show_error("Failed to write data to log file!", start_respon se, 84 "500 Logging error")
84 "500 Logging error")
85 85
86 # Update the geometrical_mean aggregations in the database 86 # Update the geometrical_mean aggregations in the database
87 interval = config.get("filterhitstats", "interval") 87 interval = config.get("filterhitstats", "interval")
88 try: 88 try:
89 db_connection = db.connect() 89 db_connection = db.connect()
90 try: 90 try:
91 db.write(db_connection, geometrical_mean.update(interval, data)) 91 db.write(db_connection, geometrical_mean.update(interval, data))
92 finally: 92 finally:
93 db_connection.close() 93 db_connection.close()
94 except: 94 except:
95 # Updating the aggregations in the database failed for whatever reason, 95 # Updating the aggregations in the database failed for whatever reason,
96 # log the details but continue to return 200 OK to the client to avoid 96 # log the details but continue to return 204 to the client to avoid the
97 # re-transmission of the data. 97 # re-transmission of data.
98 if not db.testing: 98 processing_error_log = os.path.join(config.get("filterhitstats", "log_dir"),
99 processing_error_log = os.path.join(config.get("filterhitstats", "log_dir" ), 99 "processing-errors.log")
100 "processing-errors.log") 100 with open(processing_error_log, "a+") as f:
101 with open(processing_error_log, "a+") as f: 101 message = "Problem processing data file %s:\n%s" % (
102 message = "Problem processing data file %s:\n%s" % ( 102 log_file, traceback.format_exc()
103 log_file, traceback.format_exc() 103 )
104 ) 104 print >> f, "[%s] %s" % (datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z"), message)
105 print >> f, "[%s] %s" % (datetime.now().strftime('%d/%b/%Y:%H:%M:%S %z') , message)
106 105
107 # Send back a 200 OK response 106 # Send back a 204 No Content
108 response_headers = [("Content-type", "text/plain")] 107 start_response("204 No Content", [])
109 start_response("200 OK", response_headers)
110 return [] 108 return []
LEFTRIGHT

Powered by Google App Engine
This is Rietveld