Left: | ||
Right: |
LEFT | RIGHT |
---|---|
1 # coding: utf-8 | 1 # coding: utf-8 |
2 | 2 |
3 # This file is part of the Adblock Plus web scripts, | 3 # This file is part of the Adblock Plus web scripts, |
4 # Copyright (C) 2006-2015 Eyeo GmbH | 4 # Copyright (C) 2006-2015 Eyeo GmbH |
5 # | 5 # |
6 # Adblock Plus is free software: you can redistribute it and/or modify | 6 # Adblock Plus is free software: you can redistribute it and/or modify |
7 # it under the terms of the GNU General Public License version 3 as | 7 # it under the terms of the GNU General Public License version 3 as |
8 # published by the Free Software Foundation. | 8 # published by the Free Software Foundation. |
9 # | 9 # |
10 # Adblock Plus is distributed in the hope that it will be useful, | 10 # Adblock Plus is distributed in the hope that it will be useful, |
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 # GNU General Public License for more details. | 13 # GNU General Public License for more details. |
14 # | 14 # |
15 # You should have received a copy of the GNU General Public License | 15 # You should have received a copy of the GNU General Public License |
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. | 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. |
17 | 17 |
18 import json | 18 import json |
19 import MySQLdb | 19 import os |
20 import tempfile | 20 import tempfile |
21 import time | 21 import time |
22 import os | 22 import traceback |
23 from datetime import datetime | 23 from datetime import datetime |
24 from errno import EEXIST | 24 from errno import EEXIST |
25 | 25 |
26 import MySQLdb | |
27 | |
26 from sitescripts.web import url_handler | 28 from sitescripts.web import url_handler |
27 from sitescripts.utils import get_config, setupStderr | 29 from sitescripts.utils import get_config, setupStderr |
28 from sitescripts.filterhits import common, db, geometrical_mean | 30 from sitescripts.filterhits import db, geometrical_mean |
31 from sitescripts.filterhits.web import common | |
29 | 32 |
30 def log_filterhits(data, basepath, query_string): | 33 def log_filterhits(data, basepath, query_string): |
31 """ | 34 """ |
32 This logs the provided filterhits data as JSON to a file named after | 35 This logs the provided filterhits data as JSON to a file named after |
33 the current timestamp in a directory named after the current date. | 36 the current timestamp in a directory named after the current date. |
34 """ | 37 """ |
35 now = time.gmtime() | 38 now = time.gmtime() |
36 | 39 |
37 dir_name = time.strftime("%Y-%m-%d", now) | 40 dir_name = time.strftime("%Y-%m-%d", now) |
38 path = os.path.join(basepath, dir_name) | 41 path = os.path.join(basepath, dir_name) |
39 try: | 42 try: |
40 os.makedirs(path) | 43 os.makedirs(path) |
41 except OSError as e: | 44 except OSError as e: |
42 if e.errno != EEXIST: | 45 if e.errno != EEXIST: |
43 raise | 46 raise |
44 | 47 |
45 with tempfile.NamedTemporaryFile( | 48 with tempfile.NamedTemporaryFile( |
46 prefix = str(int(time.mktime(now))) + "-", | 49 prefix = str(int(time.mktime(now))) + "-", |
47 suffix = ".log", | 50 suffix = ".log", |
48 dir = path, | 51 dir = path, |
49 delete = False | 52 delete = False |
50 ) as f: | 53 ) as f: |
51 f.write("[%s] \"%s\" %s\n" % (time.strftime('%d/%b/%Y:%H:%M:%S', now), | 54 print >> f, "[%s] %s" % (time.strftime("%d/%b/%Y:%H:%M:%S", now), query_stri ng) |
Sebastian Noack
2015/03/27 13:12:19
Same here, I'd rather go with the print statement.
kzar
2015/03/27 15:10:51
Done.
| |
52 query_string, json.dumps(data))) | 55 json.dump(data, f) |
53 return f.name | 56 return f.name |
54 | 57 |
55 @url_handler("/submit") | 58 @url_handler("/submit") |
56 def submit(environ, start_response): | 59 def submit(environ, start_response): |
57 setupStderr(environ["wsgi.errors"]) | 60 setupStderr(environ["wsgi.errors"]) |
58 config = get_config() | 61 config = get_config() |
59 | 62 |
60 # Check that this is a POST request | 63 # Check that this is a POST request |
61 if environ["REQUEST_METHOD"] != "POST": | 64 if environ["REQUEST_METHOD"] != "POST": |
62 return common.show_error("Unsupported request method", start_response) | 65 return common.show_error("Unsupported request method", start_response) |
63 | 66 |
64 # Parse the submitted JSON | 67 # Parse the submitted JSON |
65 try: | 68 try: |
66 data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])) ) | 69 data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])) ) |
67 except (KeyError, IOError, ValueError): | 70 except (KeyError, IOError, ValueError): |
68 return common.show_error("Error while parsing JSON data.", start_response) | 71 return common.show_error("Error while parsing JSON data.", start_response) |
69 | 72 |
70 # Make sure the submitted data was contained within an object at least | 73 # Make sure the submitted data was contained within an object at least |
71 if not isinstance(data, dict): | 74 if not isinstance(data, dict): |
72 return common.show_error("Error, data must be contained within an object.", start_response) | 75 return common.show_error("Error, data must be contained within an object.", start_response) |
73 | 76 |
74 # Log the data to a file | 77 # Log the data to a file |
75 if not db.testing: | 78 log_dir = config.get("filterhitstats", "log_dir") |
76 log_dir = config.get("filterhitstats", "log_dir") | 79 try: |
77 try: | 80 log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", "")) |
78 log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", "")) | 81 except (OSError, IOError): |
79 except (OSError, IOError): | 82 traceback.print_exc() |
80 return common.show_error("Failed to write data to log file!", start_respon se, | 83 return common.show_error("Failed to write data to log file!", start_response , |
81 "500 Logging error") | 84 "500 Logging error") |
82 | 85 |
83 # Update the geometrical_mean aggregations in the database | 86 # Update the geometrical_mean aggregations in the database |
84 interval = config.get("filterhitstats", "interval") | 87 interval = config.get("filterhitstats", "interval") |
85 try: | 88 try: |
86 db_connection = db.connect() | 89 db_connection = db.connect() |
87 try: | 90 try: |
88 db.write(db_connection, geometrical_mean.update(interval, data)) | 91 db.write(db_connection, geometrical_mean.update(interval, data)) |
89 finally: | 92 finally: |
90 db_connection.close() | 93 db_connection.close() |
91 except (KeyError, MySQLdb.Error), e: | 94 except: |
92 # Updating the aggregations in the database failed for whatever reason, | 95 # Updating the aggregations in the database failed for whatever reason, |
93 # log the details but continue to return 200 OK to the client to avoid | 96 # log the details but continue to return 204 to the client to avoid the |
94 # re-transmission of the data. | 97 # re-transmission of data. |
95 if not db.testing: | 98 processing_error_log = os.path.join(config.get("filterhitstats", "log_dir"), |
96 processing_error_log = os.path.join(config.get("filterhitstats", "log_dir" ), | 99 "processing-errors.log") |
97 "processing-errors.log") | 100 with open(processing_error_log, "a+") as f: |
98 with open(processing_error_log, "a+") as f: | 101 message = "Problem processing data file %s:\n%s" % ( |
99 if isinstance(e, KeyError): | 102 log_file, traceback.format_exc() |
100 message = "KeyError (%s) when processing data file %s\n" % (str(e.args [0]), log_file) | 103 ) |
101 elif isinstance(e, MySQLdb.Error): | 104 print >> f, "[%s] %s" % (datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z"), message) |
102 message = "MySQL error (%d) when processing data file %s: \"%s\"\n" % (e.args[0], log_file, e.args[1]) | |
103 f.write("[%s] %s" % (datetime.now().strftime('%d/%b/%Y:%H:%M:%S %z'), me ssage)) | |
104 | 105 |
105 # Send back a 200 OK response | 106 # Send back a 204 No Content |
106 response_headers = [("Content-type", "text/plain")] | 107 start_response("204 No Content", []) |
107 start_response("200 OK", response_headers) | |
108 return [] | 108 return [] |
LEFT | RIGHT |