Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: sitescripts/filterhits/bin/reprocess_logs.py

Issue 4615801646612480: Issue 395 - Filter hits statistics backend (Closed)
Left Patch Set: Addressed Sebastian's and Wladimir's comments. Created March 27, 2015, 3:08 p.m.
Right Patch Set: Addressed further comments from Sebastian. Created April 2, 2015, 10:13 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « sitescripts/filterhits/bin/__init__.py ('k') | sitescripts/filterhits/db.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 # coding: utf-8
2
3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH
5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation.
9 #
10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17
18 import itertools
19 import json
20 import logging
21 import os
22 import sys
23
24 import MySQLdb
25
26 from sitescripts.utils import get_config
27 from sitescripts.filterhits import db, geometrical_mean
28
29 _last_log_file = None
30
31 def log_files(dir):
32 """
33 Provides a generator of filter hits log files for the given directory.
34 Works recursively, relative path of log file is returned.
35 """
36 for root, subdirs, files in os.walk(dir):
37 for f in files:
38 if os.path.splitext(f)[1] == ".log" and f[0].isdigit():
39 yield os.path.join(root, f)
40
41 def read_data(log_file):
42 """
43 Read, parse and return the JSON data for the given log file name.
44 (As a side effect sets the global _last_log_file to the log file name.)
45 """
46 global _last_log_file
47 try:
48 with open(log_file, "r") as f:
49 f.readline()
50 data = json.load(f)
51 # Keep track of the current log file in global variable in case we need to
52 # identify it later if there's a problem. (This works because the files ar e
53 # processed lazily.)
54 _last_log_file = log_file
55 except IOError:
56 sys.exit("Could not read log file %s" % log_file)
57 return data
58
59 if __name__ == "__main__":
60 if not len(sys.argv) == 2:
61 print "Usage: python -m sitescripts.filterhits.bin.reprocess_logs /path/to/l ogs"
62 sys.exit(1)
63
64 interval = get_config().get("filterhitstats", "interval")
65
66 def read_update(f):
67 return geometrical_mean.update(interval, read_data(f))
68
69 if sys.argv[1].endswith(".log"):
70 sql = read_update(sys.argv[1])
71 else:
72 sql = itertools.chain.from_iterable(itertools.imap(read_update,
73 log_files(sys.argv[1])))
74
75 db_connection = db.connect()
76
77 try:
78 db.write(db_connection, sql)
79 except:
80 logging.error("Failed to process file %s, all changes rolled back." % _last_ log_file)
81 raise
82 finally:
83 db_connection.close()
LEFTRIGHT

Powered by Google App Engine
This is Rietveld