Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: sitescripts/stats/bin/pagegenerator.py

Issue 5182947690807296: Centralize stats processing, have the stats server pull in logs (Closed)
Left Patch Set: Created Dec. 20, 2013, 1:07 p.m.
Right Patch Set: Fixed comment and processing of non-Unicode keys Created Dec. 26, 2013, 2:09 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « sitescripts/stats/bin/logprocessor.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2013 Eyeo GmbH 4 # Copyright (C) 2006-2013 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 import os, re, codecs, simplejson, time, itertools 18 import os, re, codecs, json, time, itertools
19 from datetime import date 19 from datetime import date
20 from sitescripts.utils import get_config, setupStderr, get_custom_template_envir onment, cached 20 from sitescripts.utils import get_config, setupStderr, get_custom_template_envir onment, cached
21 import sitescripts.stats.common as common 21 import sitescripts.stats.common as common
22 from sitescripts.stats.countrycodes import countrycodes 22 from sitescripts.stats.countrycodes import countrycodes
23 23
24 @cached(float("inf")) 24 @cached(float("inf"))
25 def get_template_environment(): 25 def get_template_environment():
26 return get_custom_template_environment({ 26 return get_custom_template_environment({
27 "monthname": lambda value: date(int(value[0:4]), int(value[4:]), 1).strftime ("%b %Y"), 27 "monthname": lambda value: date(int(value[0:4]), int(value[4:]), 1).strftime ("%b %Y"),
28 "weekday": lambda value: ["Monday", "Tuesday", "Wednesday", "Thursday", "Fri day", "Saturday", "Sunday"][int(value)], 28 "weekday": lambda value: ["Monday", "Tuesday", "Wednesday", "Thursday", "Fri day", "Saturday", "Sunday"][int(value)],
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
98 baseURL = get_config().get("stats", "baseURL_" + server_type) 98 baseURL = get_config().get("stats", "baseURL_" + server_type)
99 filedata = {} 99 filedata = {}
100 current_month = None 100 current_month = None
101 for month, month_dir in get_names(server_type_dir, True): 101 for month, month_dir in get_names(server_type_dir, True):
102 if current_month == None or month > current_month: 102 if current_month == None or month > current_month:
103 current_month = month 103 current_month = month
104 104
105 for filename, path in get_names(month_dir, False): 105 for filename, path in get_names(month_dir, False):
106 filename = re.sub(r"\.json$", "", filename) 106 filename = re.sub(r"\.json$", "", filename)
107 with codecs.open(path, "rb", encoding="utf-8") as file: 107 with codecs.open(path, "rb", encoding="utf-8") as file:
108 data = simplejson.load(file) 108 data = json.load(file)
109 109
110 overview_url = "../../overview-" + common.filename_encode(filename + ".h tml") 110 overview_url = "../../overview-" + common.filename_encode(filename + ".h tml")
111 filtered_urls = {} 111 filtered_urls = {}
112 for field in common.fields: 112 for field in common.fields:
113 if field["name"] not in data: 113 if field["name"] not in data:
114 continue 114 continue
115 # Create filtered views for the first thirty values of a field if they 115 # Create filtered views for the first thirty values of a field if they
116 # have filtered data. 116 # have filtered data.
117 for name, value in get_template_environment().filters["sortfield"](dat a[field["name"]], field)[0:30]: 117 for name, value in get_template_environment().filters["sortfield"](dat a[field["name"]], field)[0:30]:
118 if filter(lambda k: k not in ("hits", "bandwidth"), value.iterkeys() ): 118 if filter(lambda k: k not in ("hits", "bandwidth"), value.iterkeys() ):
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after
158 158
159 outputfile = os.path.join(outputdir, common.filename_encode(server_type), "i ndex.html") 159 outputfile = os.path.join(outputdir, common.filename_encode(server_type), "i ndex.html")
160 generate_main_page(outputfile, current_month, baseURL, monthdata) 160 generate_main_page(outputfile, current_month, baseURL, monthdata)
161 161
162 if __name__ == '__main__': 162 if __name__ == '__main__':
163 setupStderr() 163 setupStderr()
164 164
165 datadir = get_config().get("stats", "dataDirectory") 165 datadir = get_config().get("stats", "dataDirectory")
166 outputdir = get_config().get("stats", "outputDirectory") 166 outputdir = get_config().get("stats", "outputDirectory")
167 generate_pages(datadir, outputdir) 167 generate_pages(datadir, outputdir)
LEFTRIGHT

Powered by Google App Engine
This is Rietveld