Centralize stats processing, have the stats server pull in logs

sitescripts/stats/bin/pagegenerator.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2013 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
-import os, re, codecs, simplejson, time, itertools
+import os, re, codecs, json, time, itertools
 from datetime import date
 from sitescripts.utils import get_config, setupStderr, get_custom_template_environment, cached
 import sitescripts.stats.common as common
 from sitescripts.stats.countrycodes import countrycodes
 
 @cached(float("inf"))
 def get_template_environment():
   return get_custom_template_environment({
     "monthname": lambda value: date(int(value[0:4]), int(value[4:]), 1).strftime("%b %Y"),
     "weekday": lambda value: ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"][int(value)],
@@ skipping 69 matching lines @@
     baseURL = get_config().get("stats", "baseURL_" + server_type)
     filedata = {}
     current_month = None
     for month, month_dir in get_names(server_type_dir, True):
       if current_month == None or month > current_month:
         current_month = month
 
       for filename, path in get_names(month_dir, False):
         filename = re.sub(r"\.json$", "", filename)
         with codecs.open(path, "rb", encoding="utf-8") as file:
-          data = simplejson.load(file)
+          data = json.load(file)
 
         overview_url = "../../overview-" + common.filename_encode(filename + ".html")
         filtered_urls = {}
         for field in common.fields:
           if field["name"] not in data:
             continue
           # Create filtered views for the first thirty values of a field if they
           # have filtered data.
           for name, value in get_template_environment().filters["sortfield"](data[field["name"]], field)[0:30]:
             if filter(lambda k: k not in ("hits", "bandwidth"), value.iterkeys()):
@@ skipping 39 matching lines @@
 
     outputfile = os.path.join(outputdir, common.filename_encode(server_type), "index.html")
     generate_main_page(outputfile, current_month, baseURL, monthdata)
 
 if __name__ == '__main__':
   setupStderr()
 
   datadir = get_config().get("stats", "dataDirectory")
   outputdir = get_config().get("stats", "outputDirectory")
   generate_pages(datadir, outputdir)