Centralize stats processing, have the stats server pull in logs

sitescripts/stats/bin/logprocessor.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2013 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 441 matching lines @@
 
     if info["file"] not in section:
       section[info["file"]] = {}
     section = section[info["file"]]
 
     add_record(info, section)
   return data
 
 def merge_objects(object1, object2, factor=1):
   for key, value in object2.iteritems():
-    key = unicode(key)
+    try:
+      key = unicode(key)
+    except UnicodeDecodeError:
+      key = unicode(key, encoding="latin-1")
     if isinstance(value, numbers.Number):
       object1[key] = object1.get(key, 0) + factor * value
     else:
       merge_objects(object1.setdefault(key, {}), value, factor)
 
 def save_stats(server_type, data, factor=1):
   base_dir = os.path.join(get_config().get("stats", "dataDirectory"), common.filename_encode(server_type))
   for month, month_data in data.iteritems():
     for name, file_data in month_data.iteritems():
       path = os.path.join(base_dir, common.filename_encode(month), common.filename_encode(name + ".json"))
@@ skipping 27 matching lines @@
     finally:
       fileobj.close()
     return server_type, log_file, data, ignored
   except:
     print >>sys.stderr, "Unable to process log file '%s'" % log_file
     traceback.print_exc()
     return None, None, None, None
 
 def parse_sources(sources, factor=1, verbose=False):
   pool = multiprocessing.Pool()
-  for server_type, log_file, data, ignored in pool.imap(parse_source, sources, chunksize=1):
-    if server_type == None:
-      continue
-
-    save_stats(server_type, data, factor)
-    if verbose:
-      print "Ignored files for %s" % log_file
-      print "============================================================"
-      print "\n".join(sorted(ignored))
-  pool.close()

[Sebastian Noack, 2013/12/24 10:08:42]

You should put the code above into a try-finally block.

+  try:
+    for server_type, log_file, data, ignored in pool.imap(parse_source, sources, chunksize=1):
+      if server_type == None:
+        continue
+
+      save_stats(server_type, data, factor)
+      if verbose:
+        print "Ignored files for %s" % log_file
+        print "============================================================"
+        print "\n".join(sorted(ignored))
+  finally:
+    pool.close()
 
 if __name__ == "__main__":
   setupStderr()
 
   parser = argparse.ArgumentParser(description="Processes log files and merges them into the stats database")
   parser.add_argument("--verbose", dest="verbose", action="store_const", const=True, default=False, help="Verbose mode, ignored requests will be listed")
   parser.add_argument("--revert", dest="factor", action="store_const", const=-1, default=1, help="Remove log data from the database")
   parser.add_argument("mirror_name", nargs="?", help="Name of the mirror server that the file belongs to")
   parser.add_argument("server_type", nargs="?", help="Server type like download, update or subscription")
   parser.add_argument("log_file", nargs="?", help="Log file path, can be a local file path, http:// or ssh:// URL")
   args = parser.parse_args()
 
   if args.mirror_name and args.server_type and args.log_file:
     sources = [(args.mirror_name, args.server_type, args.log_file)]
   else:
     sources = get_stats_files()
   parse_sources(sources, args.factor, args.verbose)