Update stats processing

sitescripts/stats/bin/logprocessor.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2013 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import os, sys, codecs, re, math, GeoIP, urllib, urlparse, socket, simplejson
+from collections import OrderedDict
 import sitescripts.stats.common as common
 from sitescripts.utils import get_config, setupStderr
 from datetime import datetime, timedelta
 
 log_regexp = None
 mirror_name = None
 gecko_apps = None
 
+def cache_lru(func):
+  """
+    Decorator that memoizes the return values of a single-parameter function in
+    case it is called again with the same parameter. The 1024 most recent
+    results are saved.
+  """
+
+  results = OrderedDict()
+  results.entries_left = 1024
+
+  def wrapped(arg):
+    if arg in results:
+      result = results[arg]
+      del results[arg]
+    else:
+      if results.entries_left > 0:
+        results.entries_left -= 1
+      else:
+        results.popitem(last=False)
+      result = func(arg)
+    results[arg] = result
+    return result
+  return wrapped
+
+
+def cache_last(func):
+  """
+    Decorator that memoizes the last return value of a function in case it is
+    called again with the same parameters.
+  """
+  result = {"args": None, "result": None}
+
+  def wrapped(*args):
+    if args != result["args"]:
+      result["result"] = func(*args)
+      result["args"] = args
+    return result["result"]
+  return wrapped
+
+
+@cache_lru
 def parse_ua(ua):
   # Opera might disguise itself as other browser so it needs to go first
   match = re.search(r"\bOpera/([\d\.]+)", ua)

[Felix Dahlke, 2013/08/28 17:25:32]

Memoization helped save around 5% here. In an access log with 10k lines, I had
only 435 unique user agents.

Here's the approach I used:

ua_mem_table = {}

def parse_ua_mem(ua):
  hash_code = hash(ua)
  if not hash_code in ua_mem_table:
    ua_mem_table[hash_code] = {}
  if ua in ua_mem_table[hash_code]:
    return ua_mem_table[hash_code][ua]
  browser = parse_ua(ua)
  ua_mem_table[hash_code][ua] = browser
  return browser

Then I called parse_ua_mem instead of parse_ua below.

This is relatively easy on the memory and avoids collisions. Would be a tad bit
faster to just use a single flat hash table with the ua as the key, but that'd
also blow up memory usage.

[Sebastian Noack, 2013/08/29 10:54:30]

Keys in python dicts are already unique. So when you add multiple items with the
same key (key1 == key2 == True), into a dict, only the most recent added
key-value-pair is kept. Just like with your approach. Or what do you mean?

If you don't want to grow the cache infinitely, you might want to use an LRU
cache.

   if match:
     # Opera 10+ declares itself as Opera 9.80 but adds Version/1x.x to the UA
     match2 = re.search(r"\bVersion/([\d\.]+)", ua)
     if match2:
       return "Opera", match2.group(1)
     else:
       return "Opera", match.group(1)
 
   # Opera 15+ has the same UA as Chrome but adds OPR/1x.x to it
   match = re.search(r"\bOPR/(\d+\.\d+)", ua)
@@ skipping 70 matching lines @@
   if match:
     ip = match.group(1)
 
   country = geo.country_code_by_addr(ip)
   if country in (None, "", "--"):
     country = "unknown"
   country = country.lower()
 
   return ip, country
 
+@cache_last
 def parse_time(timestr, tz_hours, tz_minutes):
   result = datetime.strptime(timestr, "%d/%b/%Y:%H:%M:%S")
   result -= timedelta(hours = tz_hours, minutes = math.copysign(tz_minutes, tz_hours))
   return result, result.strftime("%Y%m"), result.day, result.weekday(), result.hour
 
+@cache_lru
 def parse_path(path):
   urlparts = urlparse.urlparse(path)
   try:
     path = urllib.unquote(urlparts.path).decode("utf-8")
   except:
     path = urlparts.path
   return path[1:], urlparts.query
 
+@cache_lru
+def parse_query(query):
+  return urlparse.parse_qs(query)
+
+@cache_lru
+def parse_lastversion(last_version):
+  return datetime.strptime(last_version, "%Y%m%d%H%M")
+
+@cache_lru
+def get_week(date):
+  return date.isocalendar()[0:2]
+
 def parse_downloader_query(info):
-  params = urlparse.parse_qs(info["query"])
+  params = parse_query(info["query"])
   for param in ("addonName", "addonVersion", "application", "applicationVersion", "platform", "platformVersion"):
     info[param] = params.get(param, ["unknown"])[0]
 
   # Only leave the major and minor release number for application and platform
   info["applicationVersion"] = re.sub(r"^(\d+\.\d+).*", r"\1", info["applicationVersion"])
   info["platformVersion"] = re.sub(r"^(\d+\.\d+).*", r"\1", info["platformVersion"])
 
   # Chrome Adblock sends an X-Client-ID header insteads of URL parameters
   match = re.match(r"^adblock/([\d\.]+)$", info["clientid"], re.I) if info["clientid"] else None
   if match:
     info["addonName"] = "chromeadblock"
     info["addonVersion"] = match.group(1)
 
   last_version = params.get("lastVersion", ["unknown"])[0]
   if info["file"] == "notification.json" and last_version == "0" and (
       (info["addonName"] == "adblockplus" and info["addonVersion"] == "2.3.1") or
       (info["addonName"] in ("adblockpluschrome", "adblockplusopera") and info["addonVersion"] == "1.5.2")
     ):
     # Broken notification version number in these releases, treat like unknown
     last_version = "unknown"
 
   if last_version == "unknown":
     info["downloadInterval"] = "unknown"
   elif last_version == "0":
     info["downloadInterval"] = "unknown"
     info["firstDownload"] = info["firstInMonth"] = info["firstInWeek"] = info["firstInDay"] = True
   else:
     try:
-      last_update = datetime.strptime(last_version, "%Y%m%d%H%M")
+      last_update = parse_lastversion(last_version)
       diff = info["time"] - last_update
       if diff.days >= 365:
         info["downloadInterval"] = "%i year(s)" % (diff.days / 365)
       elif diff.days >= 30:
         info["downloadInterval"] = "%i month(s)" % (diff.days / 30)
       elif diff.days >= 1:
         info["downloadInterval"] = "%i day(s)" % diff.days
       else:
         info["downloadInterval"] = "%i hour(s)" % (diff.seconds / 3600)
 
       if last_update.year != info["time"].year or last_update.month != info["time"].month:
         info["firstInMonth"] = info["firstInDay"] = True
       elif last_update.day != info["time"].day:
         info["firstInDay"] = True
 
-      if last_update.isocalendar()[0:2] != info["time"].isocalendar()[0:2]:
+      if get_week(last_update) != get_week(info["time"]):
         info["firstInWeek"] = True
     except ValueError:
       info["downloadInterval"] = "unknown"
       pass
 
 def parse_addon_name(file):
   if "/" in file:
     return file.split("/")[-2]
   else:
     return None
@@ skipping 141 matching lines @@
 
 if __name__ == "__main__":
   setupStderr()
 
   verbose = (len(sys.argv) >= 2 and sys.argv[1] == "verbose")
   geo = GeoIP.open(get_config().get("stats", "geoip_db"), GeoIP.GEOIP_MEMORY_CACHE)
   result = parse_stdin(geo, verbose)
 
   with codecs.open(get_config().get("stats", "tempFile"), "wb", encoding="utf-8") as file:
     simplejson.dump(result, file, indent=2, sort_keys=True)