Update stats processing

sitescripts/stats/bin/logprocessor.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2013 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import os, sys, codecs, re, math, GeoIP, urllib, urlparse, socket, simplejson
+from collections import OrderedDict
 import sitescripts.stats.common as common
 from sitescripts.utils import get_config, setupStderr
 from datetime import datetime, timedelta
 
 log_regexp = None
 mirror_name = None
 gecko_apps = None
 
+def cache_lru(func):
+  """
+    Decorator that memoizes the return values of a single-parameter function in
+    case it is called again with the same parameter. The 1024 most recent
+    results are saved.
+  """
+
+  results = OrderedDict()
+  results.entries_left = 1024
+
+  def wrapped(arg):
+    if arg in results:
+      result = results[arg]
+      del results[arg]
+    else:
+      if results.entries_left > 0:
+        results.entries_left -= 1
+      else:
+        results.popitem(last=False)
+      result = func(arg)
+    results[arg] = result
+    return result
+  return wrapped
+
+
+def cache_last(func):
+  """
+    Decorator that memoizes the last return value of a function in case it is
+    called again with the same parameters.
+  """
+  result = {"args": None, "result": None}
+
+  def wrapped(*args):
+    if args != result["args"]:
+      result["result"] = func(*args)
+      result["args"] = args
+    return result["result"]
+  return wrapped
+
+
+@cache_lru
 def parse_ua(ua):
   # Opera might disguise itself as other browser so it needs to go first
   match = re.search(r"\bOpera/([\d\.]+)", ua)
   if match:
     # Opera 10+ declares itself as Opera 9.80 but adds Version/1x.x to the UA
     match2 = re.search(r"\bVersion/([\d\.]+)", ua)
     if match2:
       return "Opera", match2.group(1)
     else:
       return "Opera", match.group(1)
 
   # Opera 15+ has the same UA as Chrome but adds OPR/1x.x to it
   match = re.search(r"\bOPR/(\d+\.\d+)", ua)
   if match:
     return "Opera", match.group(1)
 
-  for appName in ("Fennec", "Thunderbird", "SeaMonkey", "Songbird", "K-Meleon", "Prism", "Firefox"):
-    match = re.search(r"\b%s/(\d+\.\d+)" % appName, ua)

[Sebastian Noack, 2013/08/26 16:05:22]

Instead of iterating over the list of browsers and generate a regex for each, I
would suggest to use a single regex that matches all:

\b(Fennec|Thunderbird|SeaMonkey|Songbird|K-Meleon|Prism|Firefox)/(\d+\.\d+)

[Wladimir Palant, 2013/08/27 07:34:28]

This was done like that intentionally - quite a few browsers will claim being
Firefox. However, we can indeed handle all but Firefox in one go, done that now.

-    if match:
-      if appName == "Fennec" or (appName == "Firefox" and re.search(r"\bMobile;", ua)):
-        return "Firefox Mobile", match.group(1)
-      else:
-        return appName, match.group(1)
+  # Have to check for these before Firefox, they will usually have a Firefox identifier as well
+  match = re.search(r"\b(Fennec|Thunderbird|SeaMonkey|Songbird|K-Meleon|Prism)/(\d+\.\d+)", ua)
+  if match:
+    if match.group(1) == "Fennec":
+      return "Firefox Mobile", match.group(2)
+    else:
+      return match.group(1), match.group(2)
+
+  match = re.search(r"\bFirefox/(\d+\.\d+)", ua)
+  if match:
+    if re.search(r"\bMobile;", ua):
+      return "Firefox Mobile", match.group(1)
+    else:
+      return "Firefox", match.group(1)
 
   match = re.search(r"\brv:(\d+)\.(\d+)(?:\.(\d+))?", ua)
   if match and re.search(r"\bGecko/", ua):
     if match.group(3) and int(match.group(1)) < 2:
       return "Gecko", "%s.%s.%s" % (match.group(1), match.group(2), match.group(3))

[Sebastian Noack, 2013/08/26 16:05:22]

You could just just call match.groups(), which already returns a tuple with all
3 groups in it.

[Wladimir Palant, 2013/08/27 07:34:28]

I think I rather keep this as is for consistency with the variant two lines
below. Besides, I prefer listing the groups explicitly - regexps commonly have
groups that aren't meant to be used (I avoid this here by using (?:...) but
still).

     else:
       return "Gecko", "%s.%s" % (match.group(1), match.group(2))
 
   match = re.search(r"\bCoolNovo/(\d+\.\d+\.\d+)", ua)
   if match:
     return "CoolNovo", match.group(1)
 
   match = re.search(r"\bChrome/(\d+\.\d+)", ua)
   if match:
     return "Chrome", match.group(1)
@@ skipping 38 matching lines @@
   if match:
     ip = match.group(1)
 
   country = geo.country_code_by_addr(ip)
   if country in (None, "", "--"):
     country = "unknown"
   country = country.lower()
 
   return ip, country
 
+@cache_last
 def parse_time(timestr, tz_hours, tz_minutes):
   result = datetime.strptime(timestr, "%d/%b/%Y:%H:%M:%S")
   result -= timedelta(hours = tz_hours, minutes = math.copysign(tz_minutes, tz_hours))
   return result, result.strftime("%Y%m"), result.day, result.weekday(), result.hour
 
+@cache_lru
 def parse_path(path):
   urlparts = urlparse.urlparse(path)
   try:
     path = urllib.unquote(urlparts.path).decode("utf-8")
   except:
     path = urlparts.path
   return path[1:], urlparts.query
 
+@cache_lru
+def parse_query(query):
+  return urlparse.parse_qs(query)
+
+@cache_lru
+def parse_lastversion(last_version):
+  return datetime.strptime(last_version, "%Y%m%d%H%M")
+
+@cache_lru
+def get_week(date):
+  return date.isocalendar()[0:2]
+
 def parse_downloader_query(info):
-  params = urlparse.parse_qs(info["query"])
+  params = parse_query(info["query"])
   for param in ("addonName", "addonVersion", "application", "applicationVersion", "platform", "platformVersion"):
     info[param] = params.get(param, ["unknown"])[0]
 
   # Only leave the major and minor release number for application and platform
   info["applicationVersion"] = re.sub(r"^(\d+\.\d+).*", r"\1", info["applicationVersion"])
   info["platformVersion"] = re.sub(r"^(\d+\.\d+).*", r"\1", info["platformVersion"])
 
   # Chrome Adblock sends an X-Client-ID header insteads of URL parameters
   match = re.match(r"^adblock/([\d\.]+)$", info["clientid"], re.I) if info["clientid"] else None
   if match:
     info["addonName"] = "chromeadblock"
     info["addonVersion"] = match.group(1)
 
   last_version = params.get("lastVersion", ["unknown"])[0]
   if info["file"] == "notification.json" and last_version == "0" and (
       (info["addonName"] == "adblockplus" and info["addonVersion"] == "2.3.1") or
       (info["addonName"] in ("adblockpluschrome", "adblockplusopera") and info["addonVersion"] == "1.5.2")
     ):
     # Broken notification version number in these releases, treat like unknown
     last_version = "unknown"
 
   if last_version == "unknown":
     info["downloadInterval"] = "unknown"
   elif last_version == "0":
     info["downloadInterval"] = "unknown"
     info["firstDownload"] = info["firstInMonth"] = info["firstInWeek"] = info["firstInDay"] = True
   else:
     try:
-      last_update = datetime.strptime(last_version, "%Y%m%d%H%M")
+      last_update = parse_lastversion(last_version)
       diff = info["time"] - last_update
       if diff.days >= 365:
         info["downloadInterval"] = "%i year(s)" % (diff.days / 365)
       elif diff.days >= 30:
         info["downloadInterval"] = "%i month(s)" % (diff.days / 30)
       elif diff.days >= 1:
         info["downloadInterval"] = "%i day(s)" % diff.days
       else:
         info["downloadInterval"] = "%i hour(s)" % (diff.seconds / 3600)
 
       if last_update.year != info["time"].year or last_update.month != info["time"].month:
         info["firstInMonth"] = info["firstInDay"] = True
       elif last_update.day != info["time"].day:
         info["firstInDay"] = True
 
-      if last_update.isocalendar()[0:2] != info["time"].isocalendar()[0:2]:
+      if get_week(last_update) != get_week(info["time"]):
         info["firstInWeek"] = True
     except ValueError:
       info["downloadInterval"] = "unknown"
       pass
 
 def parse_addon_name(file):
   if "/" in file:
     return file.split("/")[-2]
   else:
     return None
@@ skipping 141 matching lines @@
 
 if __name__ == "__main__":
   setupStderr()
 
   verbose = (len(sys.argv) >= 2 and sys.argv[1] == "verbose")
   geo = GeoIP.open(get_config().get("stats", "geoip_db"), GeoIP.GEOIP_MEMORY_CACHE)
   result = parse_stdin(geo, verbose)
 
   with codecs.open(get_config().get("stats", "tempFile"), "wb", encoding="utf-8") as file:
     simplejson.dump(result, file, indent=2, sort_keys=True)