Stats processing: Add previousDownload field in addition to downloadInterval

sitescripts/stats/test/logprocessor.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2013 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 106 matching lines @@
     tests = [
       (
         datetime(2013, 07, 31, 12, 03, 00),
         "easylist.txt",
         "",
         "",
         "unknown/unknown",
         "unknown/unknown",
         "unknown/unknown",
         "unknown",
+        "unknown",
         "",
       ),
       (
         datetime(2013, 07, 31, 12, 03, 00),
         "easylist.txt",
         "addonName=adblockplus&addonVersion=2.3.1&application=firefox&applicationVersion=22.0a1&platform=gecko&platformVersion=23.0&lastVersion=0",
         "-",
         "adblockplus/2.3.1",
         "firefox/22.0",
         "gecko/23.0",
         "unknown",
+        "unknown",
         "firstDownload",
       ),
       (
         datetime(2013, 07, 31, 12, 03, 00),
         "easylist.txt",
         "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion=23.0&lastVersion=201307311200",
         "-",
         "adblockplus/2.3.1",
         "unknown/unknown",
         "gecko/23.0",
         "0 hour(s)",
+        "same day",
         "",
       ),
       (
         datetime(2013, 07, 31, 12, 03, 00),
         "easylist.txt",
         "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion=23.0&lastVersion=201307302200",
         "-",
         "adblockplus/2.3.1",
         "unknown/unknown",
         "gecko/23.0",
         "14 hour(s)",
+        "1 day(s)",
         "firstInDay",
       ),
       (
         datetime(2013, 07, 31, 12, 03, 00),
         "easylist.txt",
         "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion=23.0&lastVersion=201307282200",
         "-",
         "adblockplus/2.3.1",
         "unknown/unknown",
         "gecko/23.0",
         "2 day(s)",
+        "3 day(s)",
         "firstInWeek firstInDay",
       ),
       (
         datetime(2013, 8, 2, 12, 03, 00),
         "easylist.txt",
         "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion=23.0&lastVersion=201307311200",
         "-",
         "adblockplus/2.3.1",
         "unknown/unknown",
         "gecko/23.0",
         "2 day(s)",
+        "2 day(s)",
         "firstInMonth firstInDay",
       ),
       (
         datetime(2013, 07, 31, 12, 03, 00),
         "easylist.txt",
         "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion=23.0&lastVersion=201306302200",
         "-",
         "adblockplus/2.3.1",
         "unknown/unknown",
         "gecko/23.0",
         "1 month(s)",
+        "1 month(s)",
         "firstInMonth firstInWeek firstInDay",
       ),
       (
         datetime(2013, 07, 31, 12, 03, 00),
         "easylist.txt",
         "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion=23.0&lastVersion=201305302200",
         "-",
         "adblockplus/2.3.1",
         "unknown/unknown",
         "gecko/23.0",
         "2 month(s)",
+        "2 month(s)",
         "firstInMonth firstInWeek firstInDay",
       ),
       (
         datetime(2013, 07, 31, 12, 03, 00),
         "easylist.txt",
         "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion=23.0.1&lastVersion=201206302200",
         "-",
         "adblockplus/2.3.1",
         "unknown/unknown",
         "gecko/23.0",
         "1 year(s)",
+        "1 year(s)",
         "firstInMonth firstInWeek firstInDay",
       ),
       (
         datetime(2013, 07, 31, 12, 03, 00),
         "easylist.txt",
         "_=1375142394357",
         "AdBlock/2.5.4",
         "chromeadblock/2.5.4",
         "unknown/unknown",
         "unknown/unknown",
         "unknown",
+        "unknown",
         ""
       ),
     ]
-    for time, file, query, clientid, expected_addon, expected_application, expected_platform, expected_interval, expected_flags in tests:
+    for time, file, query, clientid, expected_addon, expected_application, expected_platform, expected_interval, expected_previous, expected_flags in tests:
       info = {"time": time, "file": file, "query": query, "clientid": clientid}
       logprocessor.parse_downloader_query(info)
       self.assertEqual("%s/%s" % (info["addonName"], info["addonVersion"]), expected_addon, "Add-on for query '%s'" % query)
       self.assertEqual("%s/%s" % (info["application"], info["applicationVersion"]), expected_application, "Application for query '%s'" % query)
       self.assertEqual("%s/%s" % (info["platform"], info["platformVersion"]), expected_platform, "Platform for query '%s'" % query)
       self.assertEqual(info["downloadInterval"], expected_interval, "Download interval for query '%s'" % query)
+      self.assertEqual(info["previousDownload"], expected_previous, "Previous download for query '%s'" % query)
 
       flags = []
       for flag in ("firstDownload", "firstInMonth", "firstInWeek", "firstInDay"):
         if flag in info:
           flags.append(flag)
       self.assertEqual(" ".join(flags), expected_flags, "Flags for query '%s'" % query)
 
   def test_nameparsing(self):
     tests = [
       ("devbuilds/adblockplus/update.rdf", "adblockplus"),
@@ skipping 140 matching lines @@
           "addonName": "adblockplusie",
           "addonVersion": "2.0",
           "fullAddon": "adblockplusie 2.0",
           "application": "msie64",
           "applicationVersion": "10.0",
           "fullApplication": "msie64 10.0",
           "platform": "libadblockplus",
           "platformVersion": "1.0",
           "fullPlatform": "libadblockplus 1.0",
           "downloadInterval": "unknown",
+          "previousDownload": "unknown",
           "firstDownload": True,
         }
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /adblockplusandroid-1.1.2.apk HTTP/1.1" 200 49152 "https://adblockplus.org/en/android-install" "Mozilla/5.0 (Linux; U; Android 4.1.2; es-es; GT-I9100 Build/JZO54K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" "-" https "en-US" "downloads.adblockplus.org" "-"',
         {
           "mirror": "mirror123",
           "ip": "1.2.3.4",
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
@@ skipping 53 matching lines @@
           "addonName": "adblockplus",
           "addonVersion": "2.3.2",
           "fullAddon": "adblockplus 2.3.2",
           "application": "firefox",
           "applicationVersion": "22.0",
           "fullApplication": "firefox 22.0",
           "platform": "gecko",
           "platformVersion": "22.0",
           "fullPlatform": "gecko 22.0",
           "downloadInterval": "2 hour(s)",
+          "previousDownload": "same day",
         }
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylist.txt?_=1375446528229 HTTP/1.1" 200 326120 "-" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https "nl-NL,nl;q=0.8,en-US;q=0.6,en;q=0.4" "easylist-downloads.adblockplus.org" "AdBlock/2.6.2"',
         {
           "mirror": "mirror123",
           "ip": "1.2.3.4",
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
@@ skipping 10 matching lines @@
           "addonName": "chromeadblock",
           "addonVersion": "2.6.2",
           "fullAddon": "chromeadblock 2.6.2",
           "application": "unknown",
           "applicationVersion": "unknown",
           "fullApplication": "unknown unknown",
           "platform": "unknown",
           "platformVersion": "unknown",
           "fullPlatform": "unknown unknown",
           "downloadInterval": "unknown",
+          "previousDownload": "unknown",
         }
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.txt HTTP/1.1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"',
         {
           "mirror": "mirror123",
           "ip": "1.2.3.4",
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
@@ skipping 10 matching lines @@
           "addonName": "unknown",
           "addonVersion": "unknown",
           "fullAddon": "unknown unknown",
           "application": "unknown",
           "applicationVersion": "unknown",
           "fullApplication": "unknown unknown",
           "platform": "unknown",
           "platformVersion": "unknown",
           "fullPlatform": "unknown unknown",
           "downloadInterval": "unknown",
+          "previousDownload": "unknown",
         }
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.tpl HTTP/1.1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"',
         {
           "mirror": "mirror123",
           "ip": "1.2.3.4",
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
@@ skipping 30 matching lines @@
           "addonName": "adblockpluschrome",
           "addonVersion": "1.5.3",
           "fullAddon": "adblockpluschrome 1.5.3",
           "application": "chrome",
           "applicationVersion": "28.0",
           "fullApplication": "chrome 28.0",
           "platform": "chromium",
           "platformVersion": "28.0",
           "fullPlatform": "chromium 28.0",
           "downloadInterval": "1 day(s)",
+          "previousDownload": "2 day(s)",
           "firstInDay": True,
         }
       ),
     ]
     for line, expected_record in tests:
       logprocessor.mirror_name = "mirror123"
       self.assertEqual(logprocessor.parse_record(line, set(), FakeGeo(), FakeGeo()), expected_record, "Parsing log line '%s'" % line)
 
   def test_record_adding(self):
     tests = [
@@ skipping 47 matching lines @@
           "addonName": {"bar": {"hits": 1, "bandwidth": 200}}
         },
       ),
     ]
     for info, section, ignored_fields, expected_result in tests:
       logprocessor.add_record(info, section, ignored_fields)
       self.assertEqual(section, expected_result)
 
 if __name__ == '__main__':
   unittest.main()