Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code
Issues
All Issues

Side by Side Diff: sitescripts/stats/test/logprocessor.py

Issue 29345242: Noissue - Adapt quotes for compliance with our coding style in sitescripts (Closed)
Patch Set: Fixed raw string Created May 30, 2016, 8:47 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « sitescripts/stats/test/common.py ('k') | sitescripts/subscriptions/bin/updateSubscriptionDownloads.py » ('j') | sitescripts/urlfixer/bin/topDomains.py » ('J')
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-2016 Eyeo GmbH 2 # Copyright (C) 2006-2016 Eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 import unittest 16 import unittest
17 import sitescripts.stats.bin.logprocessor as logprocessor 17 import sitescripts.stats.bin.logprocessor as logprocessor
18 from datetime import datetime 18 from datetime import datetime
19 19
20 20
21 class Test(unittest.TestCase): 21 class Test(unittest.TestCase):
22 longMessage = True 22 longMessage = True
23 maxDiff = None 23 maxDiff = None
24 24
25 def test_uaparsing(self): 25 def test_uaparsing(self):
26 tests = [ 26 tests = [
27 ("Firefox", "25.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25. 0) Gecko/20130730 Firefox/25.0"), 27 ('Firefox', '25.0', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25. 0) Gecko/20130730 Firefox/25.0'),
28 ("Firefox", "25.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:28. 0) Gecko/20130730 Firefox/25.0"), 28 ('Firefox', '25.0', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:28. 0) Gecko/20130730 Firefox/25.0'),
29 ("Firefox", "25.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25. 0.1) Gecko/20130730 Firefox/25.0.1"), 29 ('Firefox', '25.0', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25. 0.1) Gecko/20130730 Firefox/25.0.1'),
30 ("Firefox Mobile", "15.0", "Mozilla/5.0 (Maemo; Mobile; rv:15.0) Gec ko/20120829 Firefox/15.0 Fennec/15.0"), 30 ('Firefox Mobile', '15.0', 'Mozilla/5.0 (Maemo; Mobile; rv:15.0) Gec ko/20120829 Firefox/15.0 Fennec/15.0'),
31 ("Firefox Mobile", "14.0", "Mozilla/5.0 (Android; Mobile; rv:14.0) G ecko/14.0 Firefox/14.0a2"), 31 ('Firefox Mobile', '14.0', 'Mozilla/5.0 (Android; Mobile; rv:14.0) G ecko/14.0 Firefox/14.0a2'),
32 ("Firefox Tablet", "26.0", "Mozilla/5.0 (Android; Tablet; rv:26.0) G ecko/26.0 Firefox/26.0"), 32 ('Firefox Tablet', '26.0', 'Mozilla/5.0 (Android; Tablet; rv:26.0) G ecko/26.0 Firefox/26.0'),
33 ("Thunderbird", "24.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0 ) Gecko/20100101 Thunderbird/24.0a2 Lightning/2.6a2"), 33 ('Thunderbird', '24.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0 ) Gecko/20100101 Thunderbird/24.0a2 Lightning/2.6a2'),
34 ("SeaMonkey", "2.19", "Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/2 0100101 Firefox/22.0 SeaMonkey/2.19"), 34 ('SeaMonkey', '2.19', 'Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/2 0100101 Firefox/22.0 SeaMonkey/2.19'),
35 ("K-Meleon", "1.5", "Mozilla/5.0 (Windows; U; Windows NT 6.1; ru-RU; rv:1.8.1.24pre) Gecko/20100228 K-Meleon/1.5.4"), 35 ('K-Meleon', '1.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; ru-RU; rv:1.8.1.24pre) Gecko/20100228 K-Meleon/1.5.4'),
36 ("Prism", "1.0", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv :1.9.2.20) Gecko/20110803 Prism/1.0b4"), 36 ('Prism', '1.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv :1.9.2.20) Gecko/20110803 Prism/1.0b4'),
37 ("Gecko", "22.0", "Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/20100 101 FooBar/1.0"), 37 ('Gecko', '22.0', 'Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/20100 101 FooBar/1.0'),
38 ("Opera", "11.10", "Opera/9.80 (Android; Opera Mini/15.0.1162/30.355 8; U; it) Presto/2.8.119 Version/11.10"), 38 ('Opera', '11.10', 'Opera/9.80 (Android; Opera Mini/15.0.1162/30.355 8; U; it) Presto/2.8.119 Version/11.10'),
39 ("Opera", "9.80", "Opera/9.80 (Android; Opera Mini/15.0.1162/30.3558 ; U; it) Presto/2.8.119"), 39 ('Opera', '9.80', 'Opera/9.80 (Android; Opera Mini/15.0.1162/30.3558 ; U; it) Presto/2.8.119'),
40 ("Opera", "15.0", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/5 37.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36 OPR/15.0.1147.148"), 40 ('Opera', '15.0', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/5 37.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36 OPR/15.0.1147.148'),
41 ("Chrome", "28.0", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/ 537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36"), 41 ('Chrome', '28.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/ 537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36'),
42 ("Chrome", "28.0", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/ 537.36 (KHTML, like Gecko) Iron/28.0.1550.0 Chrome/28.0.1550.0 Safari/537.36"), 42 ('Chrome', '28.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/ 537.36 (KHTML, like Gecko) Iron/28.0.1550.0 Chrome/28.0.1550.0 Safari/537.36'),
43 ("Safari", "5.0", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16"), 43 ('Safari', '5.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16'),
44 ("Mobile Safari", "4.0", "Mozilla/5.0 (Linux; U; Android 4.0.4; pt-b r; LG-E400 Build/IMM76L; CyanogenMod-9) AppleWebKit/534.30 (KHTML, like Gecko) V ersion/4.0 Mobile Safari/534.30"), 44 ('Mobile Safari', '4.0', 'Mozilla/5.0 (Linux; U; Android 4.0.4; pt-b r; LG-E400 Build/IMM76L; CyanogenMod-9) AppleWebKit/534.30 (KHTML, like Gecko) V ersion/4.0 Mobile Safari/534.30'),
45 ("CoolNovo", "2.0.9", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK it/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36 CoolNovo/2.0.9. 11"), 45 ('CoolNovo', '2.0.9', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK it/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36 CoolNovo/2.0.9. 11'),
46 ("WebKit", "", "Mozilla/5.0 (Linux; U; Android 4.2.1; zh-CN; P7 Buil d/JRO03C) AppleWebKit/534.31 (KHTML, like Gecko) UCBrowser/9.2.0.308 U3/0.8.0 Mo bile Safari/534.31"), 46 ('WebKit', '', 'Mozilla/5.0 (Linux; U; Android 4.2.1; zh-CN; P7 Buil d/JRO03C) AppleWebKit/534.31 (KHTML, like Gecko) UCBrowser/9.2.0.308 U3/0.8.0 Mo bile Safari/534.31'),
47 ("MSIE", "10.0", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1 ; WOW64; Trident/6.0)"), 47 ('MSIE', '10.0', 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1 ; WOW64; Trident/6.0)'),
48 ("MSIE", "7.0", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; Trident/7.0; .NET4.0E; .NET4.0C)"), 48 ('MSIE', '7.0', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; Trident/7.0; .NET4.0E; .NET4.0C)'),
49 ("MSIE", "11.0", "Mozilla/5.0 (Windows NT 6.3; ARM; Trident/7.0; Tou ch; rv:11.0) like Gecko"), 49 ('MSIE', '11.0', 'Mozilla/5.0 (Windows NT 6.3; ARM; Trident/7.0; Tou ch; rv:11.0) like Gecko'),
50 ("Trident", "7.0", "Mozilla/5.0 (Windows NT 6.3; ARM; Trident/7.0; T ouch) like Gecko"), 50 ('Trident', '7.0', 'Mozilla/5.0 (Windows NT 6.3; ARM; Trident/7.0; T ouch) like Gecko'),
51 ("Edge", "12", "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KH TML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0"), 51 ('Edge', '12', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KH TML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0'),
52 ("Edge", "12", "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KH TML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136"), 52 ('Edge', '12', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KH TML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136'),
53 ("Edge", "12", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.36 ( KHTML, like Gecko) Edge/12.10240"), 53 ('Edge', '12', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.36 ( KHTML, like Gecko) Edge/12.10240'),
54 ("Edge", "12", "Mozilla/5.0 (Windows NT 6.3; Win64, x64; Touch) Appl eWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0 ( Touch; Trident/7.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR 2.0.50727; .NET CLR 3.0.30729; HPNTDFJS; H9P; InfoPath"), 54 ('Edge', '12', 'Mozilla/5.0 (Windows NT 6.3; Win64, x64; Touch) Appl eWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0 ( Touch; Trident/7.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR 2.0.50727; .NET CLR 3.0.30729; HPNTDFJS; H9P; InfoPath'),
55 ("Android", "", "AndroidDownloadManager"), 55 ('Android', '', 'AndroidDownloadManager'),
56 ("Android", "4.1", "AndroidDownloadManager/4.1.1 (Linux; U; Android 4.1.1; A210 Build/JRO03H)"), 56 ('Android', '4.1', 'AndroidDownloadManager/4.1.1 (Linux; U; Android 4.1.1; A210 Build/JRO03H)'),
57 ("Android", "4.0", "Dalvik/1.6.0 (Linux; U; Android 4.0.3; KFOT Buil d/IML74K)"), 57 ('Android', '4.0', 'Dalvik/1.6.0 (Linux; U; Android 4.0.3; KFOT Buil d/IML74K)'),
58 ("Android", "4.3", "Dalvik/1.6.0 (Linux; U; Android 4.3; Nexus 7 Bui ld/JSS15J)"), 58 ('Android', '4.3', 'Dalvik/1.6.0 (Linux; U; Android 4.3; Nexus 7 Bui ld/JSS15J)'),
59 ("Android", "", "Apache-HttpClient/UNAVAILABLE (java 1.4)"), 59 ('Android', '', 'Apache-HttpClient/UNAVAILABLE (java 1.4)'),
60 ("ABP", "", "Adblock Plus"), 60 ('ABP', '', 'Adblock Plus'),
61 ("Other", "", "-"), 61 ('Other', '', '-'),
62 ] 62 ]
63 for expected_browser, expected_version, ua in tests: 63 for expected_browser, expected_version, ua in tests:
64 self.assertEqual(logprocessor.parse_ua(ua), (expected_browser, expec ted_version), "Parsing user agent '%s'" % ua) 64 self.assertEqual(logprocessor.parse_ua(ua), (expected_browser, expec ted_version), "Parsing user agent '%s'" % ua)
65 65
66 def test_ipprocessing(self): 66 def test_ipprocessing(self):
67 country = None 67 country = None
68 68
69 class FakeGeo(object): 69 class FakeGeo(object):
70 ip_checked = None 70 ip_checked = None
71 71
72 def country_code_by_addr(self, ip): 72 def country_code_by_addr(self, ip):
73 self.ip_checked = ip 73 self.ip_checked = ip
74 return country 74 return country
75 75
76 tests = [ 76 tests = [
77 ("1.2.3.4", "xy", "1.2.3.4", "v4", "xy"), 77 ('1.2.3.4', 'xy', '1.2.3.4', 'v4', 'xy'),
78 ("::ffff:1.2.3.4", "xy", "1.2.3.4", "v4", "xy"), 78 ('::ffff:1.2.3.4', 'xy', '1.2.3.4', 'v4', 'xy'),
79 ("1.2.3.4", "--", "1.2.3.4", "v4", "unknown"), 79 ('1.2.3.4', '--', '1.2.3.4', 'v4', 'unknown'),
80 ("1.2.3.4", "", "1.2.3.4", "v4", "unknown"), 80 ('1.2.3.4', '', '1.2.3.4', 'v4', 'unknown'),
81 ("::ffff:1.2.3.4", None, "1.2.3.4", "v4", "unknown"), 81 ('::ffff:1.2.3.4', None, '1.2.3.4', 'v4', 'unknown'),
82 ("::1", "xy", "::1", "v6", "xy"), 82 ('::1', 'xy', '::1', 'v6', 'xy'),
83 ("FE80:0000:0000:0000:0202:B3FF:FE1E:8329", None, "FE80:0000:0000:00 00:0202:B3FF:FE1E:8329", "v6", "unknown"), 83 ('FE80:0000:0000:0000:0202:B3FF:FE1E:8329', None, 'FE80:0000:0000:00 00:0202:B3FF:FE1E:8329', 'v6', 'unknown'),
84 ] 84 ]
85 for ip, country, expected_ip, expected_type, expected_country in tests: 85 for ip, country, expected_ip, expected_type, expected_country in tests:
86 fake_geo = FakeGeo() 86 fake_geo = FakeGeo()
87 fake_geov6 = FakeGeo() 87 fake_geov6 = FakeGeo()
88 self.assertEqual(logprocessor.process_ip(ip, fake_geo, fake_geov6), (expected_ip, expected_country), "Processing IP '%s'" % ip) 88 self.assertEqual(logprocessor.process_ip(ip, fake_geo, fake_geov6), (expected_ip, expected_country), "Processing IP '%s'" % ip)
89 if expected_type == "v4": 89 if expected_type == 'v4':
90 self.assertEqual(fake_geo.ip_checked, expected_ip, "GeoIP check for IP '%s'" % ip) 90 self.assertEqual(fake_geo.ip_checked, expected_ip, "GeoIP check for IP '%s'" % ip)
91 self.assertEqual(fake_geov6.ip_checked, None, "GeoIPv6 check for IP '%s'" % ip) 91 self.assertEqual(fake_geov6.ip_checked, None, "GeoIPv6 check for IP '%s'" % ip)
92 else: 92 else:
93 self.assertEqual(fake_geo.ip_checked, None, "GeoIP check for IP '%s'" % ip) 93 self.assertEqual(fake_geo.ip_checked, None, "GeoIP check for IP '%s'" % ip)
94 self.assertEqual(fake_geov6.ip_checked, expected_ip, "GeoIPv6 ch eck for IP '%s'" % ip) 94 self.assertEqual(fake_geov6.ip_checked, expected_ip, "GeoIPv6 ch eck for IP '%s'" % ip)
95 95
96 def test_timeparsing(self): 96 def test_timeparsing(self):
97 tests = [ 97 tests = [
98 ("31/Jul/2013:12:03:37", 0, 0, datetime(2013, 07, 31, 12, 03, 37), " 201307"), 98 ('31/Jul/2013:12:03:37', 0, 0, datetime(2013, 07, 31, 12, 03, 37), ' 201307'),
99 ("31/Jul/2013:12:03:37", 5, 0, datetime(2013, 07, 31, 7, 03, 37), "2 01307"), 99 ('31/Jul/2013:12:03:37', 5, 0, datetime(2013, 07, 31, 7, 03, 37), '2 01307'),
100 ("31/Jul/2013:12:03:37", -5, 0, datetime(2013, 07, 31, 17, 03, 37), "201307"), 100 ('31/Jul/2013:12:03:37', -5, 0, datetime(2013, 07, 31, 17, 03, 37), '201307'),
101 ("31/Jul/2013:12:03:37", 5, 30, datetime(2013, 07, 31, 6, 33, 37), " 201307"), 101 ('31/Jul/2013:12:03:37', 5, 30, datetime(2013, 07, 31, 6, 33, 37), ' 201307'),
102 ("31/Jul/2013:12:03:37", -5, 30, datetime(2013, 07, 31, 17, 33, 37), "201307"), 102 ('31/Jul/2013:12:03:37', -5, 30, datetime(2013, 07, 31, 17, 33, 37), '201307'),
103 ] 103 ]
104 for timestr, tz_hours, tz_minutes, expected_time, expected_month in test s: 104 for timestr, tz_hours, tz_minutes, expected_time, expected_month in test s:
105 self.assertEqual(logprocessor.parse_time(timestr, tz_hours, tz_minut es), 105 self.assertEqual(logprocessor.parse_time(timestr, tz_hours, tz_minut es),
106 (expected_time, expected_month, expected_time.day, expected_time.weekday(), expected_time.hour), 106 (expected_time, expected_month, expected_time.day, expected_time.weekday(), expected_time.hour),
107 "Parsing time string '%s %+03i%02i'" % (timestr, tz _hours, tz_minutes)) 107 "Parsing time string '%s %+03i%02i'" % (timestr, tz _hours, tz_minutes))
108 108
109 def test_pathparsing(self): 109 def test_pathparsing(self):
110 tests = [ 110 tests = [
111 ("/foo.txt", "foo.txt", ""), 111 ('/foo.txt', 'foo.txt', ''),
112 ("/foo.txt?", "foo.txt", ""), 112 ('/foo.txt?', 'foo.txt', ''),
113 ("/foo.txt?asdf", "foo.txt", "asdf"), 113 ('/foo.txt?asdf', 'foo.txt', 'asdf'),
114 ("http://example.com/foo.txt", "foo.txt", ""), 114 ('http://example.com/foo.txt', 'foo.txt', ''),
115 ("/xyz/foo.txt?asdf", "xyz/foo.txt", "asdf"), 115 ('/xyz/foo.txt?asdf', 'xyz/foo.txt', 'asdf'),
116 ("/xyz/foo+bar.txt?asdf", "xyz/foo+bar.txt", "asdf"), 116 ('/xyz/foo+bar.txt?asdf', 'xyz/foo+bar.txt', 'asdf'),
117 ("/xyz/foo%2Bbar%2etxt?asdf", "xyz/foo+bar.txt", "asdf"), 117 ('/xyz/foo%2Bbar%2etxt?asdf', 'xyz/foo+bar.txt', 'asdf'),
118 ("/xyz/%D1%82%D0%B5%D1%81%D1%82.txt", u"xyz/\u0442\u0435\u0441\u0442 .txt", ""), 118 ('/xyz/%D1%82%D0%B5%D1%81%D1%82.txt', u'xyz/\u0442\u0435\u0441\u0442 .txt', ''),
119 ("/xyz/foo%8Cbar.txt?asdf", "xyz/foo%8Cbar.txt", "asdf"), 119 ('/xyz/foo%8Cbar.txt?asdf', 'xyz/foo%8Cbar.txt', 'asdf'),
120 ] 120 ]
121 for path, expected_file, expected_query in tests: 121 for path, expected_file, expected_query in tests:
122 self.assertEqual(logprocessor.parse_path(path), (expected_file, expe cted_query), "Parsing path '%s'" % path) 122 self.assertEqual(logprocessor.parse_path(path), (expected_file, expe cted_query), "Parsing path '%s'" % path)
123 123
124 def test_downloaderdata(self): 124 def test_downloaderdata(self):
125 tests = [ 125 tests = [
126 ( 126 (
127 datetime(2013, 07, 31, 12, 03, 00), 127 datetime(2013, 07, 31, 12, 03, 00),
128 "easylist.txt", 128 'easylist.txt',
129 "", 129 '',
130 "", 130 '',
131 "unknown/unknown", 131 'unknown/unknown',
132 "unknown/unknown", 132 'unknown/unknown',
133 "unknown/unknown", 133 'unknown/unknown',
134 "unknown", 134 'unknown',
135 "unknown", 135 'unknown',
136 "", 136 '',
137 ), 137 ),
138 ( 138 (
139 datetime(2013, 07, 31, 12, 03, 00), 139 datetime(2013, 07, 31, 12, 03, 00),
140 "easylist.txt", 140 'easylist.txt',
141 "addonName=adblockplus&addonVersion=2.3.1&application=firefox&ap plicationVersion=22.0a1&platform=gecko&platformVersion=23.0&lastVersion=0", 141 'addonName=adblockplus&addonVersion=2.3.1&application=firefox&ap plicationVersion=22.0a1&platform=gecko&platformVersion=23.0&lastVersion=0',
142 "-", 142 '-',
143 "adblockplus/2.3.1", 143 'adblockplus/2.3.1',
144 "firefox/22.0", 144 'firefox/22.0',
145 "gecko/23.0", 145 'gecko/23.0',
146 "unknown", 146 'unknown',
147 "unknown", 147 'unknown',
148 "firstDownload", 148 'firstDownload',
149 ), 149 ),
150 ( 150 (
151 datetime(2013, 07, 31, 12, 03, 00), 151 datetime(2013, 07, 31, 12, 03, 00),
152 "easylist.txt", 152 'easylist.txt',
153 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307311200-1/0", 153 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307311200-1/0',
154 "-", 154 '-',
155 "adblockplus/2.3.1", 155 'adblockplus/2.3.1',
156 "unknown/unknown", 156 'unknown/unknown',
157 "gecko/23.0", 157 'gecko/23.0',
158 "0 hour(s)", 158 '0 hour(s)',
159 "same day", 159 'same day',
160 "", 160 '',
161 ), 161 ),
162 ( 162 (
163 datetime(2013, 07, 31, 12, 03, 00), 163 datetime(2013, 07, 31, 12, 03, 00),
164 "easylist.txt", 164 'easylist.txt',
165 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307302200-1/3", 165 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307302200-1/3',
166 "-", 166 '-',
167 "adblockplus/2.3.1", 167 'adblockplus/2.3.1',
168 "unknown/unknown", 168 'unknown/unknown',
169 "gecko/23.0", 169 'gecko/23.0',
170 "14 hour(s)", 170 '14 hour(s)',
171 "1 day(s)", 171 '1 day(s)',
172 "firstInDay", 172 'firstInDay',
173 ), 173 ),
174 ( 174 (
175 datetime(2013, 07, 31, 12, 03, 00), 175 datetime(2013, 07, 31, 12, 03, 00),
176 "easylist.txt", 176 'easylist.txt',
177 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307282200", 177 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307282200',
178 "-", 178 '-',
179 "adblockplus/2.3.1", 179 'adblockplus/2.3.1',
180 "unknown/unknown", 180 'unknown/unknown',
181 "gecko/23.0", 181 'gecko/23.0',
182 "2 day(s)", 182 '2 day(s)',
183 "3 day(s)", 183 '3 day(s)',
184 "firstInWeek firstInDay", 184 'firstInWeek firstInDay',
185 ), 185 ),
186 ( 186 (
187 datetime(2013, 8, 2, 12, 03, 00), 187 datetime(2013, 8, 2, 12, 03, 00),
188 "easylist.txt", 188 'easylist.txt',
189 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307311200", 189 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307311200',
190 "-", 190 '-',
191 "adblockplus/2.3.1", 191 'adblockplus/2.3.1',
192 "unknown/unknown", 192 'unknown/unknown',
193 "gecko/23.0", 193 'gecko/23.0',
194 "2 day(s)", 194 '2 day(s)',
195 "2 day(s)", 195 '2 day(s)',
196 "firstInMonth firstInDay", 196 'firstInMonth firstInDay',
197 ), 197 ),
198 ( 198 (
199 datetime(2013, 07, 31, 12, 03, 00), 199 datetime(2013, 07, 31, 12, 03, 00),
200 "easylist.txt", 200 'easylist.txt',
201 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201306302200", 201 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201306302200',
202 "-", 202 '-',
203 "adblockplus/2.3.1", 203 'adblockplus/2.3.1',
204 "unknown/unknown", 204 'unknown/unknown',
205 "gecko/23.0", 205 'gecko/23.0',
206 "1 month(s)", 206 '1 month(s)',
207 "1 month(s)", 207 '1 month(s)',
208 "firstInMonth firstInWeek firstInDay", 208 'firstInMonth firstInWeek firstInDay',
209 ), 209 ),
210 ( 210 (
211 datetime(2013, 07, 31, 12, 03, 00), 211 datetime(2013, 07, 31, 12, 03, 00),
212 "easylist.txt", 212 'easylist.txt',
213 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201305302200", 213 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201305302200',
214 "-", 214 '-',
215 "adblockplus/2.3.1", 215 'adblockplus/2.3.1',
216 "unknown/unknown", 216 'unknown/unknown',
217 "gecko/23.0", 217 'gecko/23.0',
218 "2 month(s)", 218 '2 month(s)',
219 "2 month(s)", 219 '2 month(s)',
220 "firstInMonth firstInWeek firstInDay", 220 'firstInMonth firstInWeek firstInDay',
221 ), 221 ),
222 ( 222 (
223 datetime(2013, 07, 31, 12, 03, 00), 223 datetime(2013, 07, 31, 12, 03, 00),
224 "easylist.txt", 224 'easylist.txt',
225 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0.1&lastVersion=201206302200", 225 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0.1&lastVersion=201206302200',
226 "-", 226 '-',
227 "adblockplus/2.3.1", 227 'adblockplus/2.3.1',
228 "unknown/unknown", 228 'unknown/unknown',
229 "gecko/23.0", 229 'gecko/23.0',
230 "1 year(s)", 230 '1 year(s)',
231 "1 year(s)", 231 '1 year(s)',
232 "firstInMonth firstInWeek firstInDay", 232 'firstInMonth firstInWeek firstInDay',
233 ), 233 ),
234 ( 234 (
235 datetime(2013, 07, 31, 12, 03, 00), 235 datetime(2013, 07, 31, 12, 03, 00),
236 "easylist.txt", 236 'easylist.txt',
237 "addonName=adblockedge&addonVersion=2.1.2&platform=gecko&platfor mVersion=23.0.1&lastVersion=201206302200", 237 'addonName=adblockedge&addonVersion=2.1.2&platform=gecko&platfor mVersion=23.0.1&lastVersion=201206302200',
238 "-", 238 '-',
239 "adblockedge/2.1.2", 239 'adblockedge/2.1.2',
240 "unknown/unknown", 240 'unknown/unknown',
241 "gecko/23.0", 241 'gecko/23.0',
242 "1 year(s)", 242 '1 year(s)',
243 "unknown", 243 'unknown',
244 "firstInMonth firstInWeek firstInDay", 244 'firstInMonth firstInWeek firstInDay',
245 ), 245 ),
246 ( 246 (
247 datetime(2013, 07, 31, 12, 03, 00), 247 datetime(2013, 07, 31, 12, 03, 00),
248 "easylist.txt", 248 'easylist.txt',
249 "_=1375142394357", 249 '_=1375142394357',
250 "AdBlock/2.5.4", 250 'AdBlock/2.5.4',
251 "chromeadblock/2.5.4", 251 'chromeadblock/2.5.4',
252 "unknown/unknown", 252 'unknown/unknown',
253 "unknown/unknown", 253 'unknown/unknown',
254 "unknown", 254 'unknown',
255 "unknown", 255 'unknown',
256 "" 256 ''
257 ), 257 ),
258 ] 258 ]
259 for time, file, query, clientid, expected_addon, expected_application, e xpected_platform, expected_interval, expected_previous, expected_flags in tests: 259 for time, file, query, clientid, expected_addon, expected_application, e xpected_platform, expected_interval, expected_previous, expected_flags in tests:
260 info = {"time": time, "file": file, "query": query, "clientid": clie ntid} 260 info = {'time': time, 'file': file, 'query': query, 'clientid': clie ntid}
261 logprocessor.parse_downloader_query(info) 261 logprocessor.parse_downloader_query(info)
262 self.assertEqual("%s/%s" % (info["addonName"], info["addonVersion"]) , expected_addon, "Add-on for query '%s'" % query) 262 self.assertEqual('%s/%s' % (info['addonName'], info['addonVersion']) , expected_addon, "Add-on for query '%s'" % query)
263 self.assertEqual("%s/%s" % (info["application"], info["applicationVe rsion"]), expected_application, "Application for query '%s'" % query) 263 self.assertEqual('%s/%s' % (info['application'], info['applicationVe rsion']), expected_application, "Application for query '%s'" % query)
264 self.assertEqual("%s/%s" % (info["platform"], info["platformVersion" ]), expected_platform, "Platform for query '%s'" % query) 264 self.assertEqual('%s/%s' % (info['platform'], info['platformVersion' ]), expected_platform, "Platform for query '%s'" % query)
265 self.assertEqual(info["downloadInterval"], expected_interval, "Downl oad interval for query '%s'" % query) 265 self.assertEqual(info['downloadInterval'], expected_interval, "Downl oad interval for query '%s'" % query)
266 self.assertEqual(info["previousDownload"], expected_previous, "Previ ous download for query '%s'" % query) 266 self.assertEqual(info['previousDownload'], expected_previous, "Previ ous download for query '%s'" % query)
267 267
268 flags = [] 268 flags = []
269 for flag in ("firstDownload", "firstInMonth", "firstInWeek", "firstI nDay"): 269 for flag in ('firstDownload', 'firstInMonth', 'firstInWeek', 'firstI nDay'):
270 if flag in info: 270 if flag in info:
271 flags.append(flag) 271 flags.append(flag)
272 self.assertEqual(" ".join(flags), expected_flags, "Flags for query ' %s'" % query) 272 self.assertEqual(' '.join(flags), expected_flags, "Flags for query ' %s'" % query)
273 273
274 def test_nameparsing(self): 274 def test_nameparsing(self):
275 tests = [ 275 tests = [
276 ("devbuilds/adblockplus/update.rdf", "adblockplus"), 276 ('devbuilds/adblockplus/update.rdf', 'adblockplus'),
277 ("adblockpluschrome-experimental/updates.xml", "adblockpluschrome-ex perimental"), 277 ('adblockpluschrome-experimental/updates.xml', 'adblockpluschrome-ex perimental'),
278 ("update.json", None), 278 ('update.json', None),
279 ] 279 ]
280 for file, expected_name in tests: 280 for file, expected_name in tests:
281 self.assertEqual(logprocessor.parse_addon_name(file), expected_name, "Getting add-on name for file '%s'" % file) 281 self.assertEqual(logprocessor.parse_addon_name(file), expected_name, "Getting add-on name for file '%s'" % file)
282 282
283 def test_geckoqueryparsing(self): 283 def test_geckoqueryparsing(self):
284 tests = [ 284 tests = [
285 ( 285 (
286 "reqVersion=2&id={d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d}&version= 2.3.1.3707&maxAppVersion=25.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e- 13a3a9e97384}&appVersion=22.0&appOS=Darwin&appABI=x86_64-gcc3&locale=en-US&curre ntAppVersion=22.0&updateType=112", 286 'reqVersion=2&id={d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d}&version= 2.3.1.3707&maxAppVersion=25.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e- 13a3a9e97384}&appVersion=22.0&appOS=Darwin&appABI=x86_64-gcc3&locale=en-US&curre ntAppVersion=22.0&updateType=112',
287 "2.3.1.3707", "firefox", "22.0" 287 '2.3.1.3707', 'firefox', '22.0'
288 ), 288 ),
289 ( 289 (
290 "reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74 &maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e973 84}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVers ion=25.0a1&updateType=97", 290 'reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74 &maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e973 84}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVers ion=25.0a1&updateType=97',
291 "1.0.4a.74", "firefox", "25.0" 291 '1.0.4a.74', 'firefox', '25.0'
292 ), 292 ),
293 ( 293 (
294 "reqVersion=2&id={d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d}&version= 1.3a.20100925&maxAppVersion=2.1b1&status=userEnabled,incompatible&appID={92650c4 d-4b8e-4d2a-b7eb-24ecf4f6b63a}&appVersion=2.19&appOS=WINNT&appABI=x86-msvc&local e=en-US&currentAppVersion=2.19&updateType=112", 294 'reqVersion=2&id={d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d}&version= 1.3a.20100925&maxAppVersion=2.1b1&status=userEnabled,incompatible&appID={92650c4 d-4b8e-4d2a-b7eb-24ecf4f6b63a}&appVersion=2.19&appOS=WINNT&appABI=x86-msvc&local e=en-US&currentAppVersion=2.19&updateType=112',
295 "1.3a.20100925", "seamonkey", "2.19" 295 '1.3a.20100925', 'seamonkey', '2.19'
296 ), 296 ),
297 ] 297 ]
298 for query, expected_version, expected_application, expected_applicationv ersion in tests: 298 for query, expected_version, expected_application, expected_applicationv ersion in tests:
299 self.assertEqual(logprocessor.parse_gecko_query(query), (expected_ve rsion, expected_application, expected_applicationversion), "Parsing Gecko query '%s'" % query) 299 self.assertEqual(logprocessor.parse_gecko_query(query), (expected_ve rsion, expected_application, expected_applicationversion), "Parsing Gecko query '%s'" % query)
300 300
301 def test_chromequeryparsing(self): 301 def test_chromequeryparsing(self):
302 tests = [ 302 tests = [
303 ( 303 (
304 "os=win&arch=x86&nacl_arch=x86-64&prod=chromecrx&prodchannel=sta ble&prodversion=28.0.1500.72&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3 .977%26uc", 304 'os=win&arch=x86&nacl_arch=x86-64&prod=chromecrx&prodchannel=sta ble&prodversion=28.0.1500.72&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3 .977%26uc',
305 "1.5.3.977", "chrome", "28.0" 305 '1.5.3.977', 'chrome', '28.0'
306 ), 306 ),
307 ( 307 (
308 "x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3.977%26uc", 308 'x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3.977%26uc',
309 "1.5.3.977", "unknown", "unknown" 309 '1.5.3.977', 'unknown', 'unknown'
310 ), 310 ),
311 ( 311 (
312 "api=15&build=256&locale=ru_ru&device=LGE%20LG-P990", 312 'api=15&build=256&locale=ru_ru&device=LGE%20LG-P990',
313 "unknown", "unknown", "unknown" 313 'unknown', 'unknown', 'unknown'
314 ), 314 ),
315 ] 315 ]
316 for query, expected_version, expected_application, expected_applicationv ersion in tests: 316 for query, expected_version, expected_application, expected_applicationv ersion in tests:
317 self.assertEqual(logprocessor.parse_chrome_query(query), (expected_v ersion, expected_application, expected_applicationversion), "Parsing Chrome quer y '%s'" % query) 317 self.assertEqual(logprocessor.parse_chrome_query(query), (expected_v ersion, expected_application, expected_applicationversion), "Parsing Chrome quer y '%s'" % query)
318 318
319 def test_updateflagparsing(self): 319 def test_updateflagparsing(self):
320 tests = [ 320 tests = [
321 ("update", "update"), 321 ('update', 'update'),
322 ("", "install"), 322 ('', 'install'),
323 ("foo", "install"), 323 ('foo', 'install'),
324 ("update&foo", "install"), 324 ('update&foo', 'install'),
325 ] 325 ]
326 for query, expected_result in tests: 326 for query, expected_result in tests:
327 self.assertEqual(logprocessor.parse_update_flag(query), expected_res ult, "Checking update flag for query '%s'" % query) 327 self.assertEqual(logprocessor.parse_update_flag(query), expected_res ult, "Checking update flag for query '%s'" % query)
328 328
329 def test_recordparsing(self): 329 def test_recordparsing(self):
330 class FakeGeo(object): 330 class FakeGeo(object):
331 def country_code_by_addr(self, ip): 331 def country_code_by_addr(self, ip):
332 return "xy" 332 return 'xy'
333 333
334 tests = [ 334 tests = [
335 ( 335 (
336 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adbloc kpluschrome/updates.xml?os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&prodchan nel=stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v% 3D1.5.3.977%26uc HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 1 0_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36" "-" https" "en-US" "downloads.adblockplus.org" "-"', 336 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adbloc kpluschrome/updates.xml?os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&prodchan nel=stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v% 3D1.5.3.977%26uc HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 1 0_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36" "-" https" "en-US" "downloads.adblockplus.org" "-"',
337 { 337 {
338 "ip": "1.2.3.4", 338 'ip': '1.2.3.4',
339 "country": "xy", 339 'country': 'xy',
340 "time": datetime(2013, 7, 31, 17, 33, 8), 340 'time': datetime(2013, 7, 31, 17, 33, 8),
341 "day": 31, 341 'day': 31,
342 "weekday": 2, 342 'weekday': 2,
343 "hour": 17, 343 'hour': 17,
344 "month": "201307", 344 'month': '201307',
345 "file": "devbuilds/adblockpluschrome/updates.xml", 345 'file': 'devbuilds/adblockpluschrome/updates.xml',
346 "query": "os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&pr odchannel=stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepk n%26v%3D1.5.3.977%26uc", 346 'query': 'os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&pr odchannel=stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepk n%26v%3D1.5.3.977%26uc',
347 "size": 867, 347 'size': 867,
348 "referrer": "-", 348 'referrer': '-',
349 "status": 200, 349 'status': 200,
350 "ua": "Chrome", 350 'ua': 'Chrome',
351 "uaversion": "28.0", 351 'uaversion': '28.0',
352 "fullua": "Chrome 28.0", 352 'fullua': 'Chrome 28.0',
353 "clientid": "-", 353 'clientid': '-',
354 "addonName": "adblockpluschrome", 354 'addonName': 'adblockpluschrome',
355 "addonVersion": "1.5.3.977", 355 'addonVersion': '1.5.3.977',
356 "fullAddon": "adblockpluschrome 1.5.3.977", 356 'fullAddon': 'adblockpluschrome 1.5.3.977',
357 "application": "chrome", 357 'application': 'chrome',
358 "applicationVersion": "28.0", 358 'applicationVersion': '28.0',
359 "fullApplication": "chrome 28.0", 359 'fullApplication': 'chrome 28.0',
360 } 360 }
361 ), 361 ),
362 ( 362 (
363 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 404 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"', 363 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 404 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"',
364 None 364 None
365 ), 365 ),
366 ( 366 (
367 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 301 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"', 367 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 301 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"',
368 { 368 {
369 "ip": "1.2.3.4", 369 'ip': '1.2.3.4',
370 "country": "xy", 370 'country': 'xy',
371 "time": datetime(2013, 7, 31, 17, 33, 8), 371 'time': datetime(2013, 7, 31, 17, 33, 8),
372 "day": 31, 372 'day': 31,
373 "weekday": 2, 373 'weekday': 2,
374 "hour": 17, 374 'hour': 17,
375 "month": "201307", 375 'month': '201307',
376 "file": "devbuilds/abpcustomization/update.rdf", 376 'file': 'devbuilds/abpcustomization/update.rdf',
377 "query": "reqVersion=2&id=customization@adblockplus.org&vers ion=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b 0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&c urrentAppVersion=25.0a1&updateType=97", 377 'query': 'reqVersion=2&id=customization@adblockplus.org&vers ion=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b 0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&c urrentAppVersion=25.0a1&updateType=97',
378 "size": 867, 378 'size': 867,
379 "referrer": "-", 379 'referrer': '-',
380 "status": 301, 380 'status': 301,
381 "ua": "Firefox", 381 'ua': 'Firefox',
382 "uaversion": "25.0", 382 'uaversion': '25.0',
383 "fullua": "Firefox 25.0", 383 'fullua': 'Firefox 25.0',
384 "clientid": "-", 384 'clientid': '-',
385 "addonName": "abpcustomization", 385 'addonName': 'abpcustomization',
386 "addonVersion": "1.0.4a.74", 386 'addonVersion': '1.0.4a.74',
387 "fullAddon": "abpcustomization 1.0.4a.74", 387 'fullAddon': 'abpcustomization 1.0.4a.74',
388 "application": "firefox", 388 'application': 'firefox',
389 "applicationVersion": "25.0", 389 'applicationVersion': '25.0',
390 "fullApplication": "firefox 25.0", 390 'fullApplication': 'firefox 25.0',
391 } 391 }
392 ), 392 ),
393 ( 393 (
394 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 302 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"', 394 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 302 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"',
395 { 395 {
396 "ip": "1.2.3.4", 396 'ip': '1.2.3.4',
397 "country": "xy", 397 'country': 'xy',
398 "time": datetime(2013, 7, 31, 17, 33, 8), 398 'time': datetime(2013, 7, 31, 17, 33, 8),
399 "day": 31, 399 'day': 31,
400 "weekday": 2, 400 'weekday': 2,
401 "hour": 17, 401 'hour': 17,
402 "month": "201307", 402 'month': '201307',
403 "file": "devbuilds/abpcustomization/update.rdf", 403 'file': 'devbuilds/abpcustomization/update.rdf',
404 "query": "reqVersion=2&id=customization@adblockplus.org&vers ion=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b 0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&c urrentAppVersion=25.0a1&updateType=97", 404 'query': 'reqVersion=2&id=customization@adblockplus.org&vers ion=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b 0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&c urrentAppVersion=25.0a1&updateType=97',
405 "size": 867, 405 'size': 867,
406 "referrer": "-", 406 'referrer': '-',
407 "status": 302, 407 'status': 302,
408 "ua": "Firefox", 408 'ua': 'Firefox',
409 "uaversion": "25.0", 409 'uaversion': '25.0',
410 "fullua": "Firefox 25.0", 410 'fullua': 'Firefox 25.0',
411 "clientid": "-", 411 'clientid': '-',
412 "addonName": "abpcustomization", 412 'addonName': 'abpcustomization',
413 "addonVersion": "1.0.4a.74", 413 'addonVersion': '1.0.4a.74',
414 "fullAddon": "abpcustomization 1.0.4a.74", 414 'fullAddon': 'abpcustomization 1.0.4a.74',
415 "application": "firefox", 415 'application': 'firefox',
416 "applicationVersion": "25.0", 416 'applicationVersion': '25.0',
417 "fullApplication": "firefox 25.0", 417 'fullApplication': 'firefox 25.0',
418 } 418 }
419 ), 419 ),
420 ( 420 (
421 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.unknown?reqVersion=2&id=customization@adblockplus.org&version= 1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-1 3a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&curre ntAppVersion=25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "down loads.adblockplus.org" "-"', 421 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.unknown?reqVersion=2&id=customization@adblockplus.org&version= 1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-1 3a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&curre ntAppVersion=25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "down loads.adblockplus.org" "-"',
422 None 422 None
423 ), 423 ),
424 ( 424 (
425 '1.2.3.4 corrupted', 425 '1.2.3.4 corrupted',
426 None 426 None
427 ), 427 ),
428 ( 428 (
429 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"', 429 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"',
430 { 430 {
431 "ip": "1.2.3.4", 431 'ip': '1.2.3.4',
432 "country": "xy", 432 'country': 'xy',
433 "time": datetime(2013, 7, 31, 17, 33, 8), 433 'time': datetime(2013, 7, 31, 17, 33, 8),
434 "day": 31, 434 'day': 31,
435 "weekday": 2, 435 'weekday': 2,
436 "hour": 17, 436 'hour': 17,
437 "month": "201307", 437 'month': '201307',
438 "file": "devbuilds/abpcustomization/update.rdf", 438 'file': 'devbuilds/abpcustomization/update.rdf',
439 "query": "reqVersion=2&id=customization@adblockplus.org&vers ion=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b 0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&c urrentAppVersion=25.0a1&updateType=97", 439 'query': 'reqVersion=2&id=customization@adblockplus.org&vers ion=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b 0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&c urrentAppVersion=25.0a1&updateType=97',
440 "size": 867, 440 'size': 867,
441 "referrer": "-", 441 'referrer': '-',
442 "status": 200, 442 'status': 200,
443 "ua": "Firefox", 443 'ua': 'Firefox',
444 "uaversion": "25.0", 444 'uaversion': '25.0',
445 "fullua": "Firefox 25.0", 445 'fullua': 'Firefox 25.0',
446 "clientid": "-", 446 'clientid': '-',
447 "addonName": "abpcustomization", 447 'addonName': 'abpcustomization',
448 "addonVersion": "1.0.4a.74", 448 'addonVersion': '1.0.4a.74',
449 "fullAddon": "abpcustomization 1.0.4a.74", 449 'fullAddon': 'abpcustomization 1.0.4a.74',
450 "application": "firefox", 450 'application': 'firefox',
451 "applicationVersion": "25.0", 451 'applicationVersion': '25.0',
452 "fullApplication": "firefox 25.0", 452 'fullApplication': 'firefox 25.0',
453 } 453 }
454 ), 454 ),
455 ( 455 (
456 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adbloc kplusie/update.json?addonName=adblockplusie&addonVersion=2.0&application=msie64& applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion= 0 HTTP/1.1" 200 867 "-" "Adblock Plus" "-" https" "en-US" "downloads.adblockplus .org" "-"', 456 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adbloc kplusie/update.json?addonName=adblockplusie&addonVersion=2.0&application=msie64& applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion= 0 HTTP/1.1" 200 867 "-" "Adblock Plus" "-" https" "en-US" "downloads.adblockplus .org" "-"',
457 { 457 {
458 "ip": "1.2.3.4", 458 'ip': '1.2.3.4',
459 "country": "xy", 459 'country': 'xy',
460 "time": datetime(2013, 7, 31, 17, 33, 8), 460 'time': datetime(2013, 7, 31, 17, 33, 8),
461 "day": 31, 461 'day': 31,
462 "weekday": 2, 462 'weekday': 2,
463 "hour": 17, 463 'hour': 17,
464 "month": "201307", 464 'month': '201307',
465 "file": "devbuilds/adblockplusie/update.json", 465 'file': 'devbuilds/adblockplusie/update.json',
466 "query": "addonName=adblockplusie&addonVersion=2.0&applicati on=msie64&applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&la stVersion=0", 466 'query': 'addonName=adblockplusie&addonVersion=2.0&applicati on=msie64&applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&la stVersion=0',
467 "size": 867, 467 'size': 867,
468 "referrer": "-", 468 'referrer': '-',
469 "status": 200, 469 'status': 200,
470 "ua": "ABP", 470 'ua': 'ABP',
471 "uaversion": "", 471 'uaversion': '',
472 "fullua": "ABP ", 472 'fullua': 'ABP ',
473 "clientid": "-", 473 'clientid': '-',
474 "addonName": "adblockplusie", 474 'addonName': 'adblockplusie',
475 "addonVersion": "2.0", 475 'addonVersion': '2.0',
476 "fullAddon": "adblockplusie 2.0", 476 'fullAddon': 'adblockplusie 2.0',
477 "application": "msie64", 477 'application': 'msie64',
478 "applicationVersion": "10.0", 478 'applicationVersion': '10.0',
479 "fullApplication": "msie64 10.0", 479 'fullApplication': 'msie64 10.0',
480 "platform": "libadblockplus", 480 'platform': 'libadblockplus',
481 "platformVersion": "1.0", 481 'platformVersion': '1.0',
482 "fullPlatform": "libadblockplus 1.0", 482 'fullPlatform': 'libadblockplus 1.0',
483 "downloadInterval": "unknown", 483 'downloadInterval': 'unknown',
484 "previousDownload": "unknown", 484 'previousDownload': 'unknown',
485 "firstDownload": True, 485 'firstDownload': True,
486 } 486 }
487 ), 487 ),
488 ( 488 (
489 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /adblockplusandro id-1.1.2.apk HTTP/1.1" 200 49152 "https://adblockplus.org/en/android-install" "M ozilla/5.0 (Linux; U; Android 4.1.2; es-es; GT-I9100 Build/JZO54K) AppleWebKit/5 34.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" "-" https "en-US" "d ownloads.adblockplus.org" "-"', 489 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /adblockplusandro id-1.1.2.apk HTTP/1.1" 200 49152 "https://adblockplus.org/en/android-install" "M ozilla/5.0 (Linux; U; Android 4.1.2; es-es; GT-I9100 Build/JZO54K) AppleWebKit/5 34.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" "-" https "en-US" "d ownloads.adblockplus.org" "-"',
490 { 490 {
491 "ip": "1.2.3.4", 491 'ip': '1.2.3.4',
492 "country": "xy", 492 'country': 'xy',
493 "time": datetime(2013, 7, 31, 17, 33, 8), 493 'time': datetime(2013, 7, 31, 17, 33, 8),
494 "day": 31, 494 'day': 31,
495 "weekday": 2, 495 'weekday': 2,
496 "hour": 17, 496 'hour': 17,
497 "month": "201307", 497 'month': '201307',
498 "file": "adblockplusandroid-1.1.2.apk", 498 'file': 'adblockplusandroid-1.1.2.apk',
499 "query": "", 499 'query': '',
500 "size": 49152, 500 'size': 49152,
501 "referrer": "https://adblockplus.org/en/android-install", 501 'referrer': 'https://adblockplus.org/en/android-install',
502 "status": 200, 502 'status': 200,
503 "ua": "Mobile Safari", 503 'ua': 'Mobile Safari',
504 "uaversion": "4.0", 504 'uaversion': '4.0',
505 "fullua": "Mobile Safari 4.0", 505 'fullua': 'Mobile Safari 4.0',
506 "clientid": "-", 506 'clientid': '-',
507 "installType": "install", 507 'installType': 'install',
508 } 508 }
509 ), 509 ),
510 ( 510 (
511 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adbloc kplus/adblockplus-2.3.2.3712.xpi?update HTTP/1.1" 200 827261 "-" "Mozilla/5.0 (M acintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0" "-" https', 511 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adbloc kplus/adblockplus-2.3.2.3712.xpi?update HTTP/1.1" 200 827261 "-" "Mozilla/5.0 (M acintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0" "-" https',
512 { 512 {
513 "ip": "1.2.3.4", 513 'ip': '1.2.3.4',
514 "country": "xy", 514 'country': 'xy',
515 "time": datetime(2013, 7, 31, 17, 33, 8), 515 'time': datetime(2013, 7, 31, 17, 33, 8),
516 "day": 31, 516 'day': 31,
517 "weekday": 2, 517 'weekday': 2,
518 "hour": 17, 518 'hour': 17,
519 "month": "201307", 519 'month': '201307',
520 "file": "devbuilds/adblockplus/adblockplus-2.3.2.3712.xpi", 520 'file': 'devbuilds/adblockplus/adblockplus-2.3.2.3712.xpi',
521 "query": "update", 521 'query': 'update',
522 "size": 827261, 522 'size': 827261,
523 "referrer": "-", 523 'referrer': '-',
524 "status": 200, 524 'status': 200,
525 "ua": "Firefox", 525 'ua': 'Firefox',
526 "uaversion": "22.0", 526 'uaversion': '22.0',
527 "fullua": "Firefox 22.0", 527 'fullua': 'Firefox 22.0',
528 "clientid": None, 528 'clientid': None,
529 "installType": "update", 529 'installType': 'update',
530 } 530 }
531 ), 531 ),
532 ( 532 (
533 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /exceptionrules.t xt?addonName=adblockplus&addonVersion=2.3.2&application=firefox&applicationVersi on=22.0&platform=gecko&platformVersion=22.0&lastVersion=201307311503 HTTP/1.1" 2 00 14303 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/201001 01 Firefox/22.0" "-" https "en-US,en;q=0.5" "easylist-downloads.adblockplus.org" "-"', 533 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /exceptionrules.t xt?addonName=adblockplus&addonVersion=2.3.2&application=firefox&applicationVersi on=22.0&platform=gecko&platformVersion=22.0&lastVersion=201307311503 HTTP/1.1" 2 00 14303 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/201001 01 Firefox/22.0" "-" https "en-US,en;q=0.5" "easylist-downloads.adblockplus.org" "-"',
534 { 534 {
535 "ip": "1.2.3.4", 535 'ip': '1.2.3.4',
536 "country": "xy", 536 'country': 'xy',
537 "time": datetime(2013, 7, 31, 17, 33, 8), 537 'time': datetime(2013, 7, 31, 17, 33, 8),
538 "day": 31, 538 'day': 31,
539 "weekday": 2, 539 'weekday': 2,
540 "hour": 17, 540 'hour': 17,
541 "month": "201307", 541 'month': '201307',
542 "file": "exceptionrules.txt", 542 'file': 'exceptionrules.txt',
543 "query": "addonName=adblockplus&addonVersion=2.3.2&applicati on=firefox&applicationVersion=22.0&platform=gecko&platformVersion=22.0&lastVersi on=201307311503", 543 'query': 'addonName=adblockplus&addonVersion=2.3.2&applicati on=firefox&applicationVersion=22.0&platform=gecko&platformVersion=22.0&lastVersi on=201307311503',
544 "size": 14303, 544 'size': 14303,
545 "referrer": "-", 545 'referrer': '-',
546 "status": 200, 546 'status': 200,
547 "ua": "Firefox", 547 'ua': 'Firefox',
548 "uaversion": "22.0", 548 'uaversion': '22.0',
549 "fullua": "Firefox 22.0", 549 'fullua': 'Firefox 22.0',
550 "clientid": "-", 550 'clientid': '-',
551 "addonName": "adblockplus", 551 'addonName': 'adblockplus',
552 "addonVersion": "2.3.2", 552 'addonVersion': '2.3.2',
553 "fullAddon": "adblockplus 2.3.2", 553 'fullAddon': 'adblockplus 2.3.2',
554 "application": "firefox", 554 'application': 'firefox',
555 "applicationVersion": "22.0", 555 'applicationVersion': '22.0',
556 "fullApplication": "firefox 22.0", 556 'fullApplication': 'firefox 22.0',
557 "platform": "gecko", 557 'platform': 'gecko',
558 "platformVersion": "22.0", 558 'platformVersion': '22.0',
559 "fullPlatform": "gecko 22.0", 559 'fullPlatform': 'gecko 22.0',
560 "downloadInterval": "2 hour(s)", 560 'downloadInterval': '2 hour(s)',
561 "previousDownload": "same day", 561 'previousDownload': 'same day',
562 } 562 }
563 ), 563 ),
564 ( 564 (
565 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylist.txt?_=1 375446528229 HTTP/1.1" 200 326120 "-" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/ 537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https "nl-NL,n l;q=0.8,en-US;q=0.6,en;q=0.4" "easylist-downloads.adblockplus.org" "AdBlock/2.6. 2"', 565 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylist.txt?_=1 375446528229 HTTP/1.1" 200 326120 "-" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/ 537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https "nl-NL,n l;q=0.8,en-US;q=0.6,en;q=0.4" "easylist-downloads.adblockplus.org" "AdBlock/2.6. 2"',
566 { 566 {
567 "ip": "1.2.3.4", 567 'ip': '1.2.3.4',
568 "country": "xy", 568 'country': 'xy',
569 "time": datetime(2013, 7, 31, 17, 33, 8), 569 'time': datetime(2013, 7, 31, 17, 33, 8),
570 "day": 31, 570 'day': 31,
571 "weekday": 2, 571 'weekday': 2,
572 "hour": 17, 572 'hour': 17,
573 "month": "201307", 573 'month': '201307',
574 "file": "easylist.txt", 574 'file': 'easylist.txt',
575 "query": "_=1375446528229", 575 'query': '_=1375446528229',
576 "size": 326120, 576 'size': 326120,
577 "referrer": "-", 577 'referrer': '-',
578 "status": 200, 578 'status': 200,
579 "ua": "Chrome", 579 'ua': 'Chrome',
580 "uaversion": "28.0", 580 'uaversion': '28.0',
581 "fullua": "Chrome 28.0", 581 'fullua': 'Chrome 28.0',
582 "clientid": "AdBlock/2.6.2", 582 'clientid': 'AdBlock/2.6.2',
583 "addonName": "chromeadblock", 583 'addonName': 'chromeadblock',
584 "addonVersion": "2.6.2", 584 'addonVersion': '2.6.2',
585 "fullAddon": "chromeadblock 2.6.2", 585 'fullAddon': 'chromeadblock 2.6.2',
586 "application": "unknown", 586 'application': 'unknown',
587 "applicationVersion": "unknown", 587 'applicationVersion': 'unknown',
588 "fullApplication": "unknown unknown", 588 'fullApplication': 'unknown unknown',
589 "platform": "unknown", 589 'platform': 'unknown',
590 "platformVersion": "unknown", 590 'platformVersion': 'unknown',
591 "fullPlatform": "unknown unknown", 591 'fullPlatform': 'unknown unknown',
592 "downloadInterval": "unknown", 592 'downloadInterval': 'unknown',
593 "previousDownload": "unknown", 593 'previousDownload': 'unknown',
594 } 594 }
595 ), 595 ),
596 ( 596 (
597 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.tx t HTTP/1.1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"', 597 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.tx t HTTP/1.1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"',
598 { 598 {
599 "ip": "1.2.3.4", 599 'ip': '1.2.3.4',
600 "country": "xy", 600 'country': 'xy',
601 "time": datetime(2013, 7, 31, 17, 33, 8), 601 'time': datetime(2013, 7, 31, 17, 33, 8),
602 "day": 31, 602 'day': 31,
603 "weekday": 2, 603 'weekday': 2,
604 "hour": 17, 604 'hour': 17,
605 "month": "201307", 605 'month': '201307',
606 "file": "easylistitaly.txt", 606 'file': 'easylistitaly.txt',
607 "query": "", 607 'query': '',
608 "size": 85879, 608 'size': 85879,
609 "referrer": "-", 609 'referrer': '-',
610 "status": 200, 610 'status': 200,
611 "ua": "Other", 611 'ua': 'Other',
612 "uaversion": "", 612 'uaversion': '',
613 "fullua": "Other ", 613 'fullua': 'Other ',
614 "clientid": "-", 614 'clientid': '-',
615 "addonName": "unknown", 615 'addonName': 'unknown',
616 "addonVersion": "unknown", 616 'addonVersion': 'unknown',
617 "fullAddon": "unknown unknown", 617 'fullAddon': 'unknown unknown',
618 "application": "unknown", 618 'application': 'unknown',
619 "applicationVersion": "unknown", 619 'applicationVersion': 'unknown',
620 "fullApplication": "unknown unknown", 620 'fullApplication': 'unknown unknown',
621 "platform": "unknown", 621 'platform': 'unknown',
622 "platformVersion": "unknown", 622 'platformVersion': 'unknown',
623 "fullPlatform": "unknown unknown", 623 'fullPlatform': 'unknown unknown',
624 "downloadInterval": "unknown", 624 'downloadInterval': 'unknown',
625 "previousDownload": "unknown", 625 'previousDownload': 'unknown',
626 } 626 }
627 ), 627 ),
628 ( 628 (
629 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.tp l HTTP/1.1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"', 629 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.tp l HTTP/1.1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"',
630 { 630 {
631 "ip": "1.2.3.4", 631 'ip': '1.2.3.4',
632 "country": "xy", 632 'country': 'xy',
633 "time": datetime(2013, 7, 31, 17, 33, 8), 633 'time': datetime(2013, 7, 31, 17, 33, 8),
634 "day": 31, 634 'day': 31,
635 "weekday": 2, 635 'weekday': 2,
636 "hour": 17, 636 'hour': 17,
637 "month": "201307", 637 'month': '201307',
638 "file": "easylistitaly.tpl", 638 'file': 'easylistitaly.tpl',
639 "query": "", 639 'query': '',
640 "size": 85879, 640 'size': 85879,
641 "referrer": "-", 641 'referrer': '-',
642 "status": 200, 642 'status': 200,
643 "ua": "Other", 643 'ua': 'Other',
644 "uaversion": "", 644 'uaversion': '',
645 "fullua": "Other ", 645 'fullua': 'Other ',
646 "clientid": "-", 646 'clientid': '-',
647 } 647 }
648 ), 648 ),
649 ( 649 (
650 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /notification.jso n?addonName=adblockpluschrome&addonVersion=1.5.3&application=chrome&applicationV ersion=28.0.1500.72&platform=chromium&platformVersion=28.0.1500.72&lastVersion=2 01307292310 HTTP/1.1" 200 299 "-" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537. 36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https', 650 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /notification.jso n?addonName=adblockpluschrome&addonVersion=1.5.3&application=chrome&applicationV ersion=28.0.1500.72&platform=chromium&platformVersion=28.0.1500.72&lastVersion=2 01307292310 HTTP/1.1" 200 299 "-" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537. 36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https',
651 { 651 {
652 "ip": "1.2.3.4", 652 'ip': '1.2.3.4',
653 "country": "xy", 653 'country': 'xy',
654 "time": datetime(2013, 7, 31, 17, 33, 8), 654 'time': datetime(2013, 7, 31, 17, 33, 8),
655 "day": 31, 655 'day': 31,
656 "weekday": 2, 656 'weekday': 2,
657 "hour": 17, 657 'hour': 17,
658 "month": "201307", 658 'month': '201307',
659 "file": "notification.json", 659 'file': 'notification.json',
660 "query": "addonName=adblockpluschrome&addonVersion=1.5.3&app lication=chrome&applicationVersion=28.0.1500.72&platform=chromium&platformVersio n=28.0.1500.72&lastVersion=201307292310", 660 'query': 'addonName=adblockpluschrome&addonVersion=1.5.3&app lication=chrome&applicationVersion=28.0.1500.72&platform=chromium&platformVersio n=28.0.1500.72&lastVersion=201307292310',
661 "size": 299, 661 'size': 299,
662 "referrer": "-", 662 'referrer': '-',
663 "status": 200, 663 'status': 200,
664 "ua": "Chrome", 664 'ua': 'Chrome',
665 "uaversion": "28.0", 665 'uaversion': '28.0',
666 "fullua": "Chrome 28.0", 666 'fullua': 'Chrome 28.0',
667 "clientid": None, 667 'clientid': None,
668 "addonName": "adblockpluschrome", 668 'addonName': 'adblockpluschrome',
669 "addonVersion": "1.5.3", 669 'addonVersion': '1.5.3',
670 "fullAddon": "adblockpluschrome 1.5.3", 670 'fullAddon': 'adblockpluschrome 1.5.3',
671 "application": "chrome", 671 'application': 'chrome',
672 "applicationVersion": "28.0", 672 'applicationVersion': '28.0',
673 "fullApplication": "chrome 28.0", 673 'fullApplication': 'chrome 28.0',
674 "platform": "chromium", 674 'platform': 'chromium',
675 "platformVersion": "28.0", 675 'platformVersion': '28.0',
676 "fullPlatform": "chromium 28.0", 676 'fullPlatform': 'chromium 28.0',
677 "downloadInterval": "1 day(s)", 677 'downloadInterval': '1 day(s)',
678 "previousDownload": "2 day(s)", 678 'previousDownload': '2 day(s)',
679 "firstInDay": True, 679 'firstInDay': True,
680 } 680 }
681 ), 681 ),
682 ] 682 ]
683 for line, expected_record in tests: 683 for line, expected_record in tests:
684 self.assertEqual(logprocessor.parse_record(line, set(), FakeGeo(), F akeGeo()), expected_record, "Parsing log line '%s'" % line) 684 self.assertEqual(logprocessor.parse_record(line, set(), FakeGeo(), F akeGeo()), expected_record, "Parsing log line '%s'" % line)
685 685
686 def test_record_adding(self): 686 def test_record_adding(self):
687 tests = [ 687 tests = [
688 ( 688 (
689 {"size": 200}, 689 {'size': 200},
690 {}, 690 {},
691 (), 691 (),
692 {"hits": 1, "bandwidth": 200}, 692 {'hits': 1, 'bandwidth': 200},
693 ), 693 ),
694 ( 694 (
695 {"size": 200}, 695 {'size': 200},
696 {"hits": 12, "bandwidth": 30}, 696 {'hits': 12, 'bandwidth': 30},
697 (), 697 (),
698 {"hits": 13, "bandwidth": 230}, 698 {'hits': 13, 'bandwidth': 230},
699 ), 699 ),
700 ( 700 (
701 {"size": 200, "ua": "Foo"}, 701 {'size': 200, 'ua': 'Foo'},
702 {"hits": 12, "bandwidth": 30}, 702 {'hits': 12, 'bandwidth': 30},
703 (), 703 (),
704 {"hits": 13, "bandwidth": 230, "ua": {"Foo": {"hits": 1, "bandwi dth": 200}}}, 704 {'hits': 13, 'bandwidth': 230, 'ua': {'Foo': {'hits': 1, 'bandwi dth': 200}}},
705 ), 705 ),
706 ( 706 (
707 {"size": 200, "ua": "Foo"}, 707 {'size': 200, 'ua': 'Foo'},
708 {"hits": 12, "bandwidth": 30, "ua": {"Bar": {"hits": 21, "bandwi dth": 1200}}}, 708 {'hits': 12, 'bandwidth': 30, 'ua': {'Bar': {'hits': 21, 'bandwi dth': 1200}}},
709 (), 709 (),
710 {"hits": 13, "bandwidth": 230, "ua": {"Bar": {"hits": 21, "bandw idth": 1200}, "Foo": {"hits": 1, "bandwidth": 200}}}, 710 {'hits': 13, 'bandwidth': 230, 'ua': {'Bar': {'hits': 21, 'bandw idth': 1200}, 'Foo': {'hits': 1, 'bandwidth': 200}}},
711 ), 711 ),
712 ( 712 (
713 {"size": 200, "ua": "Foo", "bar": "xyz"}, 713 {'size': 200, 'ua': 'Foo', 'bar': 'xyz'},
714 {"hits": 12, "bandwidth": 30, "ua": {"Foo": {"hits": 21, "bandwi dth": 1200}}}, 714 {'hits': 12, 'bandwidth': 30, 'ua': {'Foo': {'hits': 21, 'bandwi dth': 1200}}},
715 (), 715 (),
716 {"hits": 13, "bandwidth": 230, "ua": {"Foo": {"hits": 22, "bandw idth": 1400}}}, 716 {'hits': 13, 'bandwidth': 230, 'ua': {'Foo': {'hits': 22, 'bandw idth': 1400}}},
717 ), 717 ),
718 ( 718 (
719 {"size": 200, "ua": "Foo", "addonName": "bar"}, 719 {'size': 200, 'ua': 'Foo', 'addonName': 'bar'},
720 {}, 720 {},
721 (), 721 (),
722 { 722 {
723 "hits": 1, "bandwidth": 200, 723 'hits': 1, 'bandwidth': 200,
724 "ua": {"Foo": {"hits": 1, "bandwidth": 200, "addonName": {"b ar": {"hits": 1, "bandwidth": 200}}}}, 724 'ua': {'Foo': {'hits': 1, 'bandwidth': 200, 'addonName': {'b ar': {'hits': 1, 'bandwidth': 200}}}},
725 "addonName": {"bar": {"hits": 1, "bandwidth": 200, "ua": {"F oo": {"hits": 1, "bandwidth": 200}}}} 725 'addonName': {'bar': {'hits': 1, 'bandwidth': 200, 'ua': {'F oo': {'hits': 1, 'bandwidth': 200}}}}
726 }, 726 },
727 ), 727 ),
728 ( 728 (
729 {"size": 200, "ua": "Foo", "addonName": "bar", "platform": "xyz" }, 729 {'size': 200, 'ua': 'Foo', 'addonName': 'bar', 'platform': 'xyz' },
730 {}, 730 {},
731 ("platform",), 731 ('platform',),
732 { 732 {
733 "hits": 1, "bandwidth": 200, 733 'hits': 1, 'bandwidth': 200,
734 "ua": {"Foo": {"hits": 1, "bandwidth": 200}}, 734 'ua': {'Foo': {'hits': 1, 'bandwidth': 200}},
735 "addonName": {"bar": {"hits": 1, "bandwidth": 200}} 735 'addonName': {'bar': {'hits': 1, 'bandwidth': 200}}
736 }, 736 },
737 ), 737 ),
738 ] 738 ]
739 for info, section, ignored_fields, expected_result in tests: 739 for info, section, ignored_fields, expected_result in tests:
740 logprocessor.add_record(info, section, ignored_fields) 740 logprocessor.add_record(info, section, ignored_fields)
741 self.assertEqual(section, expected_result) 741 self.assertEqual(section, expected_result)
742 742
743 if __name__ == '__main__': 743 if __name__ == '__main__':
744 unittest.main() 744 unittest.main()
OLDNEW
« no previous file with comments | « sitescripts/stats/test/common.py ('k') | sitescripts/subscriptions/bin/updateSubscriptionDownloads.py » ('j') | sitescripts/urlfixer/bin/topDomains.py » ('J')

Powered by Google App Engine
This is Rietveld