sitescripts/stats/test/logprocessor.py - Issue 29934561: #1537 - Remove stats processing from sitescripts

Side by Side Diff: sitescripts/stats/test/logprocessor.py

Issue 29934561: #1537 - Remove stats processing from sitescripts (Closed) Base URL: https://hg.adblockplus.org/sitescripts

Patch Set: Created Nov. 2, 2018, 12:42 p.m.

Left:
Right:

Use n/p to move between diff chunks; N/P to move between comments.

Jump to:

View unified diff | Download patch

OLD	NEW
	(Empty)
1 # This file is part of the Adblock Plus web scripts,

2 # Copyright (C) 2006-present eyeo GmbH

3 #

4 # Adblock Plus is free software: you can redistribute it and/or modify

5 # it under the terms of the GNU General Public License version 3 as

6 # published by the Free Software Foundation.

7 #

8 # Adblock Plus is distributed in the hope that it will be useful,

9 # but WITHOUT ANY WARRANTY; without even the implied warranty of

10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

11 # GNU General Public License for more details.

12 #

13 # You should have received a copy of the GNU General Public License

14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.

15

16 import unittest

17 import sitescripts.stats.bin.logprocessor as logprocessor

18 from datetime import datetime

19

20

21 class Test(unittest.TestCase):

22 longMessage = True

23 maxDiff = None

24

25 def test_uaparsing(self):

26 tests = [

27 ('Firefox', '25.0', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25. 0) Gecko/20130730 Firefox/25.0'),

28 ('Firefox', '25.0', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:28. 0) Gecko/20130730 Firefox/25.0'),

29 ('Firefox', '25.0', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25. 0.1) Gecko/20130730 Firefox/25.0.1'),

30 ('Firefox Mobile', '15.0', 'Mozilla/5.0 (Maemo; Mobile; rv:15.0) Gec ko/20120829 Firefox/15.0 Fennec/15.0'),

31 ('Firefox Mobile', '14.0', 'Mozilla/5.0 (Android; Mobile; rv:14.0) G ecko/14.0 Firefox/14.0a2'),

32 ('Firefox Tablet', '26.0', 'Mozilla/5.0 (Android; Tablet; rv:26.0) G ecko/26.0 Firefox/26.0'),

33 ('Thunderbird', '24.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0 ) Gecko/20100101 Thunderbird/24.0a2 Lightning/2.6a2'),

34 ('SeaMonkey', '2.19', 'Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/2 0100101 Firefox/22.0 SeaMonkey/2.19'),

35 ('K-Meleon', '1.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; ru-RU; rv:1.8.1.24pre) Gecko/20100228 K-Meleon/1.5.4'),

36 ('Prism', '1.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv :1.9.2.20) Gecko/20110803 Prism/1.0b4'),

37 ('Gecko', '22.0', 'Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/20100 101 FooBar/1.0'),

38 ('Opera', '11.10', 'Opera/9.80 (Android; Opera Mini/15.0.1162/30.355 8; U; it) Presto/2.8.119 Version/11.10'),

39 ('Opera', '9.80', 'Opera/9.80 (Android; Opera Mini/15.0.1162/30.3558 ; U; it) Presto/2.8.119'),

40 ('Opera', '15.0', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/5 37.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36 OPR/15.0.1147.148'),

41 ('Chrome', '28.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/ 537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36'),

42 ('Chrome', '28.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/ 537.36 (KHTML, like Gecko) Iron/28.0.1550.0 Chrome/28.0.1550.0 Safari/537.36'),

43 ('Safari', '5.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16'),

44 ('Mobile Safari', '4.0', 'Mozilla/5.0 (Linux; U; Android 4.0.4; pt-b r; LG-E400 Build/IMM76L; CyanogenMod-9) AppleWebKit/534.30 (KHTML, like Gecko) V ersion/4.0 Mobile Safari/534.30'),

45 ('CoolNovo', '2.0.9', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK it/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36 CoolNovo/2.0.9. 11'),

46 ('WebKit', '', 'Mozilla/5.0 (Linux; U; Android 4.2.1; zh-CN; P7 Buil d/JRO03C) AppleWebKit/534.31 (KHTML, like Gecko) UCBrowser/9.2.0.308 U3/0.8.0 Mo bile Safari/534.31'),

47 ('MSIE', '10.0', 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1 ; WOW64; Trident/6.0)'),

48 ('MSIE', '7.0', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; Trident/7.0; .NET4.0E; .NET4.0C)'),

49 ('MSIE', '11.0', 'Mozilla/5.0 (Windows NT 6.3; ARM; Trident/7.0; Tou ch; rv:11.0) like Gecko'),

50 ('Trident', '7.0', 'Mozilla/5.0 (Windows NT 6.3; ARM; Trident/7.0; T ouch) like Gecko'),

51 ('Edge', '12', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KH TML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0'),

52 ('Edge', '12', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KH TML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136'),

53 ('Edge', '12', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.36 ( KHTML, like Gecko) Edge/12.10240'),

54 ('Edge', '12', 'Mozilla/5.0 (Windows NT 6.3; Win64, x64; Touch) Appl eWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0 ( Touch; Trident/7.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR 2.0.50727; .NET CLR 3.0.30729; HPNTDFJS; H9P; InfoPath'),

55 ('Android', '', 'AndroidDownloadManager'),

56 ('Android', '4.1', 'AndroidDownloadManager/4.1.1 (Linux; U; Android 4.1.1; A210 Build/JRO03H)'),

57 ('Android', '4.0', 'Dalvik/1.6.0 (Linux; U; Android 4.0.3; KFOT Buil d/IML74K)'),

58 ('Android', '4.3', 'Dalvik/1.6.0 (Linux; U; Android 4.3; Nexus 7 Bui ld/JSS15J)'),

59 ('Android', '', 'Apache-HttpClient/UNAVAILABLE (java 1.4)'),

60 ('ABP', '', 'Adblock Plus'),

61 ('Other', '', '-'),

62 ]

63 for expected_browser, expected_version, ua in tests:

64 self.assertEqual(logprocessor.parse_ua(ua), (expected_browser, expec ted_version), "Parsing user agent '%s'" % ua)

65

66 def test_ipprocessing(self):

67 country = None

68

69 class FakeGeo(object):

70 ip_checked = None

71

72 def country_code_by_addr(self, ip):

73 self.ip_checked = ip

74 return country

75

76 tests = [

77 ('1.2.3.4', 'xy', '1.2.3.4', 'v4', 'xy'),

78 ('::ffff:1.2.3.4', 'xy', '1.2.3.4', 'v4', 'xy'),

79 ('1.2.3.4', '--', '1.2.3.4', 'v4', 'unknown'),

80 ('1.2.3.4', '', '1.2.3.4', 'v4', 'unknown'),

81 ('::ffff:1.2.3.4', None, '1.2.3.4', 'v4', 'unknown'),

82 ('::1', 'xy', '::1', 'v6', 'xy'),

83 ('FE80:0000:0000:0000:0202:B3FF:FE1E:8329', None, 'FE80:0000:0000:00 00:0202:B3FF:FE1E:8329', 'v6', 'unknown'),

84 ]

85 for ip, country, expected_ip, expected_type, expected_country in tests:

86 fake_geo = FakeGeo()

87 fake_geov6 = FakeGeo()

88 self.assertEqual(logprocessor.process_ip(ip, fake_geo, fake_geov6), (expected_ip, expected_country), "Processing IP '%s'" % ip)

89 if expected_type == 'v4':

90 self.assertEqual(fake_geo.ip_checked, expected_ip, "GeoIP check for IP '%s'" % ip)

91 self.assertEqual(fake_geov6.ip_checked, None, "GeoIPv6 check for IP '%s'" % ip)

92 else:

93 self.assertEqual(fake_geo.ip_checked, None, "GeoIP check for IP '%s'" % ip)

94 self.assertEqual(fake_geov6.ip_checked, expected_ip, "GeoIPv6 ch eck for IP '%s'" % ip)

95

96 def test_timeparsing(self):

97 tests = [

98 ('31/Jul/2013:12:03:37', 0, 0, datetime(2013, 07, 31, 12, 03, 37), ' 201307'),

99 ('31/Jul/2013:12:03:37', 5, 0, datetime(2013, 07, 31, 7, 03, 37), '2 01307'),

100 ('31/Jul/2013:12:03:37', -5, 0, datetime(2013, 07, 31, 17, 03, 37), '201307'),

101 ('31/Jul/2013:12:03:37', 5, 30, datetime(2013, 07, 31, 6, 33, 37), ' 201307'),

102 ('31/Jul/2013:12:03:37', -5, 30, datetime(2013, 07, 31, 17, 33, 37), '201307'),

103 ]

104 for timestr, tz_hours, tz_minutes, expected_time, expected_month in test s:

105 self.assertEqual(logprocessor.parse_time(timestr, tz_hours, tz_minut es),

106 (expected_time, expected_month, expected_time.day, expected_time.weekday(), expected_time.hour),

107 "Parsing time string '%s %+03i%02i'" % (timestr, tz _hours, tz_minutes))

108

109 def test_pathparsing(self):

110 tests = [

111 ('/foo.txt', 'foo.txt', ''),

112 ('/foo.txt?', 'foo.txt', ''),

113 ('/foo.txt?asdf', 'foo.txt', 'asdf'),

114 ('http://example.com/foo.txt', 'foo.txt', ''),

115 ('/xyz/foo.txt?asdf', 'xyz/foo.txt', 'asdf'),

116 ('/xyz/foo+bar.txt?asdf', 'xyz/foo+bar.txt', 'asdf'),

117 ('/xyz/foo%2Bbar%2etxt?asdf', 'xyz/foo+bar.txt', 'asdf'),

118 ('/xyz/%D1%82%D0%B5%D1%81%D1%82.txt', u'xyz/\u0442\u0435\u0441\u0442 .txt', ''),

119 ('/xyz/foo%8Cbar.txt?asdf', 'xyz/foo%8Cbar.txt', 'asdf'),

120 ]

121 for path, expected_file, expected_query in tests:

122 self.assertEqual(logprocessor.parse_path(path), (expected_file, expe cted_query), "Parsing path '%s'" % path)

123

124 def test_downloaderdata(self):

125 tests = [

126 (

127 datetime(2013, 07, 31, 12, 03, 00),

128 'easylist.txt',

129 '',

130 '',

131 'unknown/unknown',

132 'unknown/unknown',

133 'unknown/unknown',

134 'unknown',

135 'unknown',

136 '',

137 ),

138 (

139 datetime(2013, 07, 31, 12, 03, 00),

140 'easylist.txt',

141 'addonName=adblockplus&addonVersion=2.3.1&application=firefox&ap plicationVersion=22.0a1&platform=gecko&platformVersion=23.0&lastVersion=0',

142 '-',

143 'adblockplus/2.3.1',

144 'firefox/22.0',

145 'gecko/23.0',

146 'unknown',

147 'unknown',

148 'firstDownload',

149 ),

150 (

151 datetime(2013, 07, 31, 12, 03, 00),

152 'easylist.txt',

153 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307311200-1/0',

154 '-',

155 'adblockplus/2.3.1',

156 'unknown/unknown',

157 'gecko/23.0',

158 '0 hour(s)',

159 'same day',

160 '',

161 ),

162 (

163 datetime(2013, 07, 31, 12, 03, 00),

164 'easylist.txt',

165 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307302200-1/3',

166 '-',

167 'adblockplus/2.3.1',

168 'unknown/unknown',

169 'gecko/23.0',

170 '14 hour(s)',

171 '1 day(s)',

172 'firstInDay',

173 ),

174 (

175 datetime(2013, 07, 31, 12, 03, 00),

176 'easylist.txt',

177 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307282200',

178 '-',

179 'adblockplus/2.3.1',

180 'unknown/unknown',

181 'gecko/23.0',

182 '2 day(s)',

183 '3 day(s)',

184 'firstInWeek firstInDay',

185 ),

186 (

187 datetime(2013, 8, 2, 12, 03, 00),

188 'easylist.txt',

189 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201307311200',

190 '-',

191 'adblockplus/2.3.1',

192 'unknown/unknown',

193 'gecko/23.0',

194 '2 day(s)',

195 '2 day(s)',

196 'firstInMonth firstInDay',

197 ),

198 (

199 datetime(2013, 07, 31, 12, 03, 00),

200 'easylist.txt',

201 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201306302200',

202 '-',

203 'adblockplus/2.3.1',

204 'unknown/unknown',

205 'gecko/23.0',

206 '1 month(s)',

207 '1 month(s)',

208 'firstInMonth firstInWeek firstInDay',

209 ),

210 (

211 datetime(2013, 07, 31, 12, 03, 00),

212 'easylist.txt',

213 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0&lastVersion=201305302200',

214 '-',

215 'adblockplus/2.3.1',

216 'unknown/unknown',

217 'gecko/23.0',

218 '2 month(s)',

219 '2 month(s)',

220 'firstInMonth firstInWeek firstInDay',

221 ),

222 (

223 datetime(2013, 07, 31, 12, 03, 00),

224 'easylist.txt',

225 'addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platfor mVersion=23.0.1&lastVersion=201206302200',

226 '-',

227 'adblockplus/2.3.1',

228 'unknown/unknown',

229 'gecko/23.0',

230 '1 year(s)',

231 '1 year(s)',

232 'firstInMonth firstInWeek firstInDay',

233 ),

234 (

235 datetime(2013, 07, 31, 12, 03, 00),

236 'easylist.txt',

237 'addonName=adblockedge&addonVersion=2.1.2&platform=gecko&platfor mVersion=23.0.1&lastVersion=201206302200',

238 '-',

239 'adblockedge/2.1.2',

240 'unknown/unknown',

241 'gecko/23.0',

242 '1 year(s)',

243 'unknown',

244 'firstInMonth firstInWeek firstInDay',

245 ),

246 (

247 datetime(2013, 07, 31, 12, 03, 00),

248 'easylist.txt',

249 '_=1375142394357',

250 'AdBlock/2.5.4',

251 'chromeadblock/2.5.4',

252 'unknown/unknown',

253 'unknown/unknown',

254 'unknown',

255 'unknown',

256 '',

257 ),

258 ]

259 for time, file, query, clientid, expected_addon, expected_application, e xpected_platform, expected_interval, expected_previous, expected_flags in tests:

260 info = {'time': time, 'file': file, 'query': query, 'clientid': clie ntid}

261 logprocessor.parse_downloader_query(info)

262 self.assertEqual('%s/%s' % (info['addonName'], info['addonVersion']) , expected_addon, "Add-on for query '%s'" % query)

263 self.assertEqual('%s/%s' % (info['application'], info['applicationVe rsion']), expected_application, "Application for query '%s'" % query)

264 self.assertEqual('%s/%s' % (info['platform'], info['platformVersion' ]), expected_platform, "Platform for query '%s'" % query)

265 self.assertEqual(info['downloadInterval'], expected_interval, "Downl oad interval for query '%s'" % query)

266 self.assertEqual(info['previousDownload'], expected_previous, "Previ ous download for query '%s'" % query)

267

268 flags = []

269 for flag in ('firstDownload', 'firstInMonth', 'firstInWeek', 'firstI nDay'):

270 if flag in info:

271 flags.append(flag)

272 self.assertEqual(' '.join(flags), expected_flags, "Flags for query ' %s'" % query)

273

274 def test_nameparsing(self):

275 tests = [

276 ('devbuilds/adblockplus/update.rdf', 'adblockplus'),

277 ('adblockpluschrome-experimental/updates.xml', 'adblockpluschrome-ex perimental'),

278 ('update.json', None),

279 ]

280 for file, expected_name in tests:

281 self.assertEqual(logprocessor.parse_addon_name(file), expected_name, "Getting add-on name for file '%s'" % file)

282

283 def test_geckoqueryparsing(self):

284 tests = [

285 (

286 'reqVersion=2&id={d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d}&version= 2.3.1.3707&maxAppVersion=25.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e- 13a3a9e97384}&appVersion=22.0&appOS=Darwin&appABI=x86_64-gcc3&locale=en-US&curre ntAppVersion=22.0&updateType=112',

287 '2.3.1.3707', 'firefox', '22.0',

288 ),

289 (

290 'reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74 &maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e973 84}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVers ion=25.0a1&updateType=97',

291 '1.0.4a.74', 'firefox', '25.0',

292 ),

293 (

294 'reqVersion=2&id={d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d}&version= 1.3a.20100925&maxAppVersion=2.1b1&status=userEnabled,incompatible&appID={92650c4 d-4b8e-4d2a-b7eb-24ecf4f6b63a}&appVersion=2.19&appOS=WINNT&appABI=x86-msvc&local e=en-US&currentAppVersion=2.19&updateType=112',

295 '1.3a.20100925', 'seamonkey', '2.19',

296 ),

297 ]

298 for query, expected_version, expected_application, expected_applicationv ersion in tests:

299 self.assertEqual(logprocessor.parse_gecko_query(query), (expected_ve rsion, expected_application, expected_applicationversion), "Parsing Gecko query '%s'" % query)

300

301 def test_chromequeryparsing(self):

302 tests = [

303 (

304 'os=win&arch=x86&nacl_arch=x86-64&prod=chromecrx&prodchannel=sta ble&prodversion=28.0.1500.72&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3 .977%26uc',

305 '1.5.3.977', 'chrome', '28.0',

306 ),

307 (

308 'x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3.977%26uc',

309 '1.5.3.977', 'unknown', 'unknown',

310 ),

311 (

312 'api=15&build=256&locale=ru_ru&device=LGE%20LG-P990',

313 'unknown', 'unknown', 'unknown',

314 ),

315 ]

316 for query, expected_version, expected_application, expected_applicationv ersion in tests:

317 self.assertEqual(logprocessor.parse_chrome_query(query), (expected_v ersion, expected_application, expected_applicationversion), "Parsing Chrome quer y '%s'" % query)

318

319 def test_updateflagparsing(self):

320 tests = [

321 ('update', 'update'),

322 ('', 'install'),

323 ('foo', 'install'),

324 ('update&foo', 'install'),

325 ]

326 for query, expected_result in tests:

327 self.assertEqual(logprocessor.parse_update_flag(query), expected_res ult, "Checking update flag for query '%s'" % query)

328

329 def test_recordparsing(self):

330 class FakeGeo(object):

331 def country_code_by_addr(self, ip):

332 return 'xy'

333

334 tests = [

335 (

336 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adbloc kpluschrome/updates.xml?os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&prodchan nel=stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v% 3D1.5.3.977%26uc HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 1 0_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36" "-" https" "en-US" "downloads.adblockplus.org" "-"',

337 {

338 'ip': '1.2.3.4',

339 'country': 'xy',

340 'time': datetime(2013, 7, 31, 17, 33, 8),

341 'day': 31,

342 'weekday': 2,

343 'hour': 17,

344 'month': '201307',

345 'file': 'devbuilds/adblockpluschrome/updates.xml',

346 'query': 'os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&pr odchannel=stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepk n%26v%3D1.5.3.977%26uc',

347 'size': 867,

348 'referrer': '-',

349 'status': 200,

350 'ua': 'Chrome',

351 'uaversion': '28.0',

352 'fullua': 'Chrome 28.0',

353 'clientid': '-',

354 'addonName': 'adblockpluschrome',

355 'addonVersion': '1.5.3.977',

356 'fullAddon': 'adblockpluschrome 1.5.3.977',

357 'application': 'chrome',

358 'applicationVersion': '28.0',

359 'fullApplication': 'chrome 28.0',

360 },

361 ),

362 (

363 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 404 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"',

364 None,

365 ),

366 (

367 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 301 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"',

368 {

369 'ip': '1.2.3.4',

370 'country': 'xy',

371 'time': datetime(2013, 7, 31, 17, 33, 8),

372 'day': 31,

373 'weekday': 2,

374 'hour': 17,

375 'month': '201307',

376 'file': 'devbuilds/abpcustomization/update.rdf',

377 'query': 'reqVersion=2&id=customization@adblockplus.org&vers ion=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b 0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&c urrentAppVersion=25.0a1&updateType=97',

378 'size': 867,

379 'referrer': '-',

380 'status': 301,

381 'ua': 'Firefox',

382 'uaversion': '25.0',

383 'fullua': 'Firefox 25.0',

384 'clientid': '-',

385 'addonName': 'abpcustomization',

386 'addonVersion': '1.0.4a.74',

387 'fullAddon': 'abpcustomization 1.0.4a.74',

388 'application': 'firefox',

389 'applicationVersion': '25.0',

390 'fullApplication': 'firefox 25.0',

391 },

392 ),

393 (

394 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 302 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"',

395 {

396 'ip': '1.2.3.4',

397 'country': 'xy',

398 'time': datetime(2013, 7, 31, 17, 33, 8),

399 'day': 31,

400 'weekday': 2,

401 'hour': 17,

402 'month': '201307',

403 'file': 'devbuilds/abpcustomization/update.rdf',

404 'query': 'reqVersion=2&id=customization@adblockplus.org&vers ion=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b 0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&c urrentAppVersion=25.0a1&updateType=97',

405 'size': 867,

406 'referrer': '-',

407 'status': 302,

408 'ua': 'Firefox',

409 'uaversion': '25.0',

410 'fullua': 'Firefox 25.0',

411 'clientid': '-',

412 'addonName': 'abpcustomization',

413 'addonVersion': '1.0.4a.74',

414 'fullAddon': 'abpcustomization 1.0.4a.74',

415 'application': 'firefox',

416 'applicationVersion': '25.0',

417 'fullApplication': 'firefox 25.0',

418 },

419 ),

420 (

421 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.unknown?reqVersion=2&id=customization@adblockplus.org&version= 1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-1 3a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&curre ntAppVersion=25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "down loads.adblockplus.org" "-"',

422 None,

423 ),

424 (

425 '1.2.3.4 corrupted',

426 None,

427 ),

428 (

429 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcus tomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0. 4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a 9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAp pVersion=25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1 ; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "download s.adblockplus.org" "-"',

430 {

431 'ip': '1.2.3.4',

432 'country': 'xy',

433 'time': datetime(2013, 7, 31, 17, 33, 8),

434 'day': 31,

435 'weekday': 2,

436 'hour': 17,

437 'month': '201307',

438 'file': 'devbuilds/abpcustomization/update.rdf',

439 'query': 'reqVersion=2&id=customization@adblockplus.org&vers ion=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b 0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&c urrentAppVersion=25.0a1&updateType=97',

440 'size': 867,

441 'referrer': '-',

442 'status': 200,

443 'ua': 'Firefox',

444 'uaversion': '25.0',

445 'fullua': 'Firefox 25.0',

446 'clientid': '-',

447 'addonName': 'abpcustomization',

448 'addonVersion': '1.0.4a.74',

449 'fullAddon': 'abpcustomization 1.0.4a.74',

450 'application': 'firefox',

451 'applicationVersion': '25.0',

452 'fullApplication': 'firefox 25.0',

453 },

454 ),

455 (

456 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adbloc kplusie/update.json?addonName=adblockplusie&addonVersion=2.0&application=msie64& applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion= 0 HTTP/1.1" 200 867 "-" "Adblock Plus" "-" https" "en-US" "downloads.adblockplus .org" "-"',

457 {

458 'ip': '1.2.3.4',

459 'country': 'xy',

460 'time': datetime(2013, 7, 31, 17, 33, 8),

461 'day': 31,

462 'weekday': 2,

463 'hour': 17,

464 'month': '201307',

465 'file': 'devbuilds/adblockplusie/update.json',

466 'query': 'addonName=adblockplusie&addonVersion=2.0&applicati on=msie64&applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&la stVersion=0',

467 'size': 867,

468 'referrer': '-',

469 'status': 200,

470 'ua': 'ABP',

471 'uaversion': '',

472 'fullua': 'ABP ',

473 'clientid': '-',

474 'addonName': 'adblockplusie',

475 'addonVersion': '2.0',

476 'fullAddon': 'adblockplusie 2.0',

477 'application': 'msie64',

478 'applicationVersion': '10.0',

479 'fullApplication': 'msie64 10.0',

480 'platform': 'libadblockplus',

481 'platformVersion': '1.0',

482 'fullPlatform': 'libadblockplus 1.0',

483 'downloadInterval': 'unknown',

484 'previousDownload': 'unknown',

485 'firstDownload': True,

486 },

487 ),

488 (

489 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /adblockplusandro id-1.1.2.apk HTTP/1.1" 200 49152 "https://adblockplus.org/en/android-install" "M ozilla/5.0 (Linux; U; Android 4.1.2; es-es; GT-I9100 Build/JZO54K) AppleWebKit/5 34.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" "-" https "en-US" "d ownloads.adblockplus.org" "-"',

490 {

491 'ip': '1.2.3.4',

492 'country': 'xy',

493 'time': datetime(2013, 7, 31, 17, 33, 8),

494 'day': 31,

495 'weekday': 2,

496 'hour': 17,

497 'month': '201307',

498 'file': 'adblockplusandroid-1.1.2.apk',

499 'query': '',

500 'size': 49152,

501 'referrer': 'https://adblockplus.org/en/android-install',

502 'status': 200,

503 'ua': 'Mobile Safari',

504 'uaversion': '4.0',

505 'fullua': 'Mobile Safari 4.0',

506 'clientid': '-',

507 'installType': 'install',

508 },

509 ),

510 (

511 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adbloc kplus/adblockplus-2.3.2.3712.xpi?update HTTP/1.1" 200 827261 "-" "Mozilla/5.0 (M acintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0" "-" https',

512 {

513 'ip': '1.2.3.4',

514 'country': 'xy',

515 'time': datetime(2013, 7, 31, 17, 33, 8),

516 'day': 31,

517 'weekday': 2,

518 'hour': 17,

519 'month': '201307',

520 'file': 'devbuilds/adblockplus/adblockplus-2.3.2.3712.xpi',

521 'query': 'update',

522 'size': 827261,

523 'referrer': '-',

524 'status': 200,

525 'ua': 'Firefox',

526 'uaversion': '22.0',

527 'fullua': 'Firefox 22.0',

528 'clientid': None,

529 'installType': 'update',

530 },

531 ),

532 (

533 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /exceptionrules.t xt?addonName=adblockplus&addonVersion=2.3.2&application=firefox&applicationVersi on=22.0&platform=gecko&platformVersion=22.0&lastVersion=201307311503 HTTP/1.1" 2 00 14303 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/201001 01 Firefox/22.0" "-" https "en-US,en;q=0.5" "easylist-downloads.adblockplus.org" "-"',

534 {

535 'ip': '1.2.3.4',

536 'country': 'xy',

537 'time': datetime(2013, 7, 31, 17, 33, 8),

538 'day': 31,

539 'weekday': 2,

540 'hour': 17,

541 'month': '201307',

542 'file': 'exceptionrules.txt',

543 'query': 'addonName=adblockplus&addonVersion=2.3.2&applicati on=firefox&applicationVersion=22.0&platform=gecko&platformVersion=22.0&lastVersi on=201307311503',

544 'size': 14303,

545 'referrer': '-',

546 'status': 200,

547 'ua': 'Firefox',

548 'uaversion': '22.0',

549 'fullua': 'Firefox 22.0',

550 'clientid': '-',

551 'addonName': 'adblockplus',

552 'addonVersion': '2.3.2',

553 'fullAddon': 'adblockplus 2.3.2',

554 'application': 'firefox',

555 'applicationVersion': '22.0',

556 'fullApplication': 'firefox 22.0',

557 'platform': 'gecko',

558 'platformVersion': '22.0',

559 'fullPlatform': 'gecko 22.0',

560 'downloadInterval': '2 hour(s)',

561 'previousDownload': 'same day',

562 },

563 ),

564 (

565 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylist.txt?_=1 375446528229 HTTP/1.1" 200 326120 "-" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/ 537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https "nl-NL,n l;q=0.8,en-US;q=0.6,en;q=0.4" "easylist-downloads.adblockplus.org" "AdBlock/2.6. 2"',

566 {

567 'ip': '1.2.3.4',

568 'country': 'xy',

569 'time': datetime(2013, 7, 31, 17, 33, 8),

570 'day': 31,

571 'weekday': 2,

572 'hour': 17,

573 'month': '201307',

574 'file': 'easylist.txt',

575 'query': '_=1375446528229',

576 'size': 326120,

577 'referrer': '-',

578 'status': 200,

579 'ua': 'Chrome',

580 'uaversion': '28.0',

581 'fullua': 'Chrome 28.0',

582 'clientid': 'AdBlock/2.6.2',

583 'addonName': 'chromeadblock',

584 'addonVersion': '2.6.2',

585 'fullAddon': 'chromeadblock 2.6.2',

586 'application': 'unknown',

587 'applicationVersion': 'unknown',

588 'fullApplication': 'unknown unknown',

589 'platform': 'unknown',

590 'platformVersion': 'unknown',

591 'fullPlatform': 'unknown unknown',

592 'downloadInterval': 'unknown',

593 'previousDownload': 'unknown',

594 },

595 ),

596 (

597 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.tx t HTTP/1.1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"',

598 {

599 'ip': '1.2.3.4',

600 'country': 'xy',

601 'time': datetime(2013, 7, 31, 17, 33, 8),

602 'day': 31,

603 'weekday': 2,

604 'hour': 17,

605 'month': '201307',

606 'file': 'easylistitaly.txt',

607 'query': '',

608 'size': 85879,

609 'referrer': '-',

610 'status': 200,

611 'ua': 'Other',

612 'uaversion': '',

613 'fullua': 'Other ',

614 'clientid': '-',

615 'addonName': 'unknown',

616 'addonVersion': 'unknown',

617 'fullAddon': 'unknown unknown',

618 'application': 'unknown',

619 'applicationVersion': 'unknown',

620 'fullApplication': 'unknown unknown',

621 'platform': 'unknown',

622 'platformVersion': 'unknown',

623 'fullPlatform': 'unknown unknown',

624 'downloadInterval': 'unknown',

625 'previousDownload': 'unknown',

626 },

627 ),

628 (

629 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.tp l HTTP/1.1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"',

630 {

631 'ip': '1.2.3.4',

632 'country': 'xy',

633 'time': datetime(2013, 7, 31, 17, 33, 8),

634 'day': 31,

635 'weekday': 2,

636 'hour': 17,

637 'month': '201307',

638 'file': 'easylistitaly.tpl',

639 'query': '',

640 'size': 85879,

641 'referrer': '-',

642 'status': 200,

643 'ua': 'Other',

644 'uaversion': '',

645 'fullua': 'Other ',

646 'clientid': '-',

647 },

648 ),

649 (

650 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /notification.jso n?addonName=adblockpluschrome&addonVersion=1.5.3&application=chrome&applicationV ersion=28.0.1500.72&platform=chromium&platformVersion=28.0.1500.72&lastVersion=2 01307292310 HTTP/1.1" 200 299 "-" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537. 36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https',

651 {

652 'ip': '1.2.3.4',

653 'country': 'xy',

654 'time': datetime(2013, 7, 31, 17, 33, 8),

655 'day': 31,

656 'weekday': 2,

657 'hour': 17,

658 'month': '201307',

659 'file': 'notification.json',

660 'query': 'addonName=adblockpluschrome&addonVersion=1.5.3&app lication=chrome&applicationVersion=28.0.1500.72&platform=chromium&platformVersio n=28.0.1500.72&lastVersion=201307292310',

661 'size': 299,

662 'referrer': '-',

663 'status': 200,

664 'ua': 'Chrome',

665 'uaversion': '28.0',

666 'fullua': 'Chrome 28.0',

667 'clientid': None,

668 'addonName': 'adblockpluschrome',

669 'addonVersion': '1.5.3',

670 'fullAddon': 'adblockpluschrome 1.5.3',

671 'application': 'chrome',

672 'applicationVersion': '28.0',

673 'fullApplication': 'chrome 28.0',

674 'platform': 'chromium',

675 'platformVersion': '28.0',

676 'fullPlatform': 'chromium 28.0',

677 'downloadInterval': '1 day(s)',

678 'previousDownload': '2 day(s)',

679 'firstInDay': True,

680 },

681 ),

682 ]

683 for line, expected_record in tests:

684 self.assertEqual(logprocessor.parse_record(line, set(), FakeGeo(), F akeGeo()), expected_record, "Parsing log line '%s'" % line)

685

686 def test_record_adding(self):

687 tests = [

688 (

689 {'size': 200},

690 {},

691 (),

692 {'hits': 1, 'bandwidth': 200},

693 ),

694 (

695 {'size': 200},

696 {'hits': 12, 'bandwidth': 30},

697 (),

698 {'hits': 13, 'bandwidth': 230},

699 ),

700 (

701 {'size': 200, 'ua': 'Foo'},

702 {'hits': 12, 'bandwidth': 30},

703 (),

704 {'hits': 13, 'bandwidth': 230, 'ua': {'Foo': {'hits': 1, 'bandwi dth': 200}}},

705 ),

706 (

707 {'size': 200, 'ua': 'Foo'},

708 {'hits': 12, 'bandwidth': 30, 'ua': {'Bar': {'hits': 21, 'bandwi dth': 1200}}},

709 (),

710 {'hits': 13, 'bandwidth': 230, 'ua': {'Bar': {'hits': 21, 'bandw idth': 1200}, 'Foo': {'hits': 1, 'bandwidth': 200}}},

711 ),

712 (

713 {'size': 200, 'ua': 'Foo', 'bar': 'xyz'},

714 {'hits': 12, 'bandwidth': 30, 'ua': {'Foo': {'hits': 21, 'bandwi dth': 1200}}},

715 (),

716 {'hits': 13, 'bandwidth': 230, 'ua': {'Foo': {'hits': 22, 'bandw idth': 1400}}},

717 ),

718 (

719 {'size': 200, 'ua': 'Foo', 'addonName': 'bar'},

720 {},

721 (),

722 {

723 'hits': 1, 'bandwidth': 200,

724 'ua': {'Foo': {'hits': 1, 'bandwidth': 200, 'addonName': {'b ar': {'hits': 1, 'bandwidth': 200}}}},

725 'addonName': {'bar': {'hits': 1, 'bandwidth': 200, 'ua': {'F oo': {'hits': 1, 'bandwidth': 200}}}},

726 },

727 ),

728 (

729 {'size': 200, 'ua': 'Foo', 'addonName': 'bar', 'platform': 'xyz' },

730 {},

731 ('platform',),

732 {

733 'hits': 1, 'bandwidth': 200,

734 'ua': {'Foo': {'hits': 1, 'bandwidth': 200}},

735 'addonName': {'bar': {'hits': 1, 'bandwidth': 200}},

736 },

737 ),

738 ]

739 for info, section, ignored_fields, expected_result in tests:

740 logprocessor.add_record(info, section, ignored_fields)

741 self.assertEqual(section, expected_result)

742

743

744 if __name__ == '__main__':

745 unittest.main()

OLD	NEW

« no previous file with comments | « sitescripts/stats/test/common.py ('k') | tox.ini » ('j') | tox.ini » ('J')