Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/stats/test/logprocessor.py

Issue 4635035198029824: Issue 1390 - Record referrer for download stats (Closed)
Patch Set: Created Sept. 17, 2014, 8:47 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « sitescripts/stats/common.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2014 Eyeo GmbH 4 # Copyright (C) 2006-2014 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
(...skipping 322 matching lines...) Expand 10 before | Expand all | Expand 10 after
333 "ip": "1.2.3.4", 333 "ip": "1.2.3.4",
334 "country": "xy", 334 "country": "xy",
335 "time": datetime(2013, 7, 31, 17, 33, 8), 335 "time": datetime(2013, 7, 31, 17, 33, 8),
336 "day": 31, 336 "day": 31,
337 "weekday": 2, 337 "weekday": 2,
338 "hour": 17, 338 "hour": 17,
339 "month": "201307", 339 "month": "201307",
340 "file": "devbuilds/adblockpluschrome/updates.xml", 340 "file": "devbuilds/adblockpluschrome/updates.xml",
341 "query": "os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&prodchannel= stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1. 5.3.977%26uc", 341 "query": "os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&prodchannel= stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1. 5.3.977%26uc",
342 "size": 867, 342 "size": 867,
343 "referrer": "-",
343 "ua": "Chrome", 344 "ua": "Chrome",
344 "uaversion": "28.0", 345 "uaversion": "28.0",
345 "fullua": "Chrome 28.0", 346 "fullua": "Chrome 28.0",
346 "clientid": "-", 347 "clientid": "-",
347 "addonName": "adblockpluschrome", 348 "addonName": "adblockpluschrome",
348 "addonVersion": "1.5.3.977", 349 "addonVersion": "1.5.3.977",
349 "fullAddon": "adblockpluschrome 1.5.3.977", 350 "fullAddon": "adblockpluschrome 1.5.3.977",
350 "application": "chrome", 351 "application": "chrome",
351 "applicationVersion": "28.0", 352 "applicationVersion": "28.0",
352 "fullApplication": "chrome 28.0", 353 "fullApplication": "chrome 28.0",
(...skipping 17 matching lines...) Expand all
370 "ip": "1.2.3.4", 371 "ip": "1.2.3.4",
371 "country": "xy", 372 "country": "xy",
372 "time": datetime(2013, 7, 31, 17, 33, 8), 373 "time": datetime(2013, 7, 31, 17, 33, 8),
373 "day": 31, 374 "day": 31,
374 "weekday": 2, 375 "weekday": 2,
375 "hour": 17, 376 "hour": 17,
376 "month": "201307", 377 "month": "201307",
377 "file": "devbuilds/abpcustomization/update.rdf", 378 "file": "devbuilds/abpcustomization/update.rdf",
378 "query": "reqVersion=2&id=customization@adblockplus.org&version=1.0.4a .74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e 97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppV ersion=25.0a1&updateType=97", 379 "query": "reqVersion=2&id=customization@adblockplus.org&version=1.0.4a .74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e 97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppV ersion=25.0a1&updateType=97",
379 "size": 867, 380 "size": 867,
381 "referrer": "-",
380 "ua": "Firefox", 382 "ua": "Firefox",
381 "uaversion": "25.0", 383 "uaversion": "25.0",
382 "fullua": "Firefox 25.0", 384 "fullua": "Firefox 25.0",
383 "clientid": "-", 385 "clientid": "-",
384 "addonName": "abpcustomization", 386 "addonName": "abpcustomization",
385 "addonVersion": "1.0.4a.74", 387 "addonVersion": "1.0.4a.74",
386 "fullAddon": "abpcustomization 1.0.4a.74", 388 "fullAddon": "abpcustomization 1.0.4a.74",
387 "application": "firefox", 389 "application": "firefox",
388 "applicationVersion": "25.0", 390 "applicationVersion": "25.0",
389 "fullApplication": "firefox 25.0", 391 "fullApplication": "firefox 25.0",
390 } 392 }
391 ), 393 ),
392 ( 394 (
393 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adblockplusie/ update.json?addonName=adblockplusie&addonVersion=2.0&application=msie64&applicat ionVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion=0 HTTP/1 .1" 200 867 "-" "Adblock Plus" "-" https" "en-US" "downloads.adblockplus.org" "- "', 395 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adblockplusie/ update.json?addonName=adblockplusie&addonVersion=2.0&application=msie64&applicat ionVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion=0 HTTP/1 .1" 200 867 "-" "Adblock Plus" "-" https" "en-US" "downloads.adblockplus.org" "- "',
394 { 396 {
395 "ip": "1.2.3.4", 397 "ip": "1.2.3.4",
396 "country": "xy", 398 "country": "xy",
397 "time": datetime(2013, 7, 31, 17, 33, 8), 399 "time": datetime(2013, 7, 31, 17, 33, 8),
398 "day": 31, 400 "day": 31,
399 "weekday": 2, 401 "weekday": 2,
400 "hour": 17, 402 "hour": 17,
401 "month": "201307", 403 "month": "201307",
402 "file": "devbuilds/adblockplusie/update.json", 404 "file": "devbuilds/adblockplusie/update.json",
403 "query": "addonName=adblockplusie&addonVersion=2.0&application=msie64& applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion= 0", 405 "query": "addonName=adblockplusie&addonVersion=2.0&application=msie64& applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion= 0",
404 "size": 867, 406 "size": 867,
407 "referrer": "-",
405 "ua": "ABP", 408 "ua": "ABP",
406 "uaversion": "", 409 "uaversion": "",
407 "fullua": "ABP ", 410 "fullua": "ABP ",
408 "clientid": "-", 411 "clientid": "-",
409 "addonName": "adblockplusie", 412 "addonName": "adblockplusie",
410 "addonVersion": "2.0", 413 "addonVersion": "2.0",
411 "fullAddon": "adblockplusie 2.0", 414 "fullAddon": "adblockplusie 2.0",
412 "application": "msie64", 415 "application": "msie64",
413 "applicationVersion": "10.0", 416 "applicationVersion": "10.0",
414 "fullApplication": "msie64 10.0", 417 "fullApplication": "msie64 10.0",
(...skipping 11 matching lines...) Expand all
426 "ip": "1.2.3.4", 429 "ip": "1.2.3.4",
427 "country": "xy", 430 "country": "xy",
428 "time": datetime(2013, 7, 31, 17, 33, 8), 431 "time": datetime(2013, 7, 31, 17, 33, 8),
429 "day": 31, 432 "day": 31,
430 "weekday": 2, 433 "weekday": 2,
431 "hour": 17, 434 "hour": 17,
432 "month": "201307", 435 "month": "201307",
433 "file": "adblockplusandroid-1.1.2.apk", 436 "file": "adblockplusandroid-1.1.2.apk",
434 "query": "", 437 "query": "",
435 "size": 49152, 438 "size": 49152,
439 "referrer": "https://adblockplus.org/en/android-install",
436 "ua": "Mobile Safari", 440 "ua": "Mobile Safari",
437 "uaversion": "4.0", 441 "uaversion": "4.0",
438 "fullua": "Mobile Safari 4.0", 442 "fullua": "Mobile Safari 4.0",
439 "clientid": "-", 443 "clientid": "-",
440 "installType": "install", 444 "installType": "install",
441 } 445 }
442 ), 446 ),
443 ( 447 (
444 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adblockplus/ad blockplus-2.3.2.3712.xpi?update HTTP/1.1" 200 827261 "-" "Mozilla/5.0 (Macintosh ; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0" "-" https', 448 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adblockplus/ad blockplus-2.3.2.3712.xpi?update HTTP/1.1" 200 827261 "-" "Mozilla/5.0 (Macintosh ; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0" "-" https',
445 { 449 {
446 "ip": "1.2.3.4", 450 "ip": "1.2.3.4",
447 "country": "xy", 451 "country": "xy",
448 "time": datetime(2013, 7, 31, 17, 33, 8), 452 "time": datetime(2013, 7, 31, 17, 33, 8),
449 "day": 31, 453 "day": 31,
450 "weekday": 2, 454 "weekday": 2,
451 "hour": 17, 455 "hour": 17,
452 "month": "201307", 456 "month": "201307",
453 "file": "devbuilds/adblockplus/adblockplus-2.3.2.3712.xpi", 457 "file": "devbuilds/adblockplus/adblockplus-2.3.2.3712.xpi",
454 "query": "update", 458 "query": "update",
455 "size": 827261, 459 "size": 827261,
460 "referrer": "-",
456 "ua": "Firefox", 461 "ua": "Firefox",
457 "uaversion": "22.0", 462 "uaversion": "22.0",
458 "fullua": "Firefox 22.0", 463 "fullua": "Firefox 22.0",
459 "clientid": None, 464 "clientid": None,
460 "installType": "update", 465 "installType": "update",
461 } 466 }
462 ), 467 ),
463 ( 468 (
464 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /exceptionrules.txt?addon Name=adblockplus&addonVersion=2.3.2&application=firefox&applicationVersion=22.0& platform=gecko&platformVersion=22.0&lastVersion=201307311503 HTTP/1.1" 200 14303 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firef ox/22.0" "-" https "en-US,en;q=0.5" "easylist-downloads.adblockplus.org" "-"', 469 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /exceptionrules.txt?addon Name=adblockplus&addonVersion=2.3.2&application=firefox&applicationVersion=22.0& platform=gecko&platformVersion=22.0&lastVersion=201307311503 HTTP/1.1" 200 14303 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firef ox/22.0" "-" https "en-US,en;q=0.5" "easylist-downloads.adblockplus.org" "-"',
465 { 470 {
466 "ip": "1.2.3.4", 471 "ip": "1.2.3.4",
467 "country": "xy", 472 "country": "xy",
468 "time": datetime(2013, 7, 31, 17, 33, 8), 473 "time": datetime(2013, 7, 31, 17, 33, 8),
469 "day": 31, 474 "day": 31,
470 "weekday": 2, 475 "weekday": 2,
471 "hour": 17, 476 "hour": 17,
472 "month": "201307", 477 "month": "201307",
473 "file": "exceptionrules.txt", 478 "file": "exceptionrules.txt",
474 "query": "addonName=adblockplus&addonVersion=2.3.2&application=firefox &applicationVersion=22.0&platform=gecko&platformVersion=22.0&lastVersion=2013073 11503", 479 "query": "addonName=adblockplus&addonVersion=2.3.2&application=firefox &applicationVersion=22.0&platform=gecko&platformVersion=22.0&lastVersion=2013073 11503",
475 "size": 14303, 480 "size": 14303,
481 "referrer": "-",
476 "ua": "Firefox", 482 "ua": "Firefox",
477 "uaversion": "22.0", 483 "uaversion": "22.0",
478 "fullua": "Firefox 22.0", 484 "fullua": "Firefox 22.0",
479 "clientid": "-", 485 "clientid": "-",
480 "addonName": "adblockplus", 486 "addonName": "adblockplus",
481 "addonVersion": "2.3.2", 487 "addonVersion": "2.3.2",
482 "fullAddon": "adblockplus 2.3.2", 488 "fullAddon": "adblockplus 2.3.2",
483 "application": "firefox", 489 "application": "firefox",
484 "applicationVersion": "22.0", 490 "applicationVersion": "22.0",
485 "fullApplication": "firefox 22.0", 491 "fullApplication": "firefox 22.0",
(...skipping 10 matching lines...) Expand all
496 "ip": "1.2.3.4", 502 "ip": "1.2.3.4",
497 "country": "xy", 503 "country": "xy",
498 "time": datetime(2013, 7, 31, 17, 33, 8), 504 "time": datetime(2013, 7, 31, 17, 33, 8),
499 "day": 31, 505 "day": 31,
500 "weekday": 2, 506 "weekday": 2,
501 "hour": 17, 507 "hour": 17,
502 "month": "201307", 508 "month": "201307",
503 "file": "easylist.txt", 509 "file": "easylist.txt",
504 "query": "_=1375446528229", 510 "query": "_=1375446528229",
505 "size": 326120, 511 "size": 326120,
512 "referrer": "-",
506 "ua": "Chrome", 513 "ua": "Chrome",
507 "uaversion": "28.0", 514 "uaversion": "28.0",
508 "fullua": "Chrome 28.0", 515 "fullua": "Chrome 28.0",
509 "clientid": "AdBlock/2.6.2", 516 "clientid": "AdBlock/2.6.2",
510 "addonName": "chromeadblock", 517 "addonName": "chromeadblock",
511 "addonVersion": "2.6.2", 518 "addonVersion": "2.6.2",
512 "fullAddon": "chromeadblock 2.6.2", 519 "fullAddon": "chromeadblock 2.6.2",
513 "application": "unknown", 520 "application": "unknown",
514 "applicationVersion": "unknown", 521 "applicationVersion": "unknown",
515 "fullApplication": "unknown unknown", 522 "fullApplication": "unknown unknown",
(...skipping 10 matching lines...) Expand all
526 "ip": "1.2.3.4", 533 "ip": "1.2.3.4",
527 "country": "xy", 534 "country": "xy",
528 "time": datetime(2013, 7, 31, 17, 33, 8), 535 "time": datetime(2013, 7, 31, 17, 33, 8),
529 "day": 31, 536 "day": 31,
530 "weekday": 2, 537 "weekday": 2,
531 "hour": 17, 538 "hour": 17,
532 "month": "201307", 539 "month": "201307",
533 "file": "easylistitaly.txt", 540 "file": "easylistitaly.txt",
534 "query": "", 541 "query": "",
535 "size": 85879, 542 "size": 85879,
543 "referrer": "-",
536 "ua": "Other", 544 "ua": "Other",
537 "uaversion": "", 545 "uaversion": "",
538 "fullua": "Other ", 546 "fullua": "Other ",
539 "clientid": "-", 547 "clientid": "-",
540 "addonName": "unknown", 548 "addonName": "unknown",
541 "addonVersion": "unknown", 549 "addonVersion": "unknown",
542 "fullAddon": "unknown unknown", 550 "fullAddon": "unknown unknown",
543 "application": "unknown", 551 "application": "unknown",
544 "applicationVersion": "unknown", 552 "applicationVersion": "unknown",
545 "fullApplication": "unknown unknown", 553 "fullApplication": "unknown unknown",
(...skipping 10 matching lines...) Expand all
556 "ip": "1.2.3.4", 564 "ip": "1.2.3.4",
557 "country": "xy", 565 "country": "xy",
558 "time": datetime(2013, 7, 31, 17, 33, 8), 566 "time": datetime(2013, 7, 31, 17, 33, 8),
559 "day": 31, 567 "day": 31,
560 "weekday": 2, 568 "weekday": 2,
561 "hour": 17, 569 "hour": 17,
562 "month": "201307", 570 "month": "201307",
563 "file": "easylistitaly.tpl", 571 "file": "easylistitaly.tpl",
564 "query": "", 572 "query": "",
565 "size": 85879, 573 "size": 85879,
574 "referrer": "-",
566 "ua": "Other", 575 "ua": "Other",
567 "uaversion": "", 576 "uaversion": "",
568 "fullua": "Other ", 577 "fullua": "Other ",
569 "clientid": "-", 578 "clientid": "-",
570 } 579 }
571 ), 580 ),
572 ( 581 (
573 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /notification.json?addonN ame=adblockpluschrome&addonVersion=1.5.3&application=chrome&applicationVersion=2 8.0.1500.72&platform=chromium&platformVersion=28.0.1500.72&lastVersion=201307292 310 HTTP/1.1" 200 299 "-" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTM L, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https', 582 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /notification.json?addonN ame=adblockpluschrome&addonVersion=1.5.3&application=chrome&applicationVersion=2 8.0.1500.72&platform=chromium&platformVersion=28.0.1500.72&lastVersion=201307292 310 HTTP/1.1" 200 299 "-" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTM L, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https',
574 { 583 {
575 "ip": "1.2.3.4", 584 "ip": "1.2.3.4",
576 "country": "xy", 585 "country": "xy",
577 "time": datetime(2013, 7, 31, 17, 33, 8), 586 "time": datetime(2013, 7, 31, 17, 33, 8),
578 "day": 31, 587 "day": 31,
579 "weekday": 2, 588 "weekday": 2,
580 "hour": 17, 589 "hour": 17,
581 "month": "201307", 590 "month": "201307",
582 "file": "notification.json", 591 "file": "notification.json",
583 "query": "addonName=adblockpluschrome&addonVersion=1.5.3&application=c hrome&applicationVersion=28.0.1500.72&platform=chromium&platformVersion=28.0.150 0.72&lastVersion=201307292310", 592 "query": "addonName=adblockpluschrome&addonVersion=1.5.3&application=c hrome&applicationVersion=28.0.1500.72&platform=chromium&platformVersion=28.0.150 0.72&lastVersion=201307292310",
584 "size": 299, 593 "size": 299,
594 "referrer": "-",
585 "ua": "Chrome", 595 "ua": "Chrome",
586 "uaversion": "28.0", 596 "uaversion": "28.0",
587 "fullua": "Chrome 28.0", 597 "fullua": "Chrome 28.0",
588 "clientid": None, 598 "clientid": None,
589 "addonName": "adblockpluschrome", 599 "addonName": "adblockpluschrome",
590 "addonVersion": "1.5.3", 600 "addonVersion": "1.5.3",
591 "fullAddon": "adblockpluschrome 1.5.3", 601 "fullAddon": "adblockpluschrome 1.5.3",
592 "application": "chrome", 602 "application": "chrome",
593 "applicationVersion": "28.0", 603 "applicationVersion": "28.0",
594 "fullApplication": "chrome 28.0", 604 "fullApplication": "chrome 28.0",
595 "platform": "chromium", 605 "platform": "chromium",
596 "platformVersion": "28.0", 606 "platformVersion": "28.0",
597 "fullPlatform": "chromium 28.0", 607 "fullPlatform": "chromium 28.0",
598 "downloadInterval": "1 day(s)", 608 "downloadInterval": "1 day(s)",
599 "previousDownload": "2 day(s)", 609 "previousDownload": "2 day(s)",
600 "firstInDay": True, 610 "firstInDay": True,
601 } 611 }
602 ), 612 ),
603 ] 613 ]
604 for line, expected_record in tests: 614 for line, expected_record in tests:
605 self.assertEqual(logprocessor.parse_record(line, set(), FakeGeo(), FakeGeo ()), expected_record, "Parsing log line '%s'" % line) 615 result = logprocessor.parse_record(line, set(), FakeGeo(), FakeGeo())
616 # assertEqual() treats identical Unicode and non-Unicode strings as
617 # different, make sure that all our strings are non-Unicode.
Sebastian Noack 2014/09/18 15:50:04 Why don't you just use unicode literals above?
Wladimir Palant 2014/09/18 18:42:50 Because I don't know which one will use Unicode -
Sebastian Noack 2014/09/19 09:48:42 According to my tests it doesn't. All strings in t
Wladimir Palant 2014/09/22 10:39:05 For some reason I cannot reproduce it any longer,
618 if result:
619 for key, value in result.iteritems():
620 if isinstance(value, unicode):
621 result[key] = str(value)
622 self.assertEqual(result, expected_record, "Parsing log line '%s'" % line)
606 623
607 def test_record_adding(self): 624 def test_record_adding(self):
608 tests = [ 625 tests = [
609 ( 626 (
610 {"size": 200}, 627 {"size": 200},
611 {}, 628 {},
612 (), 629 (),
613 {"hits": 1, "bandwidth": 200}, 630 {"hits": 1, "bandwidth": 200},
614 ), 631 ),
615 ( 632 (
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after
656 "addonName": {"bar": {"hits": 1, "bandwidth": 200}} 673 "addonName": {"bar": {"hits": 1, "bandwidth": 200}}
657 }, 674 },
658 ), 675 ),
659 ] 676 ]
660 for info, section, ignored_fields, expected_result in tests: 677 for info, section, ignored_fields, expected_result in tests:
661 logprocessor.add_record(info, section, ignored_fields) 678 logprocessor.add_record(info, section, ignored_fields)
662 self.assertEqual(section, expected_result) 679 self.assertEqual(section, expected_result)
663 680
664 if __name__ == '__main__': 681 if __name__ == '__main__':
665 unittest.main() 682 unittest.main()
OLDNEW
« no previous file with comments | « sitescripts/stats/common.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld