Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/stats/bin/logprocessor.py

Issue 5315116274810880: Issue 1811 - logprocessor crashes if unable to process one log file (Closed)
Patch Set: Created Jan. 14, 2015, 11:06 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH 4 # Copyright (C) 2006-2015 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
(...skipping 505 matching lines...) Expand 10 before | Expand all | Expand 10 after
516 516
517 lock.acquire() 517 lock.acquire()
518 try: 518 try:
519 save_stats(server_type, data, factor) 519 save_stats(server_type, data, factor)
520 finally: 520 finally:
521 lock.release() 521 lock.release()
522 return log_file, ignored 522 return log_file, ignored
523 except: 523 except:
524 print >>sys.stderr, "Unable to process log file '%s'" % log_file 524 print >>sys.stderr, "Unable to process log file '%s'" % log_file
525 traceback.print_exc() 525 traceback.print_exc()
526 return None, None, None, None 526 return None, None
527 527
528 def parse_sources(sources, factor=1, verbose=False): 528 def parse_sources(sources, factor=1, verbose=False):
529 pool = multiprocessing.Pool() 529 pool = multiprocessing.Pool()
530 lock = multiprocessing.Manager().Lock() 530 lock = multiprocessing.Manager().Lock()
531 callback = functools.partial(parse_source, factor, lock) 531 callback = functools.partial(parse_source, factor, lock)
532 try: 532 try:
533 for log_file, ignored in pool.imap_unordered(callback, sources, chunksize=1) : 533 for log_file, ignored in pool.imap_unordered(callback, sources, chunksize=1) :
534 if verbose and ignored: 534 if verbose and ignored:
535 print "Ignored files for %s" % log_file 535 print "Ignored files for %s" % log_file
536 print "============================================================" 536 print "============================================================"
(...skipping 10 matching lines...) Expand all
547 parser.add_argument("mirror_name", nargs="?", help="Name of the mirror server that the file belongs to") 547 parser.add_argument("mirror_name", nargs="?", help="Name of the mirror server that the file belongs to")
548 parser.add_argument("server_type", nargs="?", help="Server type like download, update or subscription") 548 parser.add_argument("server_type", nargs="?", help="Server type like download, update or subscription")
549 parser.add_argument("log_file", nargs="?", help="Log file path, can be a local file path, http:// or ssh:// URL") 549 parser.add_argument("log_file", nargs="?", help="Log file path, can be a local file path, http:// or ssh:// URL")
550 args = parser.parse_args() 550 args = parser.parse_args()
551 551
552 if args.mirror_name and args.server_type and args.log_file: 552 if args.mirror_name and args.server_type and args.log_file:
553 sources = [(args.mirror_name, args.server_type, args.log_file)] 553 sources = [(args.mirror_name, args.server_type, args.log_file)]
554 else: 554 else:
555 sources = get_stats_files() 555 sources = get_stats_files()
556 parse_sources(sources, args.factor, args.verbose) 556 parse_sources(sources, args.factor, args.verbose)
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld