OLD | NEW |
1 # coding: utf-8 | 1 # coding: utf-8 |
2 | 2 |
3 # This file is part of the Adblock Plus web scripts, | 3 # This file is part of the Adblock Plus web scripts, |
4 # Copyright (C) 2006-2015 Eyeo GmbH | 4 # Copyright (C) 2006-2015 Eyeo GmbH |
5 # | 5 # |
6 # Adblock Plus is free software: you can redistribute it and/or modify | 6 # Adblock Plus is free software: you can redistribute it and/or modify |
7 # it under the terms of the GNU General Public License version 3 as | 7 # it under the terms of the GNU General Public License version 3 as |
8 # published by the Free Software Foundation. | 8 # published by the Free Software Foundation. |
9 # | 9 # |
10 # Adblock Plus is distributed in the hope that it will be useful, | 10 # Adblock Plus is distributed in the hope that it will be useful, |
(...skipping 505 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
516 | 516 |
517 lock.acquire() | 517 lock.acquire() |
518 try: | 518 try: |
519 save_stats(server_type, data, factor) | 519 save_stats(server_type, data, factor) |
520 finally: | 520 finally: |
521 lock.release() | 521 lock.release() |
522 return log_file, ignored | 522 return log_file, ignored |
523 except: | 523 except: |
524 print >>sys.stderr, "Unable to process log file '%s'" % log_file | 524 print >>sys.stderr, "Unable to process log file '%s'" % log_file |
525 traceback.print_exc() | 525 traceback.print_exc() |
526 return None, None, None, None | 526 return None, None |
527 | 527 |
528 def parse_sources(sources, factor=1, verbose=False): | 528 def parse_sources(sources, factor=1, verbose=False): |
529 pool = multiprocessing.Pool() | 529 pool = multiprocessing.Pool() |
530 lock = multiprocessing.Manager().Lock() | 530 lock = multiprocessing.Manager().Lock() |
531 callback = functools.partial(parse_source, factor, lock) | 531 callback = functools.partial(parse_source, factor, lock) |
532 try: | 532 try: |
533 for log_file, ignored in pool.imap_unordered(callback, sources, chunksize=1)
: | 533 for log_file, ignored in pool.imap_unordered(callback, sources, chunksize=1)
: |
534 if verbose and ignored: | 534 if verbose and ignored: |
535 print "Ignored files for %s" % log_file | 535 print "Ignored files for %s" % log_file |
536 print "============================================================" | 536 print "============================================================" |
(...skipping 10 matching lines...) Expand all Loading... |
547 parser.add_argument("mirror_name", nargs="?", help="Name of the mirror server
that the file belongs to") | 547 parser.add_argument("mirror_name", nargs="?", help="Name of the mirror server
that the file belongs to") |
548 parser.add_argument("server_type", nargs="?", help="Server type like download,
update or subscription") | 548 parser.add_argument("server_type", nargs="?", help="Server type like download,
update or subscription") |
549 parser.add_argument("log_file", nargs="?", help="Log file path, can be a local
file path, http:// or ssh:// URL") | 549 parser.add_argument("log_file", nargs="?", help="Log file path, can be a local
file path, http:// or ssh:// URL") |
550 args = parser.parse_args() | 550 args = parser.parse_args() |
551 | 551 |
552 if args.mirror_name and args.server_type and args.log_file: | 552 if args.mirror_name and args.server_type and args.log_file: |
553 sources = [(args.mirror_name, args.server_type, args.log_file)] | 553 sources = [(args.mirror_name, args.server_type, args.log_file)] |
554 else: | 554 else: |
555 sources = get_stats_files() | 555 sources = get_stats_files() |
556 parse_sources(sources, args.factor, args.verbose) | 556 parse_sources(sources, args.factor, args.verbose) |
OLD | NEW |