OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 | 2 |
3 # This file is part of the Adblock Plus web scripts, | 3 # This file is part of the Adblock Plus web scripts, |
4 # Copyright (C) 2006-2016 Eyeo GmbH | 4 # Copyright (C) 2006-2016 Eyeo GmbH |
5 # | 5 # |
6 # Adblock Plus is free software: you can redistribute it and/or modify | 6 # Adblock Plus is free software: you can redistribute it and/or modify |
7 # it under the terms of the GNU General Public License version 3 as | 7 # it under the terms of the GNU General Public License version 3 as |
8 # published by the Free Software Foundation. | 8 # published by the Free Software Foundation. |
9 # | 9 # |
10 # Adblock Plus is distributed in the hope that it will be useful, | 10 # Adblock Plus is distributed in the hope that it will be useful, |
(...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
135 newlines = None | 135 newlines = None |
136 if re.match(r'^https?://', filename): | 136 if re.match(r'^https?://', filename): |
137 result.append('! *** Fetched from: %s ***' % filename) | 137 result.append('! *** Fetched from: %s ***' % filename) |
138 | 138 |
139 for i in range(3): | 139 for i in range(3): |
140 try: | 140 try: |
141 request = urllib2.urlopen(filename, None, timeout) | 141 request = urllib2.urlopen(filename, None, timeout) |
142 data = request.read() | 142 data = request.read() |
143 error = None | 143 error = None |
144 break | 144 break |
145 except urllib2.URLError, e: | 145 except urllib2.URLError as e: |
146 error = e | 146 error = e |
147 time.sleep(5) | 147 time.sleep(5) |
148 if error: | 148 if error: |
149 raise error | 149 raise error |
150 | 150 |
151 # We should really get the charset from the headers rather than
assuming | 151 # We should really get the charset from the headers rather than
assuming |
152 # that it is UTF-8. However, some of the Google Code mirrors are | 152 # that it is UTF-8. However, some of the Google Code mirrors are |
153 # misconfigured and will return ISO-8859-1 as charset instead of
UTF-8. | 153 # misconfigured and will return ISO-8859-1 as charset instead of
UTF-8. |
154 newlines = data.decode('utf-8').splitlines() | 154 newlines = data.decode('utf-8').splitlines() |
155 newlines = filter(lambda l: not re.search(r'^\s*!\s*(Redirect|Ho
mepage|Title|Version|Expires)\s*:', l, re.M | re.I), newlines) | 155 newlines = filter(lambda l: not re.search(r'^\s*!\s*(Redirect|Ho
mepage|Title|Version|Expires)\s*:', l, re.M | re.I), newlines) |
(...skipping 153 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
309 print '''Usage: %s source_name=source_dir ... [output_dir] | 309 print '''Usage: %s source_name=source_dir ... [output_dir] |
310 | 310 |
311 Options: | 311 Options: |
312 -h --help Print this message and exit | 312 -h --help Print this message and exit |
313 -t seconds --timeout=seconds Timeout when fetching remote subscriptions | 313 -t seconds --timeout=seconds Timeout when fetching remote subscriptions |
314 ''' % os.path.basename(sys.argv[0]) | 314 ''' % os.path.basename(sys.argv[0]) |
315 | 315 |
316 if __name__ == '__main__': | 316 if __name__ == '__main__': |
317 try: | 317 try: |
318 opts, args = getopt(sys.argv[1:], 'ht:', ['help', 'timeout=']) | 318 opts, args = getopt(sys.argv[1:], 'ht:', ['help', 'timeout=']) |
319 except GetoptError, e: | 319 except GetoptError as e: |
320 print str(e) | 320 print str(e) |
321 usage() | 321 usage() |
322 sys.exit(2) | 322 sys.exit(2) |
323 | 323 |
324 target_dir = 'subscriptions' | 324 target_dir = 'subscriptions' |
325 sources = {} | 325 sources = {} |
326 for arg in args: | 326 for arg in args: |
327 if '=' in arg: | 327 if '=' in arg: |
328 source_name, source_dir = arg.split('=', 1) | 328 source_name, source_dir = arg.split('=', 1) |
329 sources[source_name] = FileSource(source_dir) | 329 sources[source_name] = FileSource(source_dir) |
330 else: | 330 else: |
331 target_dir = arg | 331 target_dir = arg |
332 if not sources: | 332 if not sources: |
333 sources[''] = FileSource('.') | 333 sources[''] = FileSource('.') |
334 | 334 |
335 timeout = 30 | 335 timeout = 30 |
336 for option, value in opts: | 336 for option, value in opts: |
337 if option in ('-h', '--help'): | 337 if option in ('-h', '--help'): |
338 usage() | 338 usage() |
339 sys.exit() | 339 sys.exit() |
340 elif option in ('-t', '--timeout'): | 340 elif option in ('-t', '--timeout'): |
341 timeout = int(value) | 341 timeout = int(value) |
342 | 342 |
343 combine_subscriptions(sources, target_dir, timeout) | 343 combine_subscriptions(sources, target_dir, timeout) |
OLD | NEW |