| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 | 2 |
| 3 # This file is part of Adblock Plus <https://adblockplus.org/>, | 3 # This file is part of Adblock Plus <https://adblockplus.org/>, |
| 4 # Copyright (C) 2006-2016 Eyeo GmbH | 4 # Copyright (C) 2006-2016 Eyeo GmbH |
| 5 # | 5 # |
| 6 # Adblock Plus is free software: you can redistribute it and/or modify | 6 # Adblock Plus is free software: you can redistribute it and/or modify |
| 7 # it under the terms of the GNU General Public License version 3 as | 7 # it under the terms of the GNU General Public License version 3 as |
| 8 # published by the Free Software Foundation. | 8 # published by the Free Software Foundation. |
| 9 # | 9 # |
| 10 # Adblock Plus is distributed in the hope that it will be useful, | 10 # Adblock Plus is distributed in the hope that it will be useful, |
| 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 # GNU General Public License for more details. | 13 # GNU General Public License for more details. |
| 14 # | 14 # |
| 15 # You should have received a copy of the GNU General Public License | 15 # You should have received a copy of the GNU General Public License |
| 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. | 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. |
| 17 | 17 |
| 18 from collections import OrderedDict | 18 from collections import OrderedDict |
| 19 from contextlib import closing | 19 from contextlib import closing |
| 20 import json | 20 import json |
| 21 import os | 21 import os |
| 22 import subprocess | 22 import subprocess |
| 23 import threading | 23 import threading |
| 24 import time | 24 import time |
| 25 import re | 25 import re |
| 26 import urllib2 | 26 import urllib2 |
| 27 | 27 |
| 28 from sitescripts.utils import get_config | 28 from sitescripts.utils import get_config |
| 29 | 29 |
| 30 config = dict(get_config().items("content_blocker_lists")) | 30 config = dict(get_config().items('content_blocker_lists')) |
| 31 | 31 |
| 32 | 32 |
| 33 def update_abp2blocklist(): | 33 def update_abp2blocklist(): |
| 34 with open(os.devnull, "w") as devnull: | 34 with open(os.devnull, 'w') as devnull: |
| 35 abp2blocklist_path = config["abp2blocklist_path"] | 35 abp2blocklist_path = config['abp2blocklist_path'] |
| 36 if os.path.isdir(abp2blocklist_path): | 36 if os.path.isdir(abp2blocklist_path): |
| 37 subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path)
, | 37 subprocess.check_call(('hg', 'pull', '-u', '-R', abp2blocklist_path)
, |
| 38 stdout=devnull) | 38 stdout=devnull) |
| 39 else: | 39 else: |
| 40 subprocess.check_call(("hg", "clone", config["abp2blocklist_url"], | 40 subprocess.check_call(('hg', 'clone', config['abp2blocklist_url'], |
| 41 abp2blocklist_path), stdout=devnull) | 41 abp2blocklist_path), stdout=devnull) |
| 42 subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path, | 42 subprocess.check_call(('npm', 'install'), cwd=abp2blocklist_path, |
| 43 stdout=devnull) | 43 stdout=devnull) |
| 44 | 44 |
| 45 | 45 |
| 46 def download_filter_list(url): | 46 def download_filter_list(url): |
| 47 with closing(urllib2.urlopen(url)) as response: | 47 with closing(urllib2.urlopen(url)) as response: |
| 48 body = response.read() | 48 body = response.read() |
| 49 version = re.search(r"^(?:[^[!])|^!\s*Version:\s*(.+)$", | 49 version = re.search(r'^(?:[^[!])|^!\s*Version:\s*(.+)$', |
| 50 body, re.MULTILINE).group(1) | 50 body, re.MULTILINE).group(1) |
| 51 return body, url, version | 51 return body, url, version |
| 52 | 52 |
| 53 | 53 |
| 54 def generate_metadata(filter_lists, expires): | 54 def generate_metadata(filter_lists, expires): |
| 55 metadata = OrderedDict(( | 55 metadata = OrderedDict(( |
| 56 ("version", time.strftime("%Y%m%d%H%M", time.gmtime())), | 56 ('version', time.strftime('%Y%m%d%H%M', time.gmtime())), |
| 57 ("expires", expires), | 57 ('expires', expires), |
| 58 ("sources", []) | 58 ('sources', []) |
| 59 )) | 59 )) |
| 60 for body, url, version in filter_lists: | 60 for body, url, version in filter_lists: |
| 61 metadata["sources"].append({"url": url, "version": version}) | 61 metadata['sources'].append({'url': url, 'version': version}) |
| 62 return metadata | 62 return metadata |
| 63 | 63 |
| 64 | 64 |
| 65 def pipe_in(process, filter_lists): | 65 def pipe_in(process, filter_lists): |
| 66 try: | 66 try: |
| 67 for body, _, _ in filter_lists: | 67 for body, _, _ in filter_lists: |
| 68 print >>process.stdin, body | 68 print >>process.stdin, body |
| 69 finally: | 69 finally: |
| 70 process.stdin.close() | 70 process.stdin.close() |
| 71 | 71 |
| 72 | 72 |
| 73 def write_block_list(filter_lists, path, expires): | 73 def write_block_list(filter_lists, path, expires): |
| 74 block_list = generate_metadata(filter_lists, expires) | 74 block_list = generate_metadata(filter_lists, expires) |
| 75 process = subprocess.Popen(("node", "abp2blocklist.js"), | 75 process = subprocess.Popen(('node', 'abp2blocklist.js'), |
| 76 cwd=config["abp2blocklist_path"], | 76 cwd=config['abp2blocklist_path'], |
| 77 stdin=subprocess.PIPE, stdout=subprocess.PIPE) | 77 stdin=subprocess.PIPE, stdout=subprocess.PIPE) |
| 78 threading.Thread(target=pipe_in, args=(process, filter_lists)).start() | 78 threading.Thread(target=pipe_in, args=(process, filter_lists)).start() |
| 79 block_list["rules"] = json.load(process.stdout) | 79 block_list['rules'] = json.load(process.stdout) |
| 80 if process.wait(): | 80 if process.wait(): |
| 81 raise Exception("abp2blocklist returned %s" % process.returncode) | 81 raise Exception('abp2blocklist returned %s' % process.returncode) |
| 82 | 82 |
| 83 with open(path, "wb") as destination_file: | 83 with open(path, 'wb') as destination_file: |
| 84 json.dump(block_list, destination_file, indent=2, separators=(",", ": ")
) | 84 json.dump(block_list, destination_file, indent=2, separators=(',', ': ')
) |
| 85 | 85 |
| 86 if __name__ == "__main__": | 86 if __name__ == '__main__': |
| 87 update_abp2blocklist() | 87 update_abp2blocklist() |
| 88 | 88 |
| 89 easylist = download_filter_list(config["easylist_url"]) | 89 easylist = download_filter_list(config['easylist_url']) |
| 90 exceptionrules = download_filter_list(config["exceptionrules_url"]) | 90 exceptionrules = download_filter_list(config['exceptionrules_url']) |
| 91 | 91 |
| 92 write_block_list([easylist], | 92 write_block_list([easylist], |
| 93 config["easylist_content_blocker_path"], | 93 config['easylist_content_blocker_path'], |
| 94 config["easylist_content_blocker_expires"]) | 94 config['easylist_content_blocker_expires']) |
| 95 write_block_list([easylist, exceptionrules], | 95 write_block_list([easylist, exceptionrules], |
| 96 config["combined_content_blocker_path"], | 96 config['combined_content_blocker_path'], |
| 97 config["combined_content_blocker_expires"]) | 97 config['combined_content_blocker_expires']) |
| OLD | NEW |