Left: | ||
Right: |
OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # coding: utf-8 | 2 # coding: utf-8 |
3 | 3 |
4 # This file is part of Adblock Plus <https://adblockplus.org/>, | 4 # This file is part of Adblock Plus <https://adblockplus.org/>, |
5 # Copyright (C) 2006-2015 Eyeo GmbH | 5 # Copyright (C) 2006-2015 Eyeo GmbH |
6 # | 6 # |
7 # Adblock Plus is free software: you can redistribute it and/or modify | 7 # Adblock Plus is free software: you can redistribute it and/or modify |
8 # it under the terms of the GNU General Public License version 3 as | 8 # it under the terms of the GNU General Public License version 3 as |
9 # published by the Free Software Foundation. | 9 # published by the Free Software Foundation. |
10 # | 10 # |
11 # Adblock Plus is distributed in the hope that it will be useful, | 11 # Adblock Plus is distributed in the hope that it will be useful, |
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
14 # GNU General Public License for more details. | 14 # GNU General Public License for more details. |
15 # | 15 # |
16 # You should have received a copy of the GNU General Public License | 16 # You should have received a copy of the GNU General Public License |
17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. | 17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. |
18 | 18 |
19 from collections import OrderedDict | |
20 from contextlib import closing | |
21 import json | |
19 import os | 22 import os |
20 import subprocess | 23 import subprocess |
24 import threading | |
25 import time | |
26 import re | |
21 import urllib2 | 27 import urllib2 |
22 | 28 |
23 from sitescripts.utils import get_config | 29 from sitescripts.utils import get_config |
24 | 30 |
25 def _update_abp2blocklist(): | 31 config = dict(get_config().items("content_blocker_lists")) |
32 | |
33 def update_abp2blocklist(): | |
26 with open(os.devnull, "w") as devnull: | 34 with open(os.devnull, "w") as devnull: |
27 config = get_config() | 35 abp2blocklist_path = config["abp2blocklist_path"] |
28 abp2blocklist_path = config.get("content_blocker_lists", | |
29 "abp2blocklist_path") | |
30 if os.path.isdir(abp2blocklist_path): | 36 if os.path.isdir(abp2blocklist_path): |
31 subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path), | 37 subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path), |
32 stdout=devnull) | 38 stdout=devnull) |
33 else: | 39 else: |
34 abp2blocklist_url = config.get("content_blocker_lists", | 40 subprocess.check_call(("hg", "clone", config["abp2blocklist_url"], |
35 "abp2blocklist_url") | |
36 subprocess.check_call(("hg", "clone", abp2blocklist_url, | |
37 abp2blocklist_path), stdout=devnull) | 41 abp2blocklist_path), stdout=devnull) |
38 subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path, | 42 subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path, |
39 stdout=devnull) | 43 stdout=devnull) |
40 | 44 |
41 def _download(url_key): | 45 def download_filter_list(url): |
42 url = get_config().get("content_blocker_lists", url_key) | 46 with closing(urllib2.urlopen(url)) as response: |
43 response = urllib2.urlopen(url) | 47 body = response.read() |
48 version = re.search(r"^(?:[^[!])|!\s*Version:\s*(.+)$", | |
Sebastian Noack
2015/12/08 15:08:38
Indeed, we can simply use re.search instead iterat
kzar
2015/12/08 15:37:43
Well it's true about the "! foo ! Version: foo" ma
Sebastian Noack
2015/12/08 15:45:51
Well, theoretically the filter list can be empty.
| |
49 body, re.MULTILINE).group(1) | |
50 return body, url, version | |
51 | |
52 def generate_metadata(filter_lists, expires): | |
53 metadata = OrderedDict(( | |
54 ("version", time.strftime("%Y%m%d%H%M", time.gmtime())), | |
55 ("expires", expires), | |
56 ("sources", []) | |
57 )) | |
58 for body, url, version in filter_lists: | |
59 metadata["sources"].append({"url": url, "version": version}) | |
60 return metadata | |
61 | |
62 def pipe_in(process, filter_lists): | |
44 try: | 63 try: |
45 return response.read() | 64 for body, _, _ in filter_lists: |
65 print >>process.stdin, body | |
46 finally: | 66 finally: |
47 response.close() | 67 process.stdin.close() |
48 | 68 |
49 def _convert_filter_list(sources, destination_path_key): | 69 def write_block_list(filter_lists, path, expires): |
50 config = get_config() | 70 block_list = generate_metadata(filter_lists, expires) |
51 destination_path = config.get("content_blocker_lists", destination_path_key) | 71 process = subprocess.Popen(("node", "abp2blocklist.js"), |
52 with open(destination_path, "wb") as destination_file: | 72 cwd=config["abp2blocklist_path"], |
53 abp2blocklist_path = config.get("content_blocker_lists", | 73 stdin=subprocess.PIPE, stdout=subprocess.PIPE) |
54 "abp2blocklist_path") | 74 threading.Thread(target=pipe_in, args=(process, filter_lists)).start() |
55 process = subprocess.Popen(("node", "abp2blocklist.js"), | 75 block_list["rules"] = json.load(process.stdout) |
56 cwd=abp2blocklist_path, stdin=subprocess.PIPE, | 76 if process.wait(): |
57 stdout=destination_file) | |
58 try: | |
59 for source in sources: | |
60 print >>process.stdin, source | |
61 finally: | |
62 process.stdin.close() | |
63 process.wait() | |
64 | |
65 if process.returncode: | |
66 raise Exception("abp2blocklist returned %s" % process.returncode) | 77 raise Exception("abp2blocklist returned %s" % process.returncode) |
67 | 78 |
79 with open(path, "wb") as destination_file: | |
80 json.dump(block_list, destination_file, indent=2, separators=(",", ": ")) | |
81 | |
68 if __name__ == "__main__": | 82 if __name__ == "__main__": |
69 _update_abp2blocklist() | 83 update_abp2blocklist() |
70 | 84 |
71 easylist = _download("easylist_url") | 85 easylist = download_filter_list(config["easylist_url"]) |
72 exceptionrules = _download("exceptionrules_url") | 86 exceptionrules = download_filter_list(config["exceptionrules_url"]) |
73 | 87 |
74 _convert_filter_list([easylist], "easylist_content_blocker_path") | 88 write_block_list([easylist], |
75 _convert_filter_list([easylist, exceptionrules], | 89 config["easylist_content_blocker_path"], |
76 "combined_content_blocker_path") | 90 config["easylist_content_blocker_expires"]) |
91 write_block_list([easylist, exceptionrules], | |
92 config["combined_content_blocker_path"], | |
93 config["combined_content_blocker_expires"]) | |
OLD | NEW |