Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: sitescripts/content_blocker_lists/bin/generate_lists.py

Issue 29329537: Issue 3168 - Add a script for generating content blocker lists (Closed)
Left Patch Set: Keep the lists in memory, don't print anything to stdout, and more Created Nov. 13, 2015, 9:36 a.m.
Right Patch Set: Remove retcode variable, move return code check out of the with block Created Nov. 20, 2015, 7 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « sitescripts/content_blocker_lists/bin/__init__.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # coding: utf-8 2 # coding: utf-8
3 3
4 # This file is part of Adblock Plus <https://adblockplus.org/>, 4 # This file is part of Adblock Plus <https://adblockplus.org/>,
5 # Copyright (C) 2006-2015 Eyeo GmbH 5 # Copyright (C) 2006-2015 Eyeo GmbH
6 # 6 #
7 # Adblock Plus is free software: you can redistribute it and/or modify 7 # Adblock Plus is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as 8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation. 9 # published by the Free Software Foundation.
10 # 10 #
11 # Adblock Plus is distributed in the hope that it will be useful, 11 # Adblock Plus is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details. 14 # GNU General Public License for more details.
15 # 15 #
16 # You should have received a copy of the GNU General Public License 16 # You should have received a copy of the GNU General Public License
17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
18 18
19 import os 19 import os
20 import subprocess 20 import subprocess
21 import urllib2 21 import urllib2
22 22
23 from sitescripts.utils import get_config 23 from sitescripts.utils import get_config
24 24
25 def _get_config_value(key):
26 return get_config().get("content_blocker_lists", key)
Sebastian Noack 2015/11/18 20:10:37 We probably shouldn't call get_config every for ev
Felix Dahlke 2015/11/19 11:15:20 After implementing your suggestion below, I could
27
28 def _update_abp2blocklist(): 25 def _update_abp2blocklist():
29 with open(os.devnull, "w") as devnull: 26 with open(os.devnull, "w") as devnull:
30 abp2blocklist_path = _get_config_value("abp2blocklist_path") 27 config = get_config()
28 abp2blocklist_path = config.get("content_blocker_lists",
29 "abp2blocklist_path")
31 if os.path.isdir(abp2blocklist_path): 30 if os.path.isdir(abp2blocklist_path):
32 subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path), 31 subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path),
33 stdout=devnull) 32 stdout=devnull)
34 else: 33 else:
35 abp2blocklist_url = _get_config_value("abp2blocklist_url") 34 abp2blocklist_url = config.get("content_blocker_lists",
35 "abp2blocklist_url")
36 subprocess.check_call(("hg", "clone", abp2blocklist_url, 36 subprocess.check_call(("hg", "clone", abp2blocklist_url,
37 abp2blocklist_path), stdout=devnull) 37 abp2blocklist_path), stdout=devnull)
38 subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path, 38 subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path,
39 stdout=devnull) 39 stdout=devnull)
40 40
41 def _download(url): 41 def _download(url_key):
42 url = get_config().get("content_blocker_lists", url_key)
42 response = urllib2.urlopen(url) 43 response = urllib2.urlopen(url)
43 try: 44 try:
44 return response.read() 45 return response.read()
45 finally: 46 finally:
46 response.close() 47 response.close()
47 48
48 def _convert_filter_list(source, destination_path): 49 def _convert_filter_list(sources, destination_path_key):
50 config = get_config()
51 destination_path = config.get("content_blocker_lists", destination_path_key)
49 with open(destination_path, "wb") as destination_file: 52 with open(destination_path, "wb") as destination_file:
50 abp2blocklist_path = _get_config_value("abp2blocklist_path") 53 abp2blocklist_path = config.get("content_blocker_lists",
54 "abp2blocklist_path")
51 process = subprocess.Popen(("node", "abp2blocklist.js"), 55 process = subprocess.Popen(("node", "abp2blocklist.js"),
52 cwd=abp2blocklist_path, stdin=subprocess.PIPE, 56 cwd=abp2blocklist_path, stdin=subprocess.PIPE,
53 stdout=destination_file) 57 stdout=destination_file)
54 process.communicate(input=source) 58 try:
55 if process.returncode: 59 for source in sources:
56 raise Exception("abp2blocklist returned %s" % process.returncode) 60 print >>process.stdin, source
61 finally:
62 process.stdin.close()
63 process.wait()
64
65 if process.returncode:
66 raise Exception("abp2blocklist returned %s" % process.returncode)
57 67
58 if __name__ == "__main__": 68 if __name__ == "__main__":
59 _update_abp2blocklist() 69 _update_abp2blocklist()
60 70
61 easylist = _download(_get_config_value("easylist_url")) 71 easylist = _download("easylist_url")
Sebastian Noack 2015/11/18 20:10:37 As _download is only used in these two lines, and
Felix Dahlke 2015/11/19 11:15:20 Done, did the same for _convert_filter_list.
62 exceptionrules = _download(_get_config_value("exceptionrules_url")) 72 exceptionrules = _download("exceptionrules_url")
63 73
64 _convert_filter_list(easylist, 74 _convert_filter_list([easylist], "easylist_content_blocker_path")
65 _get_config_value("easylist_content_blocker_path")) 75 _convert_filter_list([easylist, exceptionrules],
66 76 "combined_content_blocker_path")
67 combined = "\n".join((easylist, exceptionrules))
68 _convert_filter_list(combined,
kzar 2015/11/18 17:35:38 Nit: Maybe avoid the combined variable? _convert_
Sebastian Noack 2015/11/18 20:10:37 Even better: _convert_filter_list("%s\n%s" % (eas
Sebastian Noack 2015/11/18 23:24:34 Or maybe even even better: Make _convert_filter_li
Felix Dahlke 2015/11/19 11:15:20 Yeah I actually like that one better too, done.
69 _get_config_value("combined_content_blocker_path"))
LEFTRIGHT

Powered by Google App Engine
This is Rietveld