Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/content_blocker_lists/bin/generate_lists.py

Issue 29329537: Issue 3168 - Add a script for generating content blocker lists (Closed)
Patch Set: Remove _get_config_value, combine lists in _convert_filter_lists Created Nov. 19, 2015, 11:13 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « sitescripts/content_blocker_lists/bin/__init__.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 # This file is part of Adblock Plus <https://adblockplus.org/>,
5 # Copyright (C) 2006-2015 Eyeo GmbH
6 #
7 # Adblock Plus is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation.
10 #
11 # Adblock Plus is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
18
19 import os
20 import subprocess
21 import urllib2
22
23 from sitescripts.utils import get_config
24
25 def _update_abp2blocklist():
26 with open(os.devnull, "w") as devnull:
27 config = get_config()
28 abp2blocklist_path = config.get("content_blocker_lists",
29 "abp2blocklist_path")
30 if os.path.isdir(abp2blocklist_path):
31 subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path),
32 stdout=devnull)
33 else:
34 abp2blocklist_url = config.get("content_blocker_lists",
35 "abp2blocklist_url")
36 subprocess.check_call(("hg", "clone", abp2blocklist_url,
37 abp2blocklist_path), stdout=devnull)
38 subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path,
39 stdout=devnull)
40
41 def _download(url_key):
42 url = get_config().get("content_blocker_lists", url_key)
43 response = urllib2.urlopen(url)
44 try:
45 return response.read()
46 finally:
47 response.close()
48
49 def _convert_filter_list(sources, destination_path_key):
50 config = get_config()
51 destination_path = config.get("content_blocker_lists", destination_path_key)
52 with open(destination_path, "wb") as destination_file:
53 abp2blocklist_path = config.get("content_blocker_lists",
54 "abp2blocklist_path")
55 process = subprocess.Popen(("node", "abp2blocklist.js"),
56 cwd=abp2blocklist_path, stdin=subprocess.PIPE,
57 stdout=destination_file)
58 try:
59 for source in sources:
60 print >>process.stdin, source
61 finally:
62 process.stdin.close()
63 if process.wait():
Felix Dahlke 2015/11/19 11:15:20 Do you think this is too obscure?
Sebastian Noack 2015/11/19 20:08:34 This ... 1. Introduces an unnecessary level of in
Felix Dahlke 2015/11/19 20:48:43 Done.
64 raise Exception("abp2blocklist returned %s" % process.returncode)
65
66 if __name__ == "__main__":
67 _update_abp2blocklist()
68
69 easylist = _download("easylist_url")
70 exceptionrules = _download("exceptionrules_url")
71
72 _convert_filter_list([easylist], "easylist_content_blocker_path")
73 _convert_filter_list([easylist, exceptionrules],
74 "combined_content_blocker_path")
OLDNEW
« no previous file with comments | « sitescripts/content_blocker_lists/bin/__init__.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld