Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: generate_lists.py

Issue 29328894: Issue 3168 - Add a script for generating new content blocker lists (Closed)
Patch Set: Address comments Created Oct. 21, 2015, 8:26 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 # This file is part of Adblock Plus <https://adblockplus.org/>,
5 # Copyright (C) 2006-2015 Eyeo GmbH
6 #
7 # Adblock Plus is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation.
10 #
11 # Adblock Plus is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
18
19 import os
20 import shutil
21 import subprocess
22 import tempfile
23 import urllib2
24
25 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
26 ABP2BLOCKLIST_URL = "https://hg.adblockplus.org/abp2blocklist"
27 ABP2BLOCKLIST_PATH = os.path.join(BASE_PATH, "abp2blocklist")
28 EASYLIST_URL = "https://easylist-downloads.adblockplus.org/easylist_noadult.txt"
29 EXCEPTIONRULES_URL = "https://easylist-downloads.adblockplus.org/exceptionrules. txt"
30 EASYLIST_CONTENT_BLOCKER_PATH = os.path.join(BASE_PATH, "easylist_content_blocke r.json")
31 COMBINED_CONTENT_BLOCKER_PATH = os.path.join(BASE_PATH, "easylist+exceptionrules _content_blocker.json")
32
33 def _update_abp2blocklist():
34 if os.path.isdir(ABP2BLOCKLIST_PATH):
35 subprocess.check_call(["hg", "pull", "-u", "-R", ABP2BLOCKLIST_PATH])
36 else:
37 subprocess.check_call(["hg", "clone", ABP2BLOCKLIST_URL,
38 ABP2BLOCKLIST_PATH])
39 subprocess.check_call(["npm", "install"], cwd=ABP2BLOCKLIST_PATH)
40
41 def _download_filter_lists():
42 try:
43 easylist_response = urllib2.urlopen(EASYLIST_URL)
44 easylist_file = tempfile.NamedTemporaryFile(mode="w+")
45 shutil.copyfileobj(easylist_response, easylist_file)
46 finally:
47 easylist_response.close()
48 try:
49 exceptionrules_response = urllib2.urlopen(EXCEPTIONRULES_URL)
50 exceptionrules_file = tempfile.NamedTemporaryFile(mode="w+")
51 shutil.copyfileobj(exceptionrules_response, exceptionrules_file)
52 finally:
53 exceptionrules_response.close()
54 return (easylist_file, exceptionrules_file)
55
56 def _concatenate_files(*source_files):
57 destination_file = tempfile.NamedTemporaryFile(mode="w+")
58 for source_file in source_files:
59 source_file.seek(0)
60 shutil.copyfileobj(source_file, destination_file)
61 return destination_file
62
63 def _convert_filter_list(source_file, destination_path):
64 source_file.seek(0)
65 with open(destination_path, "w") as destination_file:
66 subprocess.check_call(["node", "abp2blocklist.js"],
67 cwd=ABP2BLOCKLIST_PATH, stdin=source_file,
68 stdout=destination_file)
69
70 if __name__ == "__main__":
71 print "Fetching/updating abp2blocklist ..."
72 _update_abp2blocklist()
73
74 print "Downloading filter lists ..."
75 easylist_file, exceptionrules_file = _download_filter_lists()
76
77 print "Generating %s ..." % os.path.basename(EASYLIST_CONTENT_BLOCKER_PATH)
78 _convert_filter_list(easylist_file, EASYLIST_CONTENT_BLOCKER_PATH)
79
80 print "Generating %s ..." % os.path.basename(COMBINED_CONTENT_BLOCKER_PATH)
81 combined_file = _concatenate_files(easylist_file, exceptionrules_file)
82 _convert_filter_list(combined_file, COMBINED_CONTENT_BLOCKER_PATH)
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld