Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: generate_lists.py

Issue 29328894: Issue 3168 - Add a script for generating new content blocker lists (Closed)
Patch Set: Use the with statement Created Oct. 22, 2015, 3:15 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 # This file is part of Adblock Plus <https://adblockplus.org/>,
5 # Copyright (C) 2006-2015 Eyeo GmbH
6 #
7 # Adblock Plus is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation.
10 #
11 # Adblock Plus is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
18
19 import os
20 import shutil
21 import subprocess
22 import tempfile
23 import urllib2
24
25 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
26 ABP2BLOCKLIST_URL = "https://hg.adblockplus.org/abp2blocklist"
27 ABP2BLOCKLIST_PATH = os.path.join(BASE_PATH, "abp2blocklist")
28 EASYLIST_URL = "https://easylist-downloads.adblockplus.org/easylist_noadult.txt"
29 EXCEPTIONRULES_URL = "https://easylist-downloads.adblockplus.org/exceptionrules. txt"
30 EASYLIST_CONTENT_BLOCKER_PATH = os.path.join(BASE_PATH, "easylist_content_blocke r.json")
31 COMBINED_CONTENT_BLOCKER_PATH = os.path.join(BASE_PATH, "easylist+exceptionrules _content_blocker.json")
32
33 def _update_abp2blocklist():
34 if os.path.isdir(ABP2BLOCKLIST_PATH):
35 subprocess.check_call(["hg", "pull", "-u", "-R", ABP2BLOCKLIST_PATH])
36 else:
37 subprocess.check_call(["hg", "clone", ABP2BLOCKLIST_URL,
38 ABP2BLOCKLIST_PATH])
39 subprocess.check_call(["npm", "install"], cwd=ABP2BLOCKLIST_PATH)
40
41 def _download_filter_lists():
42 try:
43 easylist_response = urllib2.urlopen(EASYLIST_URL)
44 easylist_file = tempfile.NamedTemporaryFile(mode="wb+")
45 shutil.copyfileobj(easylist_response, easylist_file)
46 finally:
47 easylist_response.close()
48
49 try:
50 exceptionrules_response = urllib2.urlopen(EXCEPTIONRULES_URL)
51 exceptionrules_file = tempfile.NamedTemporaryFile(mode="wb+")
52 shutil.copyfileobj(exceptionrules_response, exceptionrules_file)
53 finally:
54 exceptionrules_response.close()
55
56 return (easylist_file, exceptionrules_file)
57
58 def _convert_filter_list(source_file, destination_path):
59 source_file.seek(0)
60 with open(destination_path, "wb") as destination_file:
61 subprocess.check_call(["node", "abp2blocklist.js"],
62 cwd=ABP2BLOCKLIST_PATH, stdin=source_file,
63 stdout=destination_file)
64
65 def _concatenate_files(*source_files):
66 destination_file = tempfile.NamedTemporaryFile(mode="wb+")
67 for source_file in source_files:
68 source_file.seek(0)
69 shutil.copyfileobj(source_file, destination_file)
70 return destination_file
71
72 if __name__ == "__main__":
73 print "Fetching/updating abp2blocklist ..."
74 _update_abp2blocklist()
75
76 print "Downloading filter lists ..."
77 easylist_file, exceptionrules_file = _download_filter_lists()
78
79 try:
80 print "Generating %s ..." % os.path.basename(EASYLIST_CONTENT_BLOCKER_PATH)
81 _convert_filter_list(easylist_file, EASYLIST_CONTENT_BLOCKER_PATH)
82
83 print "Generating %s ..." % os.path.basename(COMBINED_CONTENT_BLOCKER_PATH)
84 with _concatenate_files(easylist_file, exceptionrules_file) as combined_file :
85 _convert_filter_list(combined_file, COMBINED_CONTENT_BLOCKER_PATH)
86 finally:
87 easylist_file.close()
88 exceptionrules_file.close()
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld