Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/content_blocker_lists/bin/generate_lists.py

Issue 29345242: Noissue - Adapt quotes for compliance with our coding style in sitescripts (Closed)
Patch Set: Fixed raw string Created May 30, 2016, 8:47 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 2
3 # This file is part of Adblock Plus <https://adblockplus.org/>, 3 # This file is part of Adblock Plus <https://adblockplus.org/>,
4 # Copyright (C) 2006-2016 Eyeo GmbH 4 # Copyright (C) 2006-2016 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 from collections import OrderedDict 18 from collections import OrderedDict
19 from contextlib import closing 19 from contextlib import closing
20 import json 20 import json
21 import os 21 import os
22 import subprocess 22 import subprocess
23 import threading 23 import threading
24 import time 24 import time
25 import re 25 import re
26 import urllib2 26 import urllib2
27 27
28 from sitescripts.utils import get_config 28 from sitescripts.utils import get_config
29 29
30 config = dict(get_config().items("content_blocker_lists")) 30 config = dict(get_config().items('content_blocker_lists'))
31 31
32 32
33 def update_abp2blocklist(): 33 def update_abp2blocklist():
34 with open(os.devnull, "w") as devnull: 34 with open(os.devnull, 'w') as devnull:
35 abp2blocklist_path = config["abp2blocklist_path"] 35 abp2blocklist_path = config['abp2blocklist_path']
36 if os.path.isdir(abp2blocklist_path): 36 if os.path.isdir(abp2blocklist_path):
37 subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path) , 37 subprocess.check_call(('hg', 'pull', '-u', '-R', abp2blocklist_path) ,
38 stdout=devnull) 38 stdout=devnull)
39 else: 39 else:
40 subprocess.check_call(("hg", "clone", config["abp2blocklist_url"], 40 subprocess.check_call(('hg', 'clone', config['abp2blocklist_url'],
41 abp2blocklist_path), stdout=devnull) 41 abp2blocklist_path), stdout=devnull)
42 subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path, 42 subprocess.check_call(('npm', 'install'), cwd=abp2blocklist_path,
43 stdout=devnull) 43 stdout=devnull)
44 44
45 45
46 def download_filter_list(url): 46 def download_filter_list(url):
47 with closing(urllib2.urlopen(url)) as response: 47 with closing(urllib2.urlopen(url)) as response:
48 body = response.read() 48 body = response.read()
49 version = re.search(r"^(?:[^[!])|^!\s*Version:\s*(.+)$", 49 version = re.search(r'^(?:[^[!])|^!\s*Version:\s*(.+)$',
50 body, re.MULTILINE).group(1) 50 body, re.MULTILINE).group(1)
51 return body, url, version 51 return body, url, version
52 52
53 53
54 def generate_metadata(filter_lists, expires): 54 def generate_metadata(filter_lists, expires):
55 metadata = OrderedDict(( 55 metadata = OrderedDict((
56 ("version", time.strftime("%Y%m%d%H%M", time.gmtime())), 56 ('version', time.strftime('%Y%m%d%H%M', time.gmtime())),
57 ("expires", expires), 57 ('expires', expires),
58 ("sources", []) 58 ('sources', [])
59 )) 59 ))
60 for body, url, version in filter_lists: 60 for body, url, version in filter_lists:
61 metadata["sources"].append({"url": url, "version": version}) 61 metadata['sources'].append({'url': url, 'version': version})
62 return metadata 62 return metadata
63 63
64 64
65 def pipe_in(process, filter_lists): 65 def pipe_in(process, filter_lists):
66 try: 66 try:
67 for body, _, _ in filter_lists: 67 for body, _, _ in filter_lists:
68 print >>process.stdin, body 68 print >>process.stdin, body
69 finally: 69 finally:
70 process.stdin.close() 70 process.stdin.close()
71 71
72 72
73 def write_block_list(filter_lists, path, expires): 73 def write_block_list(filter_lists, path, expires):
74 block_list = generate_metadata(filter_lists, expires) 74 block_list = generate_metadata(filter_lists, expires)
75 process = subprocess.Popen(("node", "abp2blocklist.js"), 75 process = subprocess.Popen(('node', 'abp2blocklist.js'),
76 cwd=config["abp2blocklist_path"], 76 cwd=config['abp2blocklist_path'],
77 stdin=subprocess.PIPE, stdout=subprocess.PIPE) 77 stdin=subprocess.PIPE, stdout=subprocess.PIPE)
78 threading.Thread(target=pipe_in, args=(process, filter_lists)).start() 78 threading.Thread(target=pipe_in, args=(process, filter_lists)).start()
79 block_list["rules"] = json.load(process.stdout) 79 block_list['rules'] = json.load(process.stdout)
80 if process.wait(): 80 if process.wait():
81 raise Exception("abp2blocklist returned %s" % process.returncode) 81 raise Exception('abp2blocklist returned %s' % process.returncode)
82 82
83 with open(path, "wb") as destination_file: 83 with open(path, 'wb') as destination_file:
84 json.dump(block_list, destination_file, indent=2, separators=(",", ": ") ) 84 json.dump(block_list, destination_file, indent=2, separators=(',', ': ') )
85 85
86 if __name__ == "__main__": 86 if __name__ == '__main__':
87 update_abp2blocklist() 87 update_abp2blocklist()
88 88
89 easylist = download_filter_list(config["easylist_url"]) 89 easylist = download_filter_list(config['easylist_url'])
90 exceptionrules = download_filter_list(config["exceptionrules_url"]) 90 exceptionrules = download_filter_list(config['exceptionrules_url'])
91 91
92 write_block_list([easylist], 92 write_block_list([easylist],
93 config["easylist_content_blocker_path"], 93 config['easylist_content_blocker_path'],
94 config["easylist_content_blocker_expires"]) 94 config['easylist_content_blocker_expires'])
95 write_block_list([easylist, exceptionrules], 95 write_block_list([easylist, exceptionrules],
96 config["combined_content_blocker_path"], 96 config['combined_content_blocker_path'],
97 config["combined_content_blocker_expires"]) 97 config['combined_content_blocker_expires'])
OLDNEW

Powered by Google App Engine
This is Rietveld