Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: generate_lists.py

Issue 29328894: Issue 3168 - Add a script for generating new content blocker lists (Closed)
Left Patch Set: Flip _concatenate_files and _convert_filter_list Created Oct. 21, 2015, 8:34 p.m.
Right Patch Set: Use the with statement Created Oct. 22, 2015, 3:15 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « no previous file | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # coding: utf-8 2 # coding: utf-8
3 3
4 # This file is part of Adblock Plus <https://adblockplus.org/>, 4 # This file is part of Adblock Plus <https://adblockplus.org/>,
5 # Copyright (C) 2006-2015 Eyeo GmbH 5 # Copyright (C) 2006-2015 Eyeo GmbH
6 # 6 #
7 # Adblock Plus is free software: you can redistribute it and/or modify 7 # Adblock Plus is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as 8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation. 9 # published by the Free Software Foundation.
10 # 10 #
(...skipping 23 matching lines...) Expand all
34 if os.path.isdir(ABP2BLOCKLIST_PATH): 34 if os.path.isdir(ABP2BLOCKLIST_PATH):
35 subprocess.check_call(["hg", "pull", "-u", "-R", ABP2BLOCKLIST_PATH]) 35 subprocess.check_call(["hg", "pull", "-u", "-R", ABP2BLOCKLIST_PATH])
36 else: 36 else:
37 subprocess.check_call(["hg", "clone", ABP2BLOCKLIST_URL, 37 subprocess.check_call(["hg", "clone", ABP2BLOCKLIST_URL,
38 ABP2BLOCKLIST_PATH]) 38 ABP2BLOCKLIST_PATH])
39 subprocess.check_call(["npm", "install"], cwd=ABP2BLOCKLIST_PATH) 39 subprocess.check_call(["npm", "install"], cwd=ABP2BLOCKLIST_PATH)
40 40
41 def _download_filter_lists(): 41 def _download_filter_lists():
42 try: 42 try:
43 easylist_response = urllib2.urlopen(EASYLIST_URL) 43 easylist_response = urllib2.urlopen(EASYLIST_URL)
44 easylist_file = tempfile.NamedTemporaryFile(mode="w+") 44 easylist_file = tempfile.NamedTemporaryFile(mode="wb+")
45 shutil.copyfileobj(easylist_response, easylist_file) 45 shutil.copyfileobj(easylist_response, easylist_file)
46 finally: 46 finally:
47 easylist_response.close() 47 easylist_response.close()
Sebastian Noack 2015/10/21 20:56:23 Nit: An empty line after each finally block would
Felix Dahlke 2015/10/22 02:43:45 Done.
48
48 try: 49 try:
49 exceptionrules_response = urllib2.urlopen(EXCEPTIONRULES_URL) 50 exceptionrules_response = urllib2.urlopen(EXCEPTIONRULES_URL)
50 exceptionrules_file = tempfile.NamedTemporaryFile(mode="w+") 51 exceptionrules_file = tempfile.NamedTemporaryFile(mode="wb+")
51 shutil.copyfileobj(exceptionrules_response, exceptionrules_file) 52 shutil.copyfileobj(exceptionrules_response, exceptionrules_file)
52 finally: 53 finally:
53 exceptionrules_response.close() 54 exceptionrules_response.close()
55
54 return (easylist_file, exceptionrules_file) 56 return (easylist_file, exceptionrules_file)
55 57
56 def _convert_filter_list(source_file, destination_path): 58 def _convert_filter_list(source_file, destination_path):
57 source_file.seek(0) 59 source_file.seek(0)
58 with open(destination_path, "w") as destination_file: 60 with open(destination_path, "wb") as destination_file:
59 subprocess.check_call(["node", "abp2blocklist.js"], 61 subprocess.check_call(["node", "abp2blocklist.js"],
60 cwd=ABP2BLOCKLIST_PATH, stdin=source_file, 62 cwd=ABP2BLOCKLIST_PATH, stdin=source_file,
61 stdout=destination_file) 63 stdout=destination_file)
62 64
63 def _concatenate_files(*source_files): 65 def _concatenate_files(*source_files):
64 destination_file = tempfile.NamedTemporaryFile(mode="w+") 66 destination_file = tempfile.NamedTemporaryFile(mode="wb+")
65 for source_file in source_files: 67 for source_file in source_files:
66 source_file.seek(0) 68 source_file.seek(0)
67 shutil.copyfileobj(source_file, destination_file) 69 shutil.copyfileobj(source_file, destination_file)
68 return destination_file 70 return destination_file
69 71
70 if __name__ == "__main__": 72 if __name__ == "__main__":
71 print "Fetching/updating abp2blocklist ..." 73 print "Fetching/updating abp2blocklist ..."
72 _update_abp2blocklist() 74 _update_abp2blocklist()
73 75
74 print "Downloading filter lists ..." 76 print "Downloading filter lists ..."
75 easylist_file, exceptionrules_file = _download_filter_lists() 77 easylist_file, exceptionrules_file = _download_filter_lists()
Sebastian Noack 2015/10/21 21:14:35 Mind closing the temporary files again?
Felix Dahlke 2015/10/22 02:43:45 The OS is closing the files when the runtime termi
76 78
77 print "Generating %s ..." % os.path.basename(EASYLIST_CONTENT_BLOCKER_PATH) 79 try:
78 _convert_filter_list(easylist_file, EASYLIST_CONTENT_BLOCKER_PATH) 80 print "Generating %s ..." % os.path.basename(EASYLIST_CONTENT_BLOCKER_PATH)
81 _convert_filter_list(easylist_file, EASYLIST_CONTENT_BLOCKER_PATH)
79 82
80 print "Generating %s ..." % os.path.basename(COMBINED_CONTENT_BLOCKER_PATH) 83 print "Generating %s ..." % os.path.basename(COMBINED_CONTENT_BLOCKER_PATH)
81 combined_file = _concatenate_files(easylist_file, exceptionrules_file) 84 with _concatenate_files(easylist_file, exceptionrules_file) as combined_file :
82 _convert_filter_list(combined_file, COMBINED_CONTENT_BLOCKER_PATH) 85 _convert_filter_list(combined_file, COMBINED_CONTENT_BLOCKER_PATH)
86 finally:
87 easylist_file.close()
88 exceptionrules_file.close()
LEFTRIGHT
« no previous file | no next file » | Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Toggle Comments ('s')

Powered by Google App Engine
This is Rietveld