Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/content_blocker_lists/bin/generate_lists.py

Issue 29331148: Issue 3176 - Add metadata to content blocker lists (Closed)
Patch Set: Just grab version number instead of parsing header Created Dec. 8, 2015, 2:32 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « .sitescripts.example ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # coding: utf-8 2 # coding: utf-8
3 3
4 # This file is part of Adblock Plus <https://adblockplus.org/>, 4 # This file is part of Adblock Plus <https://adblockplus.org/>,
5 # Copyright (C) 2006-2015 Eyeo GmbH 5 # Copyright (C) 2006-2015 Eyeo GmbH
6 # 6 #
7 # Adblock Plus is free software: you can redistribute it and/or modify 7 # Adblock Plus is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as 8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation. 9 # published by the Free Software Foundation.
10 # 10 #
11 # Adblock Plus is distributed in the hope that it will be useful, 11 # Adblock Plus is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details. 14 # GNU General Public License for more details.
15 # 15 #
16 # You should have received a copy of the GNU General Public License 16 # You should have received a copy of the GNU General Public License
17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 17 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
18 18
19 from collections import OrderedDict
20 from contextlib import closing
21 import json
19 import os 22 import os
20 import subprocess 23 import subprocess
24 import threading
25 import time
26 import re
21 import urllib2 27 import urllib2
22 28
23 from sitescripts.utils import get_config 29 from sitescripts.utils import get_config
24 30
25 def _update_abp2blocklist(): 31 config = dict(get_config().items("content_blocker_lists"))
32
33 def update_abp2blocklist():
26 with open(os.devnull, "w") as devnull: 34 with open(os.devnull, "w") as devnull:
27 config = get_config() 35 abp2blocklist_path = config["abp2blocklist_path"]
28 abp2blocklist_path = config.get("content_blocker_lists",
29 "abp2blocklist_path")
30 if os.path.isdir(abp2blocklist_path): 36 if os.path.isdir(abp2blocklist_path):
31 subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path), 37 subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path),
32 stdout=devnull) 38 stdout=devnull)
33 else: 39 else:
34 abp2blocklist_url = config.get("content_blocker_lists", 40 subprocess.check_call(("hg", "clone", config["abp2blocklist_url"],
35 "abp2blocklist_url")
36 subprocess.check_call(("hg", "clone", abp2blocklist_url,
37 abp2blocklist_path), stdout=devnull) 41 abp2blocklist_path), stdout=devnull)
38 subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path, 42 subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path,
39 stdout=devnull) 43 stdout=devnull)
40 44
41 def _download(url_key): 45 def download_filter_list(url):
42 url = get_config().get("content_blocker_lists", url_key) 46 with closing(urllib2.urlopen(url)) as response:
43 response = urllib2.urlopen(url) 47 body = response.read()
48 version = re.search(r"^(?:[^[!])|!\s*Version:\s*(.+)$",
Sebastian Noack 2015/12/08 15:08:38 Indeed, we can simply use re.search instead iterat
kzar 2015/12/08 15:37:43 Well it's true about the "! foo ! Version: foo" ma
Sebastian Noack 2015/12/08 15:45:51 Well, theoretically the filter list can be empty.
49 body, re.MULTILINE).group(1)
50 return body, url, version
51
52 def generate_metadata(filter_lists, expires):
53 metadata = OrderedDict((
54 ("version", time.strftime("%Y%m%d%H%M", time.gmtime())),
55 ("expires", expires),
56 ("sources", [])
57 ))
58 for body, url, version in filter_lists:
59 metadata["sources"].append({"url": url, "version": version})
60 return metadata
61
62 def pipe_in(process, filter_lists):
44 try: 63 try:
45 return response.read() 64 for body, _, _ in filter_lists:
65 print >>process.stdin, body
46 finally: 66 finally:
47 response.close() 67 process.stdin.close()
48 68
49 def _convert_filter_list(sources, destination_path_key): 69 def write_block_list(filter_lists, path, expires):
50 config = get_config() 70 block_list = generate_metadata(filter_lists, expires)
51 destination_path = config.get("content_blocker_lists", destination_path_key) 71 process = subprocess.Popen(("node", "abp2blocklist.js"),
52 with open(destination_path, "wb") as destination_file: 72 cwd=config["abp2blocklist_path"],
53 abp2blocklist_path = config.get("content_blocker_lists", 73 stdin=subprocess.PIPE, stdout=subprocess.PIPE)
54 "abp2blocklist_path") 74 threading.Thread(target=pipe_in, args=(process, filter_lists)).start()
55 process = subprocess.Popen(("node", "abp2blocklist.js"), 75 block_list["rules"] = json.load(process.stdout)
56 cwd=abp2blocklist_path, stdin=subprocess.PIPE, 76 if process.wait():
57 stdout=destination_file)
58 try:
59 for source in sources:
60 print >>process.stdin, source
61 finally:
62 process.stdin.close()
63 process.wait()
64
65 if process.returncode:
66 raise Exception("abp2blocklist returned %s" % process.returncode) 77 raise Exception("abp2blocklist returned %s" % process.returncode)
67 78
79 with open(path, "wb") as destination_file:
80 json.dump(block_list, destination_file, indent=2, separators=(",", ": "))
81
68 if __name__ == "__main__": 82 if __name__ == "__main__":
69 _update_abp2blocklist() 83 update_abp2blocklist()
70 84
71 easylist = _download("easylist_url") 85 easylist = download_filter_list(config["easylist_url"])
72 exceptionrules = _download("exceptionrules_url") 86 exceptionrules = download_filter_list(config["exceptionrules_url"])
73 87
74 _convert_filter_list([easylist], "easylist_content_blocker_path") 88 write_block_list([easylist],
75 _convert_filter_list([easylist, exceptionrules], 89 config["easylist_content_blocker_path"],
76 "combined_content_blocker_path") 90 config["easylist_content_blocker_expires"])
91 write_block_list([easylist, exceptionrules],
92 config["combined_content_blocker_path"],
93 config["combined_content_blocker_expires"])
OLDNEW
« no previous file with comments | « .sitescripts.example ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld