Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Unified Diff: generate_lists.py

Issue 29328894: Issue 3168 - Add a script for generating new content blocker lists (Closed)
Patch Set: Close files explicitly, open files in binary mode, add whitespace Created Oct. 22, 2015, 2:42 a.m.
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: generate_lists.py
===================================================================
new file mode 100755
--- /dev/null
+++ b/generate_lists.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# This file is part of Adblock Plus <https://adblockplus.org/>,
+# Copyright (C) 2006-2015 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+import subprocess
+import tempfile
+import urllib2
+
+BASE_PATH = os.path.dirname(os.path.abspath(__file__))
+ABP2BLOCKLIST_URL = "https://hg.adblockplus.org/abp2blocklist"
+ABP2BLOCKLIST_PATH = os.path.join(BASE_PATH, "abp2blocklist")
+EASYLIST_URL = "https://easylist-downloads.adblockplus.org/easylist_noadult.txt"
+EXCEPTIONRULES_URL = "https://easylist-downloads.adblockplus.org/exceptionrules.txt"
+EASYLIST_CONTENT_BLOCKER_PATH = os.path.join(BASE_PATH, "easylist_content_blocker.json")
+COMBINED_CONTENT_BLOCKER_PATH = os.path.join(BASE_PATH, "easylist+exceptionrules_content_blocker.json")
+
+def _update_abp2blocklist():
+ if os.path.isdir(ABP2BLOCKLIST_PATH):
+ subprocess.check_call(["hg", "pull", "-u", "-R", ABP2BLOCKLIST_PATH])
+ else:
+ subprocess.check_call(["hg", "clone", ABP2BLOCKLIST_URL,
+ ABP2BLOCKLIST_PATH])
+ subprocess.check_call(["npm", "install"], cwd=ABP2BLOCKLIST_PATH)
+
+def _download_filter_lists():
+ try:
+ easylist_response = urllib2.urlopen(EASYLIST_URL)
+ easylist_file = tempfile.NamedTemporaryFile(mode="wb+")
+ shutil.copyfileobj(easylist_response, easylist_file)
+ finally:
+ easylist_response.close()
+
+ try:
+ exceptionrules_response = urllib2.urlopen(EXCEPTIONRULES_URL)
+ exceptionrules_file = tempfile.NamedTemporaryFile(mode="wb+")
+ shutil.copyfileobj(exceptionrules_response, exceptionrules_file)
+ finally:
+ exceptionrules_response.close()
+
+ return (easylist_file, exceptionrules_file)
+
+def _convert_filter_list(source_file, destination_path):
+ source_file.seek(0)
+ with open(destination_path, "wb") as destination_file:
+ subprocess.check_call(["node", "abp2blocklist.js"],
+ cwd=ABP2BLOCKLIST_PATH, stdin=source_file,
+ stdout=destination_file)
+
+def _concatenate_files(*source_files):
+ destination_file = tempfile.NamedTemporaryFile(mode="wb+")
+ for source_file in source_files:
+ source_file.seek(0)
+ shutil.copyfileobj(source_file, destination_file)
+ return destination_file
+
+if __name__ == "__main__":
+ print "Fetching/updating abp2blocklist ..."
+ _update_abp2blocklist()
+
+ print "Downloading filter lists ..."
+ easylist_file, exceptionrules_file = _download_filter_lists()
+
+ try:
+ print "Generating %s ..." % os.path.basename(EASYLIST_CONTENT_BLOCKER_PATH)
+ _convert_filter_list(easylist_file, EASYLIST_CONTENT_BLOCKER_PATH)
+
+ print "Generating %s ..." % os.path.basename(COMBINED_CONTENT_BLOCKER_PATH)
+ combined_file = _concatenate_files(easylist_file, exceptionrules_file)
Sebastian Noack 2015/10/22 02:55:06 With statement?
Felix Dahlke 2015/10/22 03:16:46 Good catch, couldn't use it above (not without int
+ try:
+ _convert_filter_list(combined_file, COMBINED_CONTENT_BLOCKER_PATH)
+ finally:
+ combined_file.close()
+ finally:
+ easylist_file.close()
+ exceptionrules_file.close()
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld