Issue 3168 - Add a script for generating content blocker lists

sitescripts/content_blocker_lists/bin/generate_lists.py

+#!/usr/bin/env python
+# coding: utf-8
+
+# This file is part of Adblock Plus <https://adblockplus.org/>,
+# Copyright (C) 2006-2015 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+import subprocess
+import tempfile
+import urllib2
+
+from sitescripts.utils import get_config
+
+def _update_abp2blocklist():
+  abp2blocklist_path = get_config().get("content_blocker_lists",
+                                        "abp2blocklist_path")
+  if os.path.isdir(abp2blocklist_path):
+    subprocess.check_call(["hg", "pull", "-u", "-R", abp2blocklist_path])
+  else:
+    abp2blocklist_url = get_config().get("content_blocker_lists",
+                                         "abp2blocklist_url")
+    subprocess.check_call(["hg", "clone", abp2blocklist_url,
+                           abp2blocklist_path])
+  subprocess.check_call(["npm", "install"], cwd=abp2blocklist_path)
+
+def _download_filter_lists():
+  try:
+    easylist_url = get_config().get("content_blocker_lists", "easylist_url")
+    easylist_response = urllib2.urlopen(easylist_url)
+    easylist_file = tempfile.NamedTemporaryFile(mode="wb+")
+    shutil.copyfileobj(easylist_response, easylist_file)
+  finally:
+    easylist_response.close()
+
+  try:
+    exceptionrules_url = get_config().get("content_blocker_lists",
+                                          "exceptionrules_url")
+    exceptionrules_response = urllib2.urlopen(exceptionrules_url)
+    exceptionrules_file = tempfile.NamedTemporaryFile(mode="wb+")
+    shutil.copyfileobj(exceptionrules_response, exceptionrules_file)
+  finally:
+    exceptionrules_response.close()
+
+  return (easylist_file, exceptionrules_file)
+
+def _convert_filter_list(source_file, destination_path):
+  source_file.seek(0)
+  with open(destination_path, "wb") as destination_file:
+    abp2blocklist_path = get_config().get("content_blocker_lists",
+                                          "abp2blocklist_path")
+    subprocess.check_call(["node", "abp2blocklist.js"],
+                          cwd=abp2blocklist_path, stdin=source_file,
+                          stdout=destination_file)
+
+def _concatenate_files(*source_files):
+  destination_file = tempfile.NamedTemporaryFile(mode="wb+")
+  for source_file in source_files:
+    source_file.seek(0)
+    shutil.copyfileobj(source_file, destination_file)
+  return destination_file
+
+if __name__ == "__main__":
+  print "Fetching/updating abp2blocklist ..."
+  _update_abp2blocklist()
+
+  print "Downloading filter lists ..."
+  easylist_file, exceptionrules_file = _download_filter_lists()
+
+  try:
+    easylist_content_blocker_path = get_config().get("content_blocker_lists",
+                                                     "easylist_content_blocker_path")
+    print "Generating %s ..." % os.path.basename(easylist_content_blocker_path)
+    _convert_filter_list(easylist_file, easylist_content_blocker_path)
+
+    combined_content_blocker_path = get_config().get("content_blocker_lists",
+                                                     "combined_content_blocker_path")
+    print "Generating %s ..." % os.path.basename(combined_content_blocker_path)
+    with _concatenate_files(easylist_file, exceptionrules_file) as combined_file:
+      _convert_filter_list(combined_file, combined_content_blocker_path)
+  finally:
+    easylist_file.close()
+    exceptionrules_file.close()