Issue 3168 - Add a script for generating content blocker lists

sitescripts/content_blocker_lists/bin/generate_lists.py

 #!/usr/bin/env python
 # coding: utf-8
 
 # This file is part of Adblock Plus <https://adblockplus.org/>,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import shutil
 import subprocess
-import tempfile
 import urllib2
 
 from sitescripts.utils import get_config
 
 def _update_abp2blocklist():
-  abp2blocklist_path = get_config().get("content_blocker_lists",
-                                        "abp2blocklist_path")
-  if os.path.isdir(abp2blocklist_path):
-    subprocess.check_call(["hg", "pull", "-u", "-R", abp2blocklist_path])
-  else:
-    abp2blocklist_url = get_config().get("content_blocker_lists",
-                                         "abp2blocklist_url")
-    subprocess.check_call(["hg", "clone", abp2blocklist_url,
-                           abp2blocklist_path])
-  subprocess.check_call(["npm", "install"], cwd=abp2blocklist_path)
+  with open(os.devnull, "w") as devnull:
+    config = get_config()
+    abp2blocklist_path = config.get("content_blocker_lists",
+                                    "abp2blocklist_path")
+    if os.path.isdir(abp2blocklist_path):
+        subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path),
+                              stdout=devnull)
+    else:
+      abp2blocklist_url = config.get("content_blocker_lists",
+                                     "abp2blocklist_url")
+      subprocess.check_call(("hg", "clone", abp2blocklist_url,
+                             abp2blocklist_path), stdout=devnull)
+    subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path,
+                          stdout=devnull)
 
-def _download_filter_lists():
+def _download(url_key):
+  url = get_config().get("content_blocker_lists", url_key)
+  response = urllib2.urlopen(url)
   try:
-    easylist_url = get_config().get("content_blocker_lists", "easylist_url")
-    easylist_response = urllib2.urlopen(easylist_url)
-    easylist_file = tempfile.NamedTemporaryFile(mode="wb+")
-    shutil.copyfileobj(easylist_response, easylist_file)
+    return response.read()
   finally:
-    easylist_response.close()
+    response.close()
 
-  try:
-    exceptionrules_url = get_config().get("content_blocker_lists",
-                                          "exceptionrules_url")
-    exceptionrules_response = urllib2.urlopen(exceptionrules_url)
-    exceptionrules_file = tempfile.NamedTemporaryFile(mode="wb+")
-    shutil.copyfileobj(exceptionrules_response, exceptionrules_file)
-  finally:
-    exceptionrules_response.close()
+def _convert_filter_list(sources, destination_path_key):
+  config = get_config()
+  destination_path = config.get("content_blocker_lists", destination_path_key)
+  with open(destination_path, "wb") as destination_file:
+    abp2blocklist_path = config.get("content_blocker_lists",
+                                    "abp2blocklist_path")
+    process = subprocess.Popen(("node", "abp2blocklist.js"),
+                               cwd=abp2blocklist_path, stdin=subprocess.PIPE,
+                               stdout=destination_file)
+    try:
+      for source in sources:
+        print >>process.stdin, source
+    finally:
+      process.stdin.close()
+      process.wait()
 
-  return (easylist_file, exceptionrules_file)
-
-def _convert_filter_list(source_file, destination_path):
-  source_file.seek(0)
-  with open(destination_path, "wb") as destination_file:
-    abp2blocklist_path = get_config().get("content_blocker_lists",
-                                          "abp2blocklist_path")
-    subprocess.check_call(["node", "abp2blocklist.js"],
-                          cwd=abp2blocklist_path, stdin=source_file,
-                          stdout=destination_file)
-
-def _concatenate_files(*source_files):
-  destination_file = tempfile.NamedTemporaryFile(mode="wb+")
-  for source_file in source_files:
-    source_file.seek(0)
-    shutil.copyfileobj(source_file, destination_file)
-  return destination_file
+  if process.returncode:
+    raise Exception("abp2blocklist returned %s" % process.returncode)
 
 if __name__ == "__main__":
-  print "Fetching/updating abp2blocklist ..."
   _update_abp2blocklist()
 
-  print "Downloading filter lists ..."
-  easylist_file, exceptionrules_file = _download_filter_lists()
+  easylist = _download("easylist_url")
+  exceptionrules = _download("exceptionrules_url")
 
-  try:
-    easylist_content_blocker_path = get_config().get("content_blocker_lists",
-                                                     "easylist_content_blocker_path")
-    print "Generating %s ..." % os.path.basename(easylist_content_blocker_path)
-    _convert_filter_list(easylist_file, easylist_content_blocker_path)
-
-    combined_content_blocker_path = get_config().get("content_blocker_lists",
-                                                     "combined_content_blocker_path")
-    print "Generating %s ..." % os.path.basename(combined_content_blocker_path)
-    with _concatenate_files(easylist_file, exceptionrules_file) as combined_file:
-      _convert_filter_list(combined_file, combined_content_blocker_path)
-  finally:
-    easylist_file.close()
-    exceptionrules_file.close()
+  _convert_filter_list([easylist], "easylist_content_blocker_path")
+  _convert_filter_list([easylist, exceptionrules],
+                       "combined_content_blocker_path")