Issue 3774 - Support multiple mirrors for the Malware Domains List

sitescripts/subscriptions/bin/updateMalwareDomainsList.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2016 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
-import os, subprocess, codecs, urllib, zipfile, tempfile, shutil
+import os, subprocess, codecs, urllib2, zipfile, tempfile, shutil, sys
+import ConfigParser
 from StringIO import StringIO
 from sitescripts.utils import get_config
 
+
+filterlist_header = '''[Adblock Plus 1.1]
+! This is a list of malware domains generated from malwaredomains.com data.
+! Homepage: http://malwaredomains.com/?page_id=2
+! Last modified: %timestamp%
+! Expires: 1d
+!'''
+
+# Default value for malwaredomains_mirrors in the config.
+# Equivalent to the following in the config:
+# malwaredomains_mirrors=

[Sebastian Noack, 2016/03/14 13:36:07]

Example configuration should be documented in .sitescripts.ini.example rather
than inside the code.

[Vasily Kuznetsov, 2016/03/14 14:18:12]

Acknowledged.

[Vasily Kuznetsov, 2016/03/14 14:47:58]

Done.

+#     http://mirror3.malwaredomains.com
+#     http://mirror1.malwaredomains.com
+#     http://mirror2.malwaredomains.com
+default_mirrors_list = [

[Sebastian Noack, 2016/03/14 13:36:07]

I'd rather wait until sitescripts.ini got updated on the server before pushing
this. In particular, if you plan on removing that hard-coded list anyway.

[Vasily Kuznetsov, 2016/03/14 14:18:11]

I'm cool with this, but I'm not very sure how to update the config, that's why I
was thinking that a two-step process would allow us to get it running quicker.

[Sebastian Noack, 2016/03/14 15:40:12]

Matze, Felix or Wladimir can change the config manually on the server. So that
shouldn't be a big deal. Feel free to go already asking them to already do so.
Filing a follow-up issue to get rid of the hard-coded defaults on the other hand
seems to be more hassle.

+    'http://mirror3.malwaredomains.com',
+    'http://mirror1.malwaredomains.com',
+    'http://mirror2.malwaredomains.com'
+]
+
+malwaredomains_path = '/files/justdomains.zip'

[Sebastian Noack, 2016/03/14 13:36:07]

Nit: Please use upper case for constant-like variables.

[Vasily Kuznetsov, 2016/03/14 14:18:11]

Acknowledged.

[Vasily Kuznetsov, 2016/03/14 14:47:58]

Done.

+
+
+def try_mirror(mirror, path):
+  try:
+    response = urllib2.urlopen(mirror + path)
+    return response.read()
+  except urllib2.HTTPError, err:
+    print >>sys.stderr, '{}: {}'.format(mirror, err)

[Sebastian Noack, 2016/03/14 13:36:07]

Any output we generate will result into an email sent to the admins, when this
scripts runs by cron on the server. And individual mirrors occasionally failing
is expected, hence nothing somebody on our end needs to look into. So we should
only generate output if all mirrors in the list fail.

[Vasily Kuznetsov, 2016/03/14 14:18:11]

Acknowledged.

[Vasily Kuznetsov, 2016/03/14 14:47:58]

Done.

+    return None
+
+
 if __name__ == '__main__':
-  repository = get_config().get('subscriptionDownloads', 'malwaredomains_repository')
+  config = get_config()
+  repository = config.get('subscriptionDownloads', 'malwaredomains_repository')
+  try:
+    mirrors = config.get('subscriptionDownloads', 'malwaredomains_mirrors')
+    mirrors_list = filter(None, [mirror.strip() for mirror in mirrors.split()])
+  except ConfigParser.NoOptionError:
+    mirrors_list = default_mirrors_list
+
   tempdir = tempfile.mkdtemp(prefix='malwaredomains')
   try:
     subprocess.check_call(['hg', '-q', 'clone',  '-U', repository, tempdir])
     subprocess.check_call(['hg', '-q', 'up', '-R', tempdir, '-r', 'default'])
 
     path = os.path.join(tempdir, 'malwaredomains_full.txt')
     file = codecs.open(path, 'wb', encoding='utf-8')
 
-    print >>file, '''[Adblock Plus 1.1]
-! This is a list of malware domains generated from malwaredomains.com data.
-! Homepage: http://malwaredomains.com/?page_id=2
-! Last modified: %timestamp%
-! Expires: 1d
-!'''
+    print >>file, filterlist_header
 
-    data = urllib.urlopen('http://mirror3.malwaredomains.com/files/justdomains.zip').read()
+    for mirror in mirrors_list:
+      data = try_mirror(mirror, malwaredomains_path)
+      if data is not None:
+        break
+    else:
+      print >>sys.stderr, 'Unable to fetch malware domains list.'
+      sys.exit(1)
+
     zip = zipfile.ZipFile(StringIO(data), 'r')
     info = zip.infolist()[0]
     for line in str(zip.read(info.filename)).splitlines():
       domain = line.strip()
       if not domain:
         continue
 
       print >>file, '||%s^' % domain.decode('idna')
     file.close();
 
     if subprocess.check_output(['hg', 'stat', '-R', tempdir]) != '':
       subprocess.check_call(['hg', '-q', 'commit', '-R', tempdir, '-A', '-u', 'hgbot', '-m', 'Updated malwaredomains.com data'])
       subprocess.check_call(['hg', '-q', 'push', '-R', tempdir])
   finally:
     shutil.rmtree(tempdir, ignore_errors=True)