Issue 3774 - Support multiple mirrors for the Malware Domains List

sitescripts/subscriptions/bin/updateMalwareDomainsList.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2016 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 19 matching lines @@
 
 DEFAULT_MIRRORS_LIST = [
     'http://mirror3.malwaredomains.com',
     'http://mirror1.malwaredomains.com',
     'http://mirror2.malwaredomains.com'
 ]
 
 MALWAREDOMAINS_PATH = '/files/justdomains.zip'
 
 
-def try_mirror(mirror, path):
+def try_mirror(mirror):
   try:
-    response = urllib2.urlopen(mirror + path)
+    response = urllib2.urlopen(mirror + MALWAREDOMAINS_PATH)
     return response.read()
   except urllib2.HTTPError:
     return None
 
 
 if __name__ == '__main__':
   config = get_config()
   repository = config.get('subscriptionDownloads', 'malwaredomains_repository')
   try:
     mirrors = config.get('subscriptionDownloads', 'malwaredomains_mirrors')
-    mirrors_list = filter(None, [mirror.strip() for mirror in mirrors.split()])
+    mirrors_list = mirrors.split()
   except ConfigParser.NoOptionError:
     mirrors_list = DEFAULT_MIRRORS_LIST
 
   tempdir = tempfile.mkdtemp(prefix='malwaredomains')
   try:
     subprocess.check_call(['hg', '-q', 'clone',  '-U', repository, tempdir])
     subprocess.check_call(['hg', '-q', 'up', '-R', tempdir, '-r', 'default'])
 
     path = os.path.join(tempdir, 'malwaredomains_full.txt')
     file = codecs.open(path, 'wb', encoding='utf-8')
 
     print >>file, FILTERLIST_HEADER
 
     for mirror in mirrors_list:
-      data = try_mirror(mirror, MALWAREDOMAINS_PATH)

[Sebastian Noack, 2016/03/14 15:40:12]

Since MALWAREDOMAINS_PATH is a global anyway, there is IMO no reason to pass it
around.

[Vasily Kuznetsov, 2016/03/14 16:20:38]

Yeah, I was thinking about it. I guess try_mirror is never intended to be used
anywhere other than this module, so it makes sense to just remove its second
argument.

+      data = try_mirror(mirror)
       if data is not None:
         break
     else:
-      print >>sys.stderr, 'Unable to fetch malware domains list.'
-      sys.exit(1)
+      sys.exit('Unable to fetch malware domains list.')
 
     zip = zipfile.ZipFile(StringIO(data), 'r')
     info = zip.infolist()[0]
     for line in str(zip.read(info.filename)).splitlines():
       domain = line.strip()
       if not domain:
         continue
 
       print >>file, '||%s^' % domain.decode('idna')
     file.close();
 
     if subprocess.check_output(['hg', 'stat', '-R', tempdir]) != '':
       subprocess.check_call(['hg', '-q', 'commit', '-R', tempdir, '-A', '-u', 'hgbot', '-m', 'Updated malwaredomains.com data'])
       subprocess.check_call(['hg', '-q', 'push', '-R', tempdir])
   finally:
     shutil.rmtree(tempdir, ignore_errors=True)