Issue 2719 - Decode domains when generating the "Malware Domains" list

sitescripts/subscriptions/bin/updateMalwareDomainsList.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import os, subprocess, codecs, urllib, zipfile, tempfile, shutil
 from StringIO import StringIO
-from sitescripts.utils import get_config, setupStderr
+from sitescripts.utils import get_config
 
 if __name__ == '__main__':
-  setupStderr()
-
   repository = get_config().get('subscriptionDownloads', 'malwaredomains_repository')
   tempdir = tempfile.mkdtemp(prefix='malwaredomains')
   try:
     subprocess.check_call(['hg', '-q', 'clone',  '-U', repository, tempdir])
     subprocess.check_call(['hg', '-q', 'up', '-R', tempdir, '-r', 'default'])
 
     path = os.path.join(tempdir, 'malwaredomains_full.txt')
     file = codecs.open(path, 'wb', encoding='utf-8')
 
     print >>file, '''[Adblock Plus 1.1]
 ! This is a list of malware domains generated from malwaredomains.com data.
 ! Homepage: http://malwaredomains.com/?page_id=2
 ! Last modified: %timestamp%
 ! Expires: 1d
 !'''
 
     data = urllib.urlopen('http://mirror3.malwaredomains.com/files/justdomains.zip').read()
     zip = zipfile.ZipFile(StringIO(data), 'r')
     info = zip.infolist()[0]
     for line in str(zip.read(info.filename)).splitlines():
       if not line:
         continue
-      print >>file, '||%s^' % line.strip().decode('iso-8859-1')
+      print >>file, '||%s^' % line.strip().decode('idna')
     file.close();
 
     if subprocess.check_output(['hg', 'stat', '-R', tempdir]) != '':
       subprocess.check_call(['hg', '-q', 'commit', '-R', tempdir, '-A', '-u', 'hgbot', '-m', 'Updated malwaredomains.com data'])
       subprocess.check_call(['hg', '-q', 'push', '-R', tempdir])
   finally:
     shutil.rmtree(tempdir, ignore_errors=True)