Issue #6707 - Make the generated malware domain filter list encode domains as Punycode

sitescripts/subscriptions/bin/updateMalwareDomainsList.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import subprocess
-import codecs
 import contextlib
 import urllib2
 import zipfile
 import tempfile
 import shutil
 import sys
 from StringIO import StringIO
 from sitescripts.utils import get_config
 
 
@@ skipping 24 matching lines @@
     section = 'subscriptionDownloads'
     repository = config.get(section, 'malwaredomains_repository')
     mirrors = config.get(section, 'malwaredomains_mirrors').split()
 
     tempdir = tempfile.mkdtemp(prefix='malwaredomains')
     try:
         subprocess.check_call(['hg', '-q', 'clone', '-U', repository, tempdir])
         subprocess.check_call(['hg', '-q', 'up', '-R', tempdir, '-r', 'default'])
 
         path = os.path.join(tempdir, 'malwaredomains_full.txt')
-        file = codecs.open(path, 'wb')
+        file = open(path, 'wb')
 
         print >>file, FILTERLIST_HEADER
 
         error_report = ['Unable to fetch malware domains list', 'Errors:']
         for mirror in mirrors:
             error_message, data = try_mirror(mirror)
             if data is not None:
                 break
             error_report.append(error_message)
         else:
             sys.exit('\n'.join(error_report))
 
         zf = zipfile.ZipFile(StringIO(data), 'r')
         info = zf.infolist()[0]
         for line in str(zf.read(info.filename)).splitlines():
             domain = line.strip()
             if not domain:
                 continue
 
-            print >>file, '||%s^' % domain.decode('idna').encode('punycode')

[Sebastian Noack, 2018/07/05 14:55:28]

This seems incorrect. Let's take "xn--fuson-1sa.org" (actual domain in the
Malware Domains List):

  >>> 'xn--fuson-1sa.org'.decode('idna').encode('punycode')
  b'fuson.org-i5a'

This is incorrect, we need 'xn--fuson-1sa.org' here (with the "xn--" annotation,
like it was in the input).

As far as I understand, this line should be just changed like this:

  print >>file, '||%s^' % domain

[Tudor Avram, 2018/07/06 11:53:16]

Done.

+            print >>file, '||%s^' % domain
         file.close()
 
         if subprocess.check_output(['hg', 'stat', '-R', tempdir]) != '':
-            subprocess.check_call(['hg', '-q', 'commit', '-R', tempdir, '-A', '-u', 'hgbot', '-m', 'Updated malwaredomains.com data'])
+            subprocess.check_call(['hg', '-q', 'commit', '-R', tempdir, '-A',
+                                   '-u', 'hgbot', '-m',
+                                   'Updated malwaredomains.com data'])
             subprocess.check_call(['hg', '-q', 'push', '-R', tempdir])
     finally:
         shutil.rmtree(tempdir, ignore_errors=True)
 
 
 if __name__ == '__main__':
     main()