Issue #6707 - Make the generated malware domain filter list encode domains as Punycode

sitescripts/subscriptions/test/test_updateMalwareDomainsList.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
@@ skipping 40 matching lines @@
 
 @pytest.fixture(autouse=True)
 def urlopen(mocker):
     """Mock urlopen function"""
     real_urlopen = urllib2.urlopen
 
     def mock_urlopen(url):
         if url.startswith('good'):
             zf_data = io.BytesIO()
             with zipfile.ZipFile(zf_data, 'w') as zf:
-                idna_url = 'xn--xhq02ykwbp4a.cn\n'

[Vasily Kuznetsov, 2018/07/03 16:01:52]

I think it would make sense to have 2 domains in the list just to make sure that
each domain is treated separately by the recoding process. The reason for this
is that if we take something like 'xn--domain1\nxn--domain2\n' and then recode
it without breaking into separate lines, the result is different from breaking
it into separate lines and then processing each line separately (this is what we
want).

The checking in line 77 will need to be a bit more sophisticated to check that
the result is really what we want.

[Tudor Avram, 2018/07/06 11:53:16]

Done.

-                zf.writestr('justdomains', idna_url)
+                idna_urls = ['xn--xhq02ykwbp4a.cn', 'xn--fiq80yua78t.tw']
+                zf.writestr('justdomains', '\n'.join(idna_urls) + '\n')
             return io.BytesIO(zf_data.getvalue())
         if url.startswith('bad'):
             raise urllib2.HTTPError('Bad', '42', 'No good', [], None)
         if url.startswith('ugly'):
             raise urllib2.URLError('Ugly')
         return real_urlopen(url)
 
     mocker.patch('urllib2.urlopen', mock_urlopen)
 
 
 def test_good(md_repo):
     main()
     subprocess.check_call(['hg', 'up'], cwd=md_repo.strpath)
     result = md_repo.join('malwaredomains_full.txt').read()
-    assert '.cn-w48dx81cufdl6b' in result
+    puny_urls = ['xn--xhq02ykwbp4a.cn', 'xn--fiq80yua78t.tw']
+    for url in puny_urls:
+        assert url in result
 
 
 def test_bad(md_repo, config):
     config.set(CONF_SECTION, 'malwaredomains_mirrors', 'bad')
     try:
         main()
     except SystemExit as exc:
         err_lines = str(exc).splitlines()
         assert len(err_lines) == 3
         assert 'Failed to fetch bad/files/justdomains.zip' in err_lines[2]
         assert '42: No good' in err_lines[2]
 
 
 def test_ugly(md_repo, config):
     config.set(CONF_SECTION, 'malwaredomains_mirrors', 'bad\nugly')
     try:
         main()
     except SystemExit as exc:
         err_lines = str(exc).splitlines()
         assert len(err_lines) == 4
         assert 'Failed to fetch bad/files/justdomains.zip' in err_lines[2]
         assert 'Failed to fetch ugly/files/justdomains.zip' in err_lines[3]
         assert 'Ugly' in err_lines[3]