Index: sitescripts/subscriptions/bin/updateMalwareDomainsList.py
===================================================================
--- a/sitescripts/subscriptions/bin/updateMalwareDomainsList.py
+++ b/sitescripts/subscriptions/bin/updateMalwareDomainsList.py
@@ -11,16 +11,17 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Adblock Plus. If not, see .
import os
import subprocess
import codecs
+import contextlib
import urllib2
import zipfile
import tempfile
import shutil
import sys
from StringIO import StringIO
from sitescripts.utils import get_config
@@ -31,53 +32,63 @@
! Last modified: %timestamp%
! Expires: 1d
!'''
MALWAREDOMAINS_PATH = '/files/justdomains.zip'
def try_mirror(mirror):
+ url = mirror + MALWAREDOMAINS_PATH
try:
- response = urllib2.urlopen(mirror + MALWAREDOMAINS_PATH)
- return response.read()
- except urllib2.HTTPError:
- return None
+ with contextlib.closing(urllib2.urlopen(url)) as response:
+ return None, response.read()
+ except urllib2.HTTPError as exc:
+ exc.close()
+ except urllib2.URLError as exc:
+ pass
+ return 'Failed to fetch {}: {}'.format(url, exc), None
-if __name__ == '__main__':
+def main():
config = get_config()
section = 'subscriptionDownloads'
repository = config.get(section, 'malwaredomains_repository')
mirrors = config.get(section, 'malwaredomains_mirrors').split()
tempdir = tempfile.mkdtemp(prefix='malwaredomains')
try:
subprocess.check_call(['hg', '-q', 'clone', '-U', repository, tempdir])
subprocess.check_call(['hg', '-q', 'up', '-R', tempdir, '-r', 'default'])
path = os.path.join(tempdir, 'malwaredomains_full.txt')
file = codecs.open(path, 'wb', encoding='utf-8')
print >>file, FILTERLIST_HEADER
+ error_report = ['Unable to fetch malware domains list', 'Errors:']
for mirror in mirrors:
- data = try_mirror(mirror)
+ error_message, data = try_mirror(mirror)
if data is not None:
break
+ error_report.append(error_message)
else:
- sys.exit('Unable to fetch malware domains list.')
+ sys.exit('\n'.join(error_report))
- zip = zipfile.ZipFile(StringIO(data), 'r')
- info = zip.infolist()[0]
- for line in str(zip.read(info.filename)).splitlines():
+ zf = zipfile.ZipFile(StringIO(data), 'r')
+ info = zf.infolist()[0]
+ for line in str(zf.read(info.filename)).splitlines():
domain = line.strip()
if not domain:
continue
print >>file, '||%s^' % domain.decode('idna')
file.close()
if subprocess.check_output(['hg', 'stat', '-R', tempdir]) != '':
subprocess.check_call(['hg', '-q', 'commit', '-R', tempdir, '-A', '-u', 'hgbot', '-m', 'Updated malwaredomains.com data'])
subprocess.check_call(['hg', '-q', 'push', '-R', tempdir])
finally:
shutil.rmtree(tempdir, ignore_errors=True)
+
+
+if __name__ == '__main__':
+ main()
Index: sitescripts/subscriptions/test/test_updateMalwareDomainsList.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/sitescripts/subscriptions/test/test_updateMalwareDomainsList.py
@@ -0,0 +1,99 @@
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2016 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus. If not, see .
+
+import io
+import ConfigParser
+import subprocess
+import urllib2
+import zipfile
+
+import pytest
+
+from sitescripts.subscriptions.bin.updateMalwareDomainsList import main
+
+CONF_SECTION = 'subscriptionDownloads'
+
+
+@pytest.fixture(autouse=True)
+def md_repo(tmpdir):
+ """A mock of our malware domains repo"""
+ repo_dir = tmpdir.mkdir('md_repo')
+ repo_dir.join('foo').write('foo')
+ subprocess.check_call(['hg', 'init'], cwd=repo_dir.strpath)
+ subprocess.check_call(['hg', 'commit', '-q', '-m', 'foo', '-A'],
+ cwd=repo_dir.strpath)
+ return repo_dir
+
+
+@pytest.fixture(autouse=True)
+def config(mocker, md_repo):
+ """A mock of sitescripts config"""
+ config = ConfigParser.ConfigParser()
+ config.add_section(CONF_SECTION)
+ config.set(CONF_SECTION, 'malwaredomains_repository', md_repo.strpath)
+ config.set(CONF_SECTION, 'malwaredomains_mirrors', 'good')
+ module = 'sitescripts.subscriptions.bin.updateMalwareDomainsList'
+ mocker.patch(module + '.get_config', lambda: config)
+ return config
+
+
+@pytest.fixture(autouse=True)
+def urlopen(mocker):
+ """Mock urlopen function"""
+ real_urlopen = urllib2.urlopen
+
+ def mock_urlopen(url):
+ if url.startswith('good'):
+ zf_data = io.BytesIO()
+ with zipfile.ZipFile(zf_data, 'w') as zf:
+ zf.writestr('justdomains', 'success\n')
+ return io.BytesIO(zf_data.getvalue())
+ if url.startswith('bad'):
+ raise urllib2.HTTPError('Bad', '42', 'No good', [], None)
+ if url.startswith('ugly'):
+ raise urllib2.URLError('Ugly')
+ return real_urlopen(url)
+
+ mocker.patch('urllib2.urlopen', mock_urlopen)
+
+
+def test_good(md_repo):
+ main()
+ subprocess.check_call(['hg', 'up'], cwd=md_repo.strpath)
+ result = md_repo.join('malwaredomains_full.txt').read()
+ assert 'success' in result
+
+
+def test_bad(md_repo, config):
+ config.set(CONF_SECTION, 'malwaredomains_mirrors', 'bad')
+ try:
+ main()
+ except SystemExit as exc:
+ err_lines = str(exc).splitlines()
+ assert len(err_lines) == 3
+ assert 'Failed to fetch bad/files/justdomains.zip' in err_lines[2]
+ assert '42: No good' in err_lines[2]
+
+
+def test_ugly(md_repo, config):
+ config.set(CONF_SECTION, 'malwaredomains_mirrors', 'bad\nugly')
+ try:
+ main()
+ except SystemExit as exc:
+ err_lines = str(exc).splitlines()
+ assert len(err_lines) == 4
+ assert 'Failed to fetch bad/files/justdomains.zip' in err_lines[2]
+ assert 'Failed to fetch ugly/files/justdomains.zip' in err_lines[3]
+ assert 'Ugly' in err_lines[3]
Index: tox.ini
===================================================================
--- a/tox.ini
+++ b/tox.ini
@@ -99,10 +99,11 @@
'' .sitescripts.test --write
python ensure_dependencies.py
py.test \
--cov-config tox.ini --cov-report term --cov sitescripts \
sitescripts/hg/test \
sitescripts/notifications/test \
sitescripts/stats/test \
sitescripts/formmail/test \
- sitescripts/extensions/test
+ sitescripts/extensions/test \
+ sitescripts/subscriptions/test
flake8 sitescripts multiplexer.py multiplexer.fcgi