Index: README.md =================================================================== --- a/README.md +++ b/README.md @@ -86,6 +86,28 @@ You can clone the necessary repositories to a local directory and add `-i` options accordingly. +## Rendering diffs + +A diff allows a client running ad blocking software such as Adblock Plus to update +the filter lists incrementally, instead of downloading a new copy of a full list +during each update. This is meant to lessen the amount of resources used when updating +filter lists (e.g. network data, memory usage, battery consumption, etc.), allowing +clients to update their lists more frequently using less resources. + +Python-abp contains a script that produces the diff between two versions of a +filter list called `fldiff`: + + $ fldiff base.txt latest.txt output.txt + +This will produce a diff that shows how a client may get from `base.txt` to +`latest.txt`, and write the output to `output.txt`. The output argument is +optional. If ommitted, the data will be written to `stdout`. + +The script produces three types of lines, as specified in the [technical specification][5]: +* Special comments of the form `! :[ ]` +* Added filters of the form `+ ` +* Removed filter of the form `- ` + ## Library API Python-abp can also be used as a library for parsing filter lists. For example @@ -182,3 +204,4 @@ [2]: http://pytest.org/ [3]: https://tox.readthedocs.org/ [4]: https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt + [5]: https://docs.google.com/document/d/1SoEqaOBZRCfkh1s5Kds5A5RwUC_nqbYYlGH72sbsSgQ/ Index: abp/filters/diff_script.py =================================================================== new file mode 100644 --- /dev/null +++ b/abp/filters/diff_script.py @@ -0,0 +1,60 @@ +# This file is part of Adblock Plus , +# Copyright (C) 2006-present eyeo GmbH +# +# Adblock Plus is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# Adblock Plus is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Adblock Plus. If not, see . + +"""Command line script for rendering Adblock Plus filter list diffs.""" + +from __future__ import print_function + +import argparse +import io +import sys + +from .renderer import render_diff + +__all__ = ['main'] + + +def parse_args(): + parser = argparse.ArgumentParser(description='Render a filter list diff.') + parser.add_argument( + 'base', help='the older filter list that needs to be updated', + nargs='?') + parser.add_argument( + 'latest', help='the most recent version of the filter list', + nargs='?') + parser.add_argument( + 'outfile', help='output file for filter list diff', + default='-', nargs='?') + return parser.parse_args() + + +def main(): + """Entry point for the diff rendering script (fldiff).""" + args = parse_args() + + with io.open(args.base, 'r', encoding='utf-8') as base, \ + io.open(args.latest, 'r', encoding='utf-8') as latest: + + lines = render_diff(base, latest) + if args.outfile == '-': + outfile = io.open(sys.stdout.fileno(), 'w', + closefd=False, + encoding=sys.stdout.encoding or 'utf-8') + else: + outfile = io.open(args.outfile, 'w', encoding='utf-8') + + with outfile: + for line in lines: + print(line, file=outfile) Index: abp/filters/renderer.py =================================================================== --- a/abp/filters/renderer.py +++ b/abp/filters/renderer.py @@ -217,6 +217,9 @@ yield latest.to_string() for key in set(base_metadata) - set(latest_metadata): yield '! {}:'.format(base_metadata[key].key) + # The removed filters are listed first because, in case a filter is both + # removed and added, (and the client processes the diff in order), the + # filter will be added. for rule in base_rules - latest_rules: yield '- {}'.format(rule) for rule in latest_rules - base_rules: Index: setup.py =================================================================== --- a/setup.py +++ b/setup.py @@ -52,7 +52,8 @@ packages=['abp', 'abp.filters'], cmdclass={'devenv': DevEnvCommand}, entry_points={ - 'console_scripts': ['flrender=abp.filters.render_script:main'], + 'console_scripts': ['flrender=abp.filters.render_script:main', + 'fldiff=abp.filters.diff_script:main'], }, include_package_data=True, license='GPLv3', Index: tests/test_diff_script.py =================================================================== new file mode 100644 --- /dev/null +++ b/tests/test_diff_script.py @@ -0,0 +1,85 @@ +# This file is part of Adblock Plus , +# Copyright (C) 2006-present eyeo GmbH +# +# Adblock Plus is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# Adblock Plus is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Adblock Plus. If not, see . + +"""Functional tests for the diff script.""" + +from __future__ import unicode_literals + +import pytest +import subprocess +import io + +from test_differ import BASE, LATEST + + +@pytest.fixture +def rootdir(tmpdir): + """Directory with example filter lists.""" + rootdir = tmpdir.join('root') + rootdir.mkdir() + rootdir.join('base.txt').write_text(BASE, encoding='utf8') + rootdir.join('latest.txt').write_text(LATEST, encoding='utf8') + + return rootdir + + +@pytest.fixture +def dstfile(tmpdir): + """Destination file for saving the diff output.""" + return tmpdir.join('dst') + + +def run_script(*args, **kw): + """Run diff rendering script with given arguments and return its output.""" + cmd = ['fldiff'] + list(args) + + proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, + stdout=subprocess.PIPE, stdin=subprocess.PIPE, + **kw) + stdout, stderr = proc.communicate() + return proc.returncode, stderr.decode('utf-8'), stdout.decode('utf-8') + + +def test_diff_with_outfile(rootdir, dstfile): + run_script(str(rootdir.join('base.txt')), + str(rootdir.join('latest.txt')), + str(dstfile)) + with io.open(str(dstfile), encoding='utf-8') as dst: + result = dst.read() + assert '+ &ad_channel=\xa3' in result + + +def test_no_outfile(rootdir): + _, _, out = run_script(str(rootdir.join('base.txt')), + str(rootdir.join('latest.txt'))) + assert '[Adblock Plus Diff]' in out + + +def test_no_base_file(rootdir): + code, err, _ = run_script('wrong.txt', str(rootdir.join('latest.txt'))) + assert code == 1 + assert 'No such file or directory' in err + + +def test_no_latest_file(rootdir): + code, err, _ = run_script(str(rootdir.join('base.txt')), 'wrong.txt') + assert code == 1 + assert 'No such file or directory' in err + + +def test_diff_to_self(rootdir): + _, _, out = run_script(str(rootdir.join('latest.txt')), + str(rootdir.join('latest.txt'))) + assert out == '[Adblock Plus Diff]\n' Index: tests/test_differ.py =================================================================== --- a/tests/test_differ.py +++ b/tests/test_differ.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License # along with Adblock Plus. If not, see . +from __future__ import unicode_literals from abp.filters.renderer import render_diff @@ -53,7 +54,7 @@ ! *** easylist:easylist/easylist_general_block.txt *** &act=ads_ &ad_box_ -&ad_channel= +&ad_channel=\U000000a3 test ''' @@ -63,7 +64,7 @@ ! Expires: ! Version: 123 - &ad.vid=$~xmlhttprequest -+ &ad_channel= ++ &ad_channel=\U000000a3 '''