Issue 6685 - Offer incremental filter list downloads

abp/filters/renderer.py

 # This file is part of Adblock Plus <https://adblockplus.org/>,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 """Combine filter list fragments to produce filter lists."""
 
 from __future__ import unicode_literals
 
 import itertools
 import logging
 import time
 
 from .parser import parse_filterlist, Comment, Metadata
 from .sources import NotFound
 
-__all__ = ['IncludeError', 'MissingHeader', 'render_filterlist']
+__all__ = ['IncludeError', 'MissingHeader', 'render_filterlist', 'render_diff']
 
 _logger = logging.getLogger(__name__)
 
 
 class IncludeError(Exception):
     """Error in processing include instruction."""
 
     def __init__(self, error, stack):
         stack_str = ' from '.join(map("'{}'".format, reversed(stack)))
         if stack_str:
@@ skipping 79 matching lines @@
     """Insert metadata comment with version (a.k.a. date)."""
     first_line, rest = _first_and_rest(lines)
     version = Metadata('Version', time.strftime('%Y%m%d%H%M', time.gmtime()))
     return itertools.chain([first_line, version], rest)
 
 
 def _remove_duplicates(lines):
     """Remove duplicate metadata and headers."""
     # Always remove checksum -- a checksum coming from a fragment
     # will not match for the rendered list.
-    seen = {'Checksum'}
+    seen = {'checksum'}
     for i, line in enumerate(lines):
         if line.type == 'metadata':
-            if line.key not in seen:
-                seen.add(line.key)
+            key = line.key.lower()
+            if key not in seen:
+                seen.add(key)
                 yield line
         elif line.type == 'header':
             if i == 0:
                 yield line
         else:
             yield line
 
 
 def _validate(lines):
     """Validate the final list."""
@@ skipping 31 matching lines @@
         lead to rendering an invalid filter list, so we immediately abort.
 
     """
     _logger.info('Rendering: %s', name)
     lines, default_source = _get_and_parse_fragment(name, sources, top_source)
     lines = _process_includes(sources, default_source, [name], lines)
     for proc in [_process_timestamps, _insert_version, _remove_duplicates,
                  _validate]:
         lines = proc(lines)
     return lines
+
+
+def _split_list_for_diff(list_in):
+    """Split a filter list into metadata, keys, and rules."""
+    metadata = {}
+    rules = set()
+    for line in parse_filterlist(list_in):
+        if line.type == 'metadata':
+            metadata[line.key.lower()] = line
+        elif line.type == 'filter':
+            rules.add(line.to_string())
+    return metadata, rules
+
+
+def render_diff(base, latest):
+    """Return a diff between two filter lists.
+
+    Parameters
+    ----------
+    base : iterator of str
+        The base (old) list that we want to update to latest.
+    lastest : iterator  of str
+        The latest (most recent) list that we want to update to.
+
+    Returns
+    -------
+    iterable of str
+        A diff between two lists (https://issues.adblockplus.org/ticket/6685)
+
+    """
+    latest_md, latest_rules = _split_list_for_diff(latest)
+    base_md, base_rules = _split_list_for_diff(base)

[Sebastian Noack, 2018/08/29 23:46:26]

Nit: Is it necessary to abbreviate metadata to md in order to avoid wrapping? If
so that is fine, but otherwise, I think it reads better spelled out.

[rhowell, 2018/08/30 16:23:30]

Done.

+    add_fl = latest_rules - base_rules

[Sebastian Noack, 2018/08/29 23:46:25]

Nit: These expressions could be inlined below. This would also be consistent
with the second loop iterating over removed metadata keys.

[rhowell, 2018/08/30 16:23:30]

Done.

+    remove_fl = base_rules - latest_rules
+
+    yield '[Adblock Plus Diff]'
+    for key, latest in latest_md.items():
+        base = base_md.get(key)
+        if not base or base.value != latest.value:
+            yield latest.to_string()
+    for key in set(base_md) - set(latest_md):
+        yield '! {}:'.format(base_md[key].key)
+    for item in remove_fl:
+        yield '- {}'.format(item)
+    for item in add_fl:
+        yield '+ {}'.format(item)