Issue 6950 - Don't include headers and metadata from includes into output

abp/filters/renderer.py

 # This file is part of Adblock Plus <https://adblockplus.org/>,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
@@ skipping 71 matching lines @@
 
             try:
                 included, inherited_source = _get_and_parse_fragment(
                     name, sources, default_source, include_stack)
                 all_included = _process_includes(
                     sources, inherited_source, include_stack, included)
 
                 _logger.info('- including: %s', name)
                 yield Comment('*** {} ***'.format(name))
                 for line in all_included:
-                    yield line
+                    if line.type not in {'header', 'metadata'}:
+                        yield line
             except (NotFound, ValueError) as exc:
                 raise IncludeError(exc, include_stack)
         else:
             yield line
 
 
 def _process_timestamps(lines):
     """Convert timestamp markers into actual timestamps."""
     for line in lines:
-        if line.type == 'comment' and '%timestamp%' in line.text:
+        if line.type == 'metadata' and line.value == '%timestamp%':

[Vasily Kuznetsov, 2018/09/14 15:09:35]

This function now looks better as the result of our greater tolerance to funny
metadata keys.

             timestamp = time.strftime('%d %b %Y %H:%M UTC', time.gmtime())
-            yield Comment(text=line.text.replace('%timestamp%', timestamp))
+            yield Metadata(line.key, timestamp)
         else:
             yield line
 
 
 def _first_and_rest(iterable):
     """Return the first item from the iterable and the rest as an iterator."""
     iterator = iter(iterable)
     first_item = next(iterator)
     return first_item, iterator
 
 
 def _insert_version(lines):
     """Insert metadata comment with version (a.k.a. date)."""
     first_line, rest = _first_and_rest(lines)
     version = Metadata('Version', time.strftime('%Y%m%d%H%M', time.gmtime()))
     return itertools.chain([first_line, version], rest)
 
 
-def _remove_duplicates(lines):
-    """Remove duplicate metadata and headers."""
-    # Always remove checksum -- a checksum coming from a fragment
-    # will not match for the rendered list.
-    seen = {'checksum'}
-    for i, line in enumerate(lines):
-        if line.type == 'metadata':
-            key = line.key.lower()
-            if key not in seen:
-                seen.add(key)
-                yield line
-        elif line.type == 'header':
-            if i == 0:
-                yield line
-        else:
+def _remove_checksum(lines):
+    """Remove metadata comments giving a checksum.
+
+    Adblock Plus is no longer verifying checksums, so we don't have to
+    calculate the checksum for the resulting filter list. But we have
+    to strip them for compatibility with older versions of Adblock Plus
+    and other ad blockers which might still verify a checksum if given.
+    """
+    for line in lines:
+        if line.type != 'metadata' or line.key.lower() != 'checksum':
             yield line
 
 
 def _validate(lines):
     """Validate the final list."""
     first_line, rest = _first_and_rest(lines)
     if first_line.type != 'header':
         raise MissingHeader('No header found at the beginning of the input.')
     return itertools.chain([first_line], rest)
 
@@ skipping 22 matching lines @@
     ParseError
         When any of the fragments contain lines that can't be parsed.
     MissingHeader
         If the top level fragment doesn't start with a valid header. This would
         lead to rendering an invalid filter list, so we immediately abort.
 
     """
     _logger.info('Rendering: %s', name)
     lines, default_source = _get_and_parse_fragment(name, sources, top_source)
     lines = _process_includes(sources, default_source, [name], lines)
-    for proc in [_process_timestamps, _insert_version, _remove_duplicates,
+    for proc in [_process_timestamps, _insert_version, _remove_checksum,
                  _validate]:
         lines = proc(lines)
     return lines
 
 
 def _split_list_for_diff(list_in):
     """Split a filter list into metadata and rules."""
     metadata = {}
     rules = set()
     for line in parse_filterlist(list_in):
@@ skipping 27 matching lines @@
     for key, latest in latest_metadata.items():
         base = base_metadata.get(key)
         if not base or base.value != latest.value:
             yield latest.to_string()
     for key in set(base_metadata) - set(latest_metadata):
         yield '! {}:'.format(base_metadata[key].key)
     for rule in base_rules - latest_rules:
         yield '- {}'.format(rule)
     for rule in latest_rules - base_rules:
         yield '+ {}'.format(rule)