Issue 6685 - Offer incremental filter list downloads

abp/filters/fldiff.py

Index: abp/filters/fldiff.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/abp/filters/fldiff.py
@@ -0,0 +1,74 @@
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-present eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Generate a diff between two filter lists."""
+
+import re
+
+from .renderer import _insert_checksum
+
+
+def generate_diff(base, latest):
+    """Return a diff between two filter lists."""
+    # Collect the special comments
+    diff = '[Adblock Plus Diff 2.0]\n'
+    sp_comment = re.compile(r'!\s(.*):\s(.*)')
+    latest_fl = set()
+    latest_md = set()
+    latest_keys = set()
+    base_fl = set()
+    base_md = set()
+    base_keys = set()
+    for line in latest.splitlines():
+        m = sp_comment.match(line)
+        if m:
+            latest_md.add(line)
+            latest_keys.add(m.groups()[0])
+        elif not line.startswith('!'):
+            latest_fl.add(line.strip())
+
+    # Get the diff between the rest of the lines
+    for line in base.splitlines():
+        if not line.startswith('!'):
+            base_fl.add(line.strip())
+        else:
+            m = sp_comment.match(line)
+            if m:
+                base_md.add(line)
+                base_keys.add(m.groups()[0])
+    new_md = latest_md - base_md
+    removed_keys = base_keys - latest_keys
+    add_fl = latest_fl - base_fl
+    remove_fl = base_fl - latest_fl
+    for item in new_md:
+        diff += item + '\n'
+    for key in removed_keys:
+        # If a special comment has been removed, enter it with a blank value
+        # sp the client will set it back to the default value
+        diff += '! {}:\n'.format(key)
+    for item in add_fl:
+        diff += '+ {}\n'.format(item)
+    for item in remove_fl:
+        diff += '- {}\n'.format(item)
+
+    # Add the checksum
+    _insert_checksum(diff)
+    return diff
+
+
+def publish_diff(diff, outfile):
+    """Write the diff to the outfile."""
+    with open(outfile, 'w+') as outfile:
+        outfile.write(diff)