Issue 6685 - Offer incremental filter list downloads

abp/create_diff.py

Index: abp/create_diff.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/abp/create_diff.py
@@ -0,0 +1,71 @@
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-present eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Generate a diff between two filter lists."""
+
+import re
+import difflib
+import hashlib
+import base64
+
+
+def generate_diff(base, latest):
+    """Return a diff between two filter lists."""
+    # Write the comments
+    diff = '[Adblock Plus Diff 2.0]\n'
+    sp_comment = re.compile(r'!\s(.*):\s(.*)')
+    a = []
+    b = []
+    ver = re.compile(r'(.*/)\d+(.txt)')
+    for line in latest.splitlines():
+        m = sp_comment.match(line)
+        if m:
+            if m.groups()[0] not in {'Checksum', 'Diff-URL'}:
+                diff += '! {}: {}\n'.format(m.groups()[0], m.groups()[1])
+            if m.groups()[0] == 'Version':
+                version = m.groups()[1]
+            if m.groups()[0] == 'Diff-URL':
+                url = ver.match(m.groups()[1])
+                diff += '! Diff-URL: {}{}{}\n'.format(url.groups()[0], version,
+                                                      url.groups()[1])
+        elif not line.startswith('!'):
+            a.append(line.strip())
+
+    # Get the diff between the rest of the lines
+    for line in base.splitlines():
+        if not line.startswith('!'):
+            b.append(line.strip())
+    diffy = difflib.ndiff(b, a)
+    for item in diffy:
+        if item.startswith('+ ') or item.startswith('- '):
+            diff += item
+            diff += '\n'
+
+    # Add the checksum
+    md5sum = hashlib.md5()
+    for line in diff.splitlines():
+        if not line.strip():
+            md5sum.update(line.encode('utf-8') + b'\n')
+    checksum = base64.b64encode(md5sum.digest()).rstrip(b'=')
+    signed_diff = (diff[:24] +
+                   '! Checksum: {}\n'.format(checksum.decode('utf-8')) +
+                   diff[24:])
+    return signed_diff
+
+
+def publish_diff(diff, outfile):
+    """Write the diff to the outfile."""
+    with open(outfile, 'w+') as outfile:
+        outfile.write(diff)