Issue 3176 - Add metadata to content blocker lists

sitescripts/content_blocker_lists/bin/generate_lists.py

 #!/usr/bin/env python
 # coding: utf-8
 
 # This file is part of Adblock Plus <https://adblockplus.org/>,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
+from contextlib import closing
+from datetime import datetime
+import json
 import os
 import subprocess
+import re
 import urllib2
 
 from sitescripts.utils import get_config
 
-def _update_abp2blocklist():
+config = dict(get_config().items("content_blocker_lists"))
+
+def update_abp2blocklist():
   with open(os.devnull, "w") as devnull:
-    config = get_config()
-    abp2blocklist_path = config.get("content_blocker_lists",
-                                    "abp2blocklist_path")
+    abp2blocklist_path = config["abp2blocklist_path"]
     if os.path.isdir(abp2blocklist_path):
         subprocess.check_call(("hg", "pull", "-u", "-R", abp2blocklist_path),
                               stdout=devnull)
     else:
-      abp2blocklist_url = config.get("content_blocker_lists",
-                                     "abp2blocklist_url")
-      subprocess.check_call(("hg", "clone", abp2blocklist_url,
+      subprocess.check_call(("hg", "clone", config["abp2blocklist_url"],
                              abp2blocklist_path), stdout=devnull)
     subprocess.check_call(("npm", "install"), cwd=abp2blocklist_path,
                           stdout=devnull)
 
-def _download(url_key):
-  url = get_config().get("content_blocker_lists", url_key)
-  response = urllib2.urlopen(url)
+def download_filter_list(url):
+  filter_list = {}
+  with closing(urllib2.urlopen(url)) as response:
+    filter_list["body"] = response.read()
+  filter_list["header"] = parse_filter_list_header(filter_list["body"])
+  filter_list["header"]["url"] = url
+  return filter_list
+
+def parse_filter_list_header(filter_list):
+  body_start = re.search(r"^[^![]", filter_list, re.MULTILINE).start()
+  field_re = re.compile(r"^!\s*([^:\s]+):\s*(.+)$", re.MULTILINE)
+  return { match.group(1): match.group(2)
+           for match in field_re.finditer(filter_list, 0, body_start) }
+
+def generate_metadata(filter_lists, expires):
+  metadata = {

[Sebastian Noack, 2015/11/30 16:00:07]

Nit: Use an OrderedDict to make sure that the metadata are at the top, to make
the file easier to read for humans.

[kzar, 2015/11/30 17:06:00]

Done.

+    "sources": [],
+    "version": datetime.utcnow().strftime("%Y%m%d%H%M"),
+    "expires": expires
+  }
+  for filter_list in filter_lists:
+    metadata["sources"].append({ k.lower(): filter_list["header"][k]
+                                 for k in ["url", "Version"]})
+  return metadata
+
+def write_block_list(filter_lists, path, expires):
+  block_list = generate_metadata(filter_lists, expires)
+  process = subprocess.Popen(("node", "abp2blocklist.js"),
+                             cwd=config["abp2blocklist_path"],
+                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
   try:
-    return response.read()
+    for filter_list in filter_lists:
+      print >>process.stdin, filter_list["body"]
+    block_list["rules"] = json.loads(process.communicate()[0])

[Sebastian Noack, 2015/11/30 15:49:19]

Yes, this works as abp2blocklist (currently) reads all input before generating
any output. However, that assumption is unsafe. And if that ever changes the
print call above will hang forever while waiting for the subprocess's stdin's
write buffer becoming empty again.

The solution: Use a thread to write to process.stdin. And while on it use
json.load to lazily read from process.stdout.

[kzar, 2015/11/30 17:06:00]

I've not done much threaded programming in Python before, this look OK?

   finally:
-    response.close()
-
-def _convert_filter_list(sources, destination_path_key):
-  config = get_config()
-  destination_path = config.get("content_blocker_lists", destination_path_key)
-  with open(destination_path, "wb") as destination_file:
-    abp2blocklist_path = config.get("content_blocker_lists",
-                                    "abp2blocklist_path")
-    process = subprocess.Popen(("node", "abp2blocklist.js"),
-                               cwd=abp2blocklist_path, stdin=subprocess.PIPE,
-                               stdout=destination_file)
-    try:
-      for source in sources:
-        print >>process.stdin, source
-    finally:
-      process.stdin.close()
-      process.wait()
-
+    process.stdin.close()
+    process.wait()
   if process.returncode:
     raise Exception("abp2blocklist returned %s" % process.returncode)
+  with open(path, "wb") as destination_file:
+    json.dump(block_list, destination_file,
+              sort_keys=True, indent=2, separators=(",", ": "))
 
 if __name__ == "__main__":
-  _update_abp2blocklist()
+  update_abp2blocklist()
 
-  easylist = _download("easylist_url")
-  exceptionrules = _download("exceptionrules_url")
+  easylist = download_filter_list(config["easylist_url"])
+  exceptionrules = download_filter_list(config["exceptionrules_url"])
 
-  _convert_filter_list([easylist], "easylist_content_blocker_path")
-  _convert_filter_list([easylist, exceptionrules],
-                       "combined_content_blocker_path")
+  write_block_list([easylist],
+                   config["easylist_content_blocker_path"],
+                   config["easylist_content_blocker_expires"])
+  write_block_list([easylist, exceptionrules],
+                   config["combined_content_blocker_path"],
+                   config["combined_content_blocker_expires"])