Issue 3956 - Convert domain whitelisting filters

lib/abp2blocklist.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-2016 Eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 /** @module abp2blocklist */
 
 "use strict";
 
 let filterClasses = require("filterClasses");
 let tldjs = require("tldjs");
 let punycode = require("punycode");
 
 const selectorLimit = 5000;
 const typeMap = filterClasses.RegExpFilter.typeMap;
+const whitelistableRequestTypes = (typeMap.IMAGE
+                                   | typeMap.STYLESHEET
+                                   | typeMap.SCRIPT
+                                   | typeMap.FONT
+                                   | typeMap.MEDIA
+                                   | typeMap.POPUP
+                                   | typeMap.OBJECT
+                                   | typeMap.OBJECT_SUBREQUEST
+                                   | typeMap.XMLHTTPREQUEST
+                                   | typeMap.PING
+                                   | typeMap.SUBDOCUMENT
+                                   | typeMap.OTHER);
 
 function parseDomains(domains, included, excluded)
 {
   for (let domain in domains)
   {
     if (domain != "")
     {
       let enabled = domains[domain];
       domain = punycode.toASCII(domain.toLowerCase());
 
@@ skipping 173 matching lines @@
   {
     result.push(domain);
 
     if (tldjs.getDomain(domain) == domain)
       result.push("www." + domain);
   }
 
   return result;
 }
 
-function convertFilter(filter, action, withResourceTypes)
+function convertFilterAddRules(rules, filter, action, withResourceTypes)
 {
   let parsed = parseFilterRegexpSource(filter.regexpSource);
 
   // For the special case of $document whitelisting filters with just a domain
   // we can generate an equivalent blocking rule exception using if-domain.
-  if (filter.contentType == typeMap.DOCUMENT && parsed.justHostname)

[Sebastian Noack, 2016/05/12 12:12:25]

For filters like example.com$document,image we would have to generate both, one
rule with if-domain and one with url-filter.

[kzar, 2016/05/16 16:22:36]

Done.

-    return {trigger: {"url-filter": ".*",

[Sebastian Noack, 2016/05/12 12:12:26]

Nit: Mind wrapping the nested object for better readability?

[Sebastian Noack, 2016/05/12 12:12:26]

Wouldn't an empty string be sufficient as url-filter to match all URLs?

[kzar, 2016/05/16 16:22:36]

Done.

[kzar, 2016/05/16 16:22:36]

Unfortunately this causes a "Extension compilation failed: Invalid url-filter
object." exception.

-            "if-domain": addDomainPrefix([parsed.hostname])},
-            action: {type: "ignore-previous-rules"}};
+  if (filter instanceof filterClasses.WhitelistFilter &&
+      filter.contentType & typeMap.DOCUMENT &&
+      parsed.justHostname)
+  {
+    rules.push({
+      trigger: {
+        "url-filter": ".*",
+        "if-domain": addDomainPrefix([parsed.hostname])
+      },
+      action: {type: "ignore-previous-rules"}
+    });
+    // If the filter contains other supported options we'll need to generate
+    // further rules for it, but if not we can simply return now.
+    if (!(filter.contentType | whitelistableRequestTypes))
+      return;
+  }
 
   let trigger = {"url-filter": parsed.regexp};
 
-  // Limit rules to to HTTP(S) URLs

[Sebastian Noack, 2016/05/12 12:12:26]

Typo: to to

[kzar, 2016/05/16 16:22:36]

Done.

+  // Limit rules to HTTP(S) URLs
   if (!/^(\^|http)/i.test(trigger["url-filter"]))
     trigger["url-filter"] = "^https?://.*" + trigger["url-filter"];
 
   // For rules containing only a hostname we know that we're matching against
   // a lowercase string unless the matchCase option was passed.
   if (parsed.canSafelyMatchAsLowercase && !filter.matchCase)
     trigger["url-filter"] = trigger["url-filter"].toLowerCase();
 
   if (parsed.canSafelyMatchAsLowercase || filter.matchCase)
     trigger["url-filter-is-case-sensitive"] = true;
 
   let included = [];
   let excluded = [];
 
   parseDomains(filter.domains, included, excluded);
 
   if (withResourceTypes)
     trigger["resource-type"] = getResourceTypes(filter);
   if (filter.thirdParty != null)
     trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"];
 
   if (included.length > 0)
     trigger["if-domain"] = addDomainPrefix(included);
   else if (excluded.length > 0)
     trigger["unless-domain"] = addDomainPrefix(excluded);
 
-  return {trigger: trigger, action: {type: action}};
+  rules.push({trigger: trigger, action: {type: action}});
 }
 
 function hasNonASCI(obj)
 {
   if (typeof obj == "string")
   {
     if (/[^\x00-\x7F]/.test(obj))
       return true;
   }
 
@@ skipping 90 matching lines @@
     return;
   if (filter instanceof filterClasses.RegExpFilter &&
       filter.regexpSource == null)
     return;
 
   if (filter instanceof filterClasses.BlockingFilter)
     this.requestFilters.push(filter);
 
   if (filter instanceof filterClasses.WhitelistFilter)
   {
-    if (filter.contentType & (typeMap.DOCUMENT
-                              | typeMap.IMAGE
-                              | typeMap.STYLESHEET
-                              | typeMap.SCRIPT
-                              | typeMap.FONT
-                              | typeMap.MEDIA
-                              | typeMap.POPUP
-                              | typeMap.OBJECT
-                              | typeMap.OBJECT_SUBREQUEST
-                              | typeMap.XMLHTTPREQUEST
-                              | typeMap.PING
-                              | typeMap.SUBDOCUMENT
-                              | typeMap.OTHER))
+    if (filter.contentType & (typeMap.DOCUMENT | whitelistableRequestTypes))
       this.requestExceptions.push(filter);
 
       if (filter.contentType & typeMap.ELEMHIDE)
         this.elemhideExceptions.push(filter);
   }
 
   if (filter instanceof filterClasses.ElemHideFilter)
     this.elemhideFilters.push(filter);
 
   if (filter instanceof filterClasses.ElemHideException)
   {
     let domains = this.elemhideSelectorExceptions[filter.selector];
     if (!domains)
       domains = this.elemhideSelectorExceptions[filter.selector] = [];
 
     parseDomains(filter.domains, domains, []);
   }
 };
 
 /**
  * Generate content blocker list for all filters that were added
  *
  * @returns   {Filter}   filter    Filter to convert
  */
 ContentBlockerList.prototype.generateRules = function(filter)
 {
   let rules = [];
 
-  function addRule(rule)
-  {
-    if (!hasNonASCI(rule))
-      rules.push(rule);
-  }
-
   let groupedElemhideFilters = new Map();
   for (let filter of this.elemhideFilters)
   {
     let result = convertElemHideFilter(filter, this.elemhideSelectorExceptions);
     if (!result)
       continue;
 
     if (result.matchDomains.length == 0)
       result.matchDomains = ["^https?://"];
 
     for (let matchDomain of result.matchDomains)
     {
       let group = groupedElemhideFilters.get(matchDomain) || [];
       group.push(result.selector);
       groupedElemhideFilters.set(matchDomain, group);
     }
   }
 
   groupedElemhideFilters.forEach((selectors, matchDomain) =>
   {
     while (selectors.length)
     {
       let selector = selectors.splice(0, selectorLimit).join(", ");
 
       // As of Safari 9.0 element IDs are matched as lowercase. We work around
       // this by converting to the attribute format [id="elementID"]
       selector = convertIDSelectorsToAttributeSelectors(selector);
 
-      addRule({
+      rules.push({
         trigger: {"url-filter": matchDomain,
                   "url-filter-is-case-sensitive": true},
         action: {type: "css-display-none",
                  selector: selector}
       });
     }
   });
 
   for (let filter of this.elemhideExceptions)
-    addRule(convertFilter(filter, "ignore-previous-rules", false));
+    convertFilterAddRules(rules, filter, "ignore-previous-rules", false);
   for (let filter of this.requestFilters)
-    addRule(convertFilter(filter, "block", true));
+    convertFilterAddRules(rules, filter, "block", true);
   for (let filter of this.requestExceptions)
-    addRule(convertFilter(filter, "ignore-previous-rules", true));
-
-  return rules;
+    convertFilterAddRules(rules, filter, "ignore-previous-rules", true);
+
+  return rules.filter(rule => !hasNonASCI(rule));
 };