Issue 5325 - Add support for separator characters

lib/abp2blocklist.js

 /*
  * This file is part of Adblock Plus <https://adblockplus.org/>,
  * Copyright (C) 2006-2017 eyeo GmbH
  *
  * Adblock Plus is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * Adblock Plus is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ skipping 58 matching lines @@
   let subdomains = [];
   let suffixLength = domain.length + 1;
 
   for (let name of list)
   {
     if (name.length > suffixLength && name.slice(-suffixLength) == "." + domain)
       subdomains.push(name.slice(0, -suffixLength));
   }
 
   return subdomains;
+}
+
+function extractFilterDomains(filters)
+{
+  let domains = new Set();
+  for (let filter of filters)
+  {
+    let parsed = parseFilterRegexpSource(filter.regexpSource);
+    if (parsed.justHostname)
+      domains.add(parsed.hostname);
+  }
+  return domains;
 }
 
 function convertElemHideFilter(filter, elemhideSelectorExceptions)
 {
   let included = [];
   let excluded = [];
   let rules = [];
 
   parseDomains(filter.domains, included, excluded);
 
@@ skipping 13 matching lines @@
  *                   case, a hostname string (or undefined) and a bool
  *                   indicating if the source only contains a hostname or not:
  *                     {regexp: "...",
  *                      canSafelyMatchAsLowercase: true/false,
  *                      hostname: "...",
  *                      justHostname: true/false}
  */
 function parseFilterRegexpSource(text)
 {
   let regexp = [];
-  let lastIndex = text.length - 1;
+
+  // Convert the text into an array of Unicode characters.
+  //
+  // In the case of surrogate pairs (the smiley emoji, for example), one
+  // Unicode code point is represented by two JavaScript characters together.
+  // We want to iterate over Unicode code points rather than JavaScript
+  // characters.
+  let characters = Array.from(text);
+
+  let lastIndex = characters.length - 1;
   let hostname;
   let hostnameStart = null;
   let hostnameFinished = false;
   let justHostname = false;
   let canSafelyMatchAsLowercase = false;
 
-  for (let i = 0; i < text.length; i++)
-  {
-    let c = text[i];
+  for (let i = 0; i < characters.length; i++)
+  {
+    let c = characters[i];
 
     if (hostnameFinished)
       justHostname = false;
 
     // If we're currently inside the hostname we have to be careful not to
     // escape any characters until after we have converted it to punycode.
     if (hostnameStart != null && !hostnameFinished)
     {
       let endingChar = (c == "*" || c == "^" ||
                         c == "?" || c == "/" || c == "|");
       if (!endingChar && i != lastIndex)
         continue;
 
       hostname = punycode.toASCII(
-        text.substring(hostnameStart, endingChar ? i : i + 1)
+        characters.slice(hostnameStart, endingChar ? i : i + 1).join("")
+                  .toLowerCase()
       );
       hostnameFinished = justHostname = true;
       regexp.push(escapeRegExp(hostname));
       if (!endingChar)
         break;
     }
 
     switch (c)
     {
       case "*":
-        if (regexp.length > 0 && i < lastIndex && text[i + 1] != "*")
+        if (regexp.length > 0 && i < lastIndex && characters[i + 1] != "*")
           regexp.push(".*");
         break;
       case "^":
-        let separator = "[^-_.%" + (justHostname ? "" : "A-Z") + "a-z0-9]";

[kzar, 2017/07/07 12:46:18]

I don't understand the logic with justHostname here, could you explain?

[kzar, 2017/07/07 12:46:18]

Like I mentioned in the issue shouldn't the period be escaped?

[Manish Jethani, 2017/07/08 06:29:10]

For filters like "example.co^" and "example.co^*^hello^, when the code
encounters the first "^", justHostname is true. This means we're still in the
hostname part, therefore there's no need to include the "A-Z", as hostnames are
always lower case. Safari actually Punycode-encodes the hostname part before
running the regular expression on it.

We could just leave the "A-Z" there, it wouldn't hurt, but it's cleaner this
way, especially because if the filter contains only the hostname part then the
following code will covert the entire regular expression to lower case thus
leaving us with "a-za-z".

[Manish Jethani, 2017/07/08 06:29:10]

I replied there as well. No, the period must not be escaped when it's inside
square brackets. Within square brackets it has its literal meaning.

[kzar, 2017/07/10 12:39:56]

Ah OK, thanks. Would you mind adding a comment to explain that?

[kzar, 2017/07/10 12:39:56]

Acknowledged.

[Manish Jethani, 2017/07/11 17:10:33]

Done.

I've also made the code a bit more verbose here so it's easier to understand
what "separator" means.

[kzar, 2017/07/12 09:16:31]

Thanks, while a little verbose like you mention I think it's a lot easier to
understand now.

+        let alphabet = "a-z";
+        // If justHostname is true and we've encountered a "^", it means we're
+        // still in the hostname part of the URL. Since hostnames are always
+        // lower case (Punycode), there's no need to include "A-Z" in the
+        // pattern. Further, subsequent code may lower-case the entire regular
+        // expression (if the URL contains only the hostname part), leaving us
+        // with "a-za-z", which would be redundant.
+        if (!justHostname)
+          alphabet = "A-Z" + alphabet;
+        let digits = "0-9";
+        // Note that the "-" must appear first here in order to retain its
+        // literal meaning within the brackets.
+        let specialCharacters = "-_.%";
+        let separator = "[^" + specialCharacters + alphabet + digits + "]";
         if (i == 0)

[Manish Jethani, 2017/06/19 10:39:54]

Support separators at the beginning of the filter text.

This is in fact given as an example in the filters documentation, we need to
support this case.

[kzar, 2017/07/07 12:46:18]

Acknowledged.

           regexp.push("^https?://(.*" + separator + ")?");
         else if (i == lastIndex)
           regexp.push("(" + separator + ".*)?$");
         else
           regexp.push(separator);
         break;
       case "|":
         if (i == 0)
         {
           regexp.push("^");
           break;
         }
         if (i == lastIndex)
         {
           regexp.push("$");
           break;
         }
-        if (i == 1 && text[0] == "|")
+        if (i == 1 && characters[0] == "|")
         {
           hostnameStart = i + 1;
           canSafelyMatchAsLowercase = true;
           regexp.push("https?://([^/]+\\.)?");
           break;
         }
         regexp.push("\\|");
         break;
       case "/":
         if (!hostnameFinished &&
-            text.charAt(i-2) == ":" && text.charAt(i-1) == "/")
+            characters[i - 2] == ":" && characters[i - 1] == "/")
         {
           hostnameStart = i + 1;
           canSafelyMatchAsLowercase = true;
         }
         regexp.push("/");
         break;
       case ".": case "+": case "$": case "?":
       case "{": case "}": case "(": case ")":
       case "[": case "]": case "\\":
         regexp.push("\\", c);
         break;
       default:
         if (hostnameFinished && (c >= "a" && c <= "z" ||
                                  c >= "A" && c <= "Z"))
           canSafelyMatchAsLowercase = false;
-        regexp.push(c);
+        regexp.push(c == "%" ? c : encodeURI(c));
     }
   }
 
   return {
     regexp: regexp.join(""),
     canSafelyMatchAsLowercase: canSafelyMatchAsLowercase,
     hostname: hostname,
     justHostname: justHostname
   };
 }
@@ skipping 66 matching lines @@
   let included = [];
   let excluded = [];
 
   parseDomains(filter.domains, included, excluded);
 
   if (exceptionDomains)
     excluded = excluded.concat(exceptionDomains);
 
   if (withResourceTypes)
   {
-    trigger["resource-type"] = getResourceTypes(filter);
-
-    if (trigger["resource-type"].length == 0)
+    let resourceTypes = getResourceTypes(filter);
+
+    // Content blocker rules can't differentiate between sub-document requests
+    // (iframes) and top-level document requests. To avoid too many false
+    // positives, we prevent rules with no hostname part from blocking document
+    // requests.
+    //
+    // Once Safari 11 becomes our minimum supported version, we could change
+    // our approach here to use the new "unless-top-url" property instead.
+    if (filter instanceof filterClasses.BlockingFilter && !parsed.hostname)
+      resourceTypes = resourceTypes.filter(type => type != "document");
+
+    if (resourceTypes.length == 0)
       return;
+
+    trigger["resource-type"] = resourceTypes;
   }
 
   if (filter.thirdParty != null)
     trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"];
 
   if (included.length > 0)
   {
     trigger["if-domain"] = [];
 
     for (let name of included)
@@ skipping 15 matching lines @@
       {
         trigger["if-domain"].push("*" + name);
       }
     }
   }
   else if (excluded.length > 0)
   {
     trigger["unless-domain"] = excluded.map(name => "*" + name);
   }
   else if (filter instanceof filterClasses.BlockingFilter &&
-           filter.contentType & typeMap.SUBDOCUMENT)
-  {
+           filter.contentType & typeMap.SUBDOCUMENT && parsed.hostname)
+  {
+    // Rules with a hostname part are still allowed to block document requests,
+    // but we add an exception for top-level documents.
+    //
+    // Note that we can only do this if there's no "unless-domain" property for
+    // now. This also only works in Safari 11 onwards, while older versions
+    // simply ignore this property. Once Safari 11 becomes our minimum
+    // supported version, we can merge "unless-domain" into "unless-top-url".
     trigger["unless-top-url"] = [trigger["url-filter"]];
     if (trigger["url-filter-is-case-sensitive"])
       trigger["top-url-filter-is-case-sensitive"] = true;
   }
 
   rules.push({trigger: trigger, action: {type: action}});
-}
-
-function hasNonASCI(obj)
-{
-  if (typeof obj == "string")
-  {
-    if (/[^\x00-\x7F]/.test(obj))
-      return true;
-  }
-
-  if (typeof obj == "object")
-  {
-    if (obj instanceof Array)
-      for (let item of obj)
-        if (hasNonASCI(item))
-          return true;
-
-    let names = Object.getOwnPropertyNames(obj);
-    for (let name of names)
-      if (hasNonASCI(obj[name]))
-        return true;
-  }
-
-  return false;
 }
 
 function convertIDSelectorsToAttributeSelectors(selector)
 {
   // First we figure out where all the IDs are
   let sep = "";
   let start = null;
   let positions = [];
   for (let i = 0; i < selector.length; i++)
   {
@@ skipping 33 matching lines @@
   {
     newSelector.push(selector.substring(i, pos.start));
     newSelector.push('[id=', selector.substring(pos.start + 1, pos.end), ']');
     i = pos.end;
   }
   newSelector.push(selector.substring(i));
 
   return newSelector.join("");
 }
 
-function addCSSRules(rules, selectors, matchDomain)
-{
+function addCSSRules(rules, selectors, matchDomain, exceptionDomains)
+{
+  let unlessDomain = exceptionDomains.size > 0 ? [] : null;
+
+  exceptionDomains.forEach(name => unlessDomain.push("*" + name));
+
   while (selectors.length)
   {
     let selector = selectors.splice(0, selectorLimit).join(", ");
 
     // As of Safari 9.0 element IDs are matched as lowercase. We work around
     // this by converting to the attribute format [id="elementID"]
     selector = convertIDSelectorsToAttributeSelectors(selector);
 
-    rules.push({
+    let rule = {
       trigger: {"url-filter": matchDomain,
                 "url-filter-is-case-sensitive": true},
       action: {type: "css-display-none",
                selector: selector}
-    });
+    };
+
+    if (unlessDomain)
+      rule.trigger["unless-domain"] = unlessDomain;
+
+    rules.push(rule);
   }
 }
 
 let ContentBlockerList =
 /**
  * Create a new Adblock Plus filter to content blocker list converter
  *
  * @constructor
  */
 exports.ContentBlockerList = function ()
@@ skipping 76 matching lines @@
     {
       for (let matchDomain of result.matchDomains)
       {
         let group = groupedElemhideFilters.get(matchDomain) || [];
         group.push(result.selector);
         groupedElemhideFilters.set(matchDomain, group);
       }
     }
   }
 
-  addCSSRules(rules, genericSelectors, "^https?://");
-
-  // Right after the generic element hiding filters, add the exceptions that
-  // should apply only to those filters.
-  for (let filter of this.generichideExceptions)
-    convertFilterAddRules(rules, filter, "ignore-previous-rules", false);
+  // Separate out the element hiding exceptions that have only a hostname part
+  // from the rest. This allows us to implement a workaround for issue #5345
+  // (WebKit bug #167423), but as a bonus it also reduces the number of
+  // generated rules. The downside is that the exception will only apply to the
+  // top-level document, not to iframes. We have to live with this until the
+  // WebKit bug is fixed in all supported versions of Safari.
+  // https://bugs.webkit.org/show_bug.cgi?id=167423
+  //
+  // Note that as a result of this workaround we end up with a huge rule set in
+  // terms of the amount of memory used. This can cause Node.js to throw
+  // "JavaScript heap out of memory". To avoid this, call Node.js with
+  // --max_old_space_size=4096
+  let elemhideExceptionDomains = extractFilterDomains(this.elemhideExceptions);
+
+  let genericSelectorExceptionDomains =
+    extractFilterDomains(this.generichideExceptions);
+  elemhideExceptionDomains.forEach(name =>
+  {
+    genericSelectorExceptionDomains.add(name);
+  });
+
+  addCSSRules(rules, genericSelectors, "^https?://",
+              genericSelectorExceptionDomains);
 
   groupedElemhideFilters.forEach((selectors, matchDomain) =>
   {
-    addCSSRules(rules, selectors, matchDomain);
+    addCSSRules(rules, selectors, matchDomain, elemhideExceptionDomains);
   });
-
-  for (let filter of this.elemhideExceptions)
-    convertFilterAddRules(rules, filter, "ignore-previous-rules", false);
 
   let requestFilterExceptionDomains = [];
   for (let filter of this.genericblockExceptions)
   {
     let parsed = parseFilterRegexpSource(filter.regexpSource);
     if (parsed.hostname)
       requestFilterExceptionDomains.push(parsed.hostname);
   }
 
   for (let filter of this.requestFilters)
   {
     convertFilterAddRules(rules, filter, "block", true,
                           requestFilterExceptionDomains);
   }
 
   for (let filter of this.requestExceptions)
     convertFilterAddRules(rules, filter, "ignore-previous-rules", true);
 
-  return rules.filter(rule => !hasNonASCI(rule));
+  return rules;
 };