Issue 395 - Filter hits statistics backend

sitescripts/filterhits/geometrical_mean.py

Index: sitescripts/filterhits/geometrical_mean.py
diff --git a/sitescripts/filterhits/geometrical_mean.py b/sitescripts/filterhits/geometrical_mean.py
new file mode 100644
index 0000000000000000000000000000000000000000..eeb393388482ad8b36d93892a751543c447d9e2a
--- /dev/null
+++ b/sitescripts/filterhits/geometrical_mean.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2015 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import itertools
+from sitescripts.filterhits import db
+
+def filter_hits(data):
+  """
+  Generator that provides all filter hits for the given data,
+  in tuples like (filter, domain, hits, latest).
+  """
+  for filter, filter_data in data['filters'].iteritems():
+    domains = itertools.chain(filter_data.get("thirdParty", {}).iteritems(),
+                              filter_data.get("firstParty", {}).iteritems())
+    for domain, domain_data in domains:
+      yield (filter, domain, domain_data["hits"], domain_data["latest"] / 1000)
+
+def update_query(interval, filter, domain, hits, latest):
+  """
+  Function that takes the fields for a filter hit and returns them arranged
+  as the update SQL requires along with the SQL itself.
+  """
+  return (("""INSERT IGNORE INTO `filters`
+              (filter, sha1) VALUES (%s, UNHEX(SHA1(filter)))""",
+           filter),
+          ("""INSERT INTO `geometrical_mean`
+              (filter_sha1, domain, hits, timestamp)
+              VALUES (UNHEX(SHA1(%s)), %s, %s, FROM_UNIXTIME(%s))
+              ON DUPLICATE KEY UPDATE
+              hits = (
+                POW(hits, 1 - (UNIX_TIMESTAMP(VALUES(timestamp)) -
+                               UNIX_TIMESTAMP(timestamp)) / %s) *
+                POW(VALUES(hits), (UNIX_TIMESTAMP(VALUES(timestamp)) -
+                                   UNIX_TIMESTAMP(timestamp)) / %s)),
+              timestamp = VALUES(timestamp)""",
+           filter, domain, hits, int(latest), interval, interval))
+
+def update(interval, data):
+  """
+  Returns an iterator of all the SQL and parameters needed to
+  update the aggregations for the given data + interval in the database.
+  """
+  for fields in filter_hits(data):
+    for query in update_query(interval, *fields):
+      yield query