Issue 395 - Filter hits statistics backend

sitescripts/filterhits/bin/process_logs.py

Index: sitescripts/filterhits/bin/process_logs.py
diff --git a/sitescripts/filterhits/bin/process_logs.py b/sitescripts/filterhits/bin/process_logs.py
new file mode 100755
index 0000000000000000000000000000000000000000..e28d563f5009f2e8e7260bb7444d589c70e6b20e
--- /dev/null
+++ b/sitescripts/filterhits/bin/process_logs.py
@@ -0,0 +1,89 @@
+# coding: utf-8

[Wladimir Palant, 2015/03/27 16:30:48]

One more note: the purpose of this script wasn't clear to me initially, rename
it into reprocess_logs.py?

[kzar, 2015/03/27 22:15:00]

Done.

+
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2015 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import MySQLdb
+import itertools
+import json
+import os
+import sys
+
+from sitescripts.utils import get_config
+from sitescripts.filterhits import db, geometrical_mean
+
+_last_log_file = None
+
+def log_files(dir):
+  """
+  Provides a generator of filter hits log files for the given directory.
+  Works recursively, relative path of log file is returned.
+  """
+  for root, subdirs, files in os.walk(dir):
+    for f in files:
+      if os.path.splitext(f)[1] == ".log" and f[0].isdigit():
+        yield os.path.join(root, f)
+
+def read_data(log_file):
+  """
+  Read, parse and return the JSON data for the given log file name.
+  (As a side effect sets the global _last_log_file to the log file name.)
+  """
+  global _last_log_file
+  try:
+    with open(log_file, "r") as f:
+      f.readline()
+      data = json.load(f)
+      # Keep track of the current log file in global variable in case we need to
+      # identify it later if there's a problem. (This works because the files are
+      # processed lazily.)
+      _last_log_file = log_file
+  except IOError:
+    sys.exit("Could not read log file %s" % log_file)
+  return data
+
+if __name__ == "__main__":
+  if not len(sys.argv) == 2:
+    print "Usage: python -m sitescripts.filterhits.bin.process_logs /path/to/logs"
+    sys.exit(1)
+
+  interval = get_config().get("filterhitstats", "interval")
+
+  def read_update(f):
+    return geometrical_mean.update(interval, read_data(f))
+
+  if sys.argv[1].endswith(".log"):
+    sql = read_update(sys.argv[1])
+  else:
+    sql = itertools.chain.from_iterable(itertools.imap(read_update,
+                                                       log_files(sys.argv[1])))
+
+  try:
+    db_connection = db.connect()
+  except MySQLdb.Error:
+    sys.exit("Failed to connect to the MySQL database.")

[Wladimir Palant, 2015/03/27 16:29:06]

Given that Sebastian doesn't seem to disagree, could you just leave that
exception alone here instead of turning it into a meaningless message?

[kzar, 2015/03/27 22:15:00]

Done.

+
+  try:
+    db.write(db_connection, sql)
+  except (KeyError, MySQLdb.Error), e:

[Wladimir Palant, 2015/03/27 16:29:06]

The json module seems to raise a ValueError exception on invalid syntax. It
seems that for this exception we won't tell which file caused it.

Do we even care about the exception type here? How about:

except:
  logging.error("Failed to process file %s, all changes rolled back." %
_last_log_file)
  raise

[kzar, 2015/03/27 22:15:00]

Done.

+    if isinstance(e, KeyError):
+      message = "KeyError (%s)" % str(e.args[0])
+    elif isinstance(e, MySQLdb.Error):
+      message = "MySQL error (%d) \"%s\"" % (e.args[0], e.args[1])
+    sys.exit("Failed to process file %s, all changes rolled back. %s\n" % (
+      _last_log_file, message
+    ))
+  finally:
+    db_connection.close()