Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Unified Diff: sitescripts/filterhits/bin/process_logs.py

Issue 4615801646612480: Issue 395 - Filter hits statistics backend (Closed)
Patch Set: Addressed more comments. Created March 2, 2015, 10:37 a.m.
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « sitescripts/filterhits/bin/__init__.py ('k') | sitescripts/filterhits/common.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: sitescripts/filterhits/bin/process_logs.py
diff --git a/sitescripts/filterhits/bin/process_logs.py b/sitescripts/filterhits/bin/process_logs.py
new file mode 100755
index 0000000000000000000000000000000000000000..168033588d564ba37047f0aa84a7b22259952965
--- /dev/null
+++ b/sitescripts/filterhits/bin/process_logs.py
@@ -0,0 +1,87 @@
+# coding: utf-8
+
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2015 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
+
+import MySQLdb, itertools, json, os, sys
+from sitescripts.utils import get_config
+from sitescripts.filterhits import common, db, geometrical_mean
+
+last_log_file = None
+
+def log_files(dir):
+ """
+ Provides a generator of filter hits log files for the given directory.
+ Works recursively, relative path of log file is returned.
+ """
+ for root, subdirs, files in os.walk(dir):
+ for f in files:
+ if os.path.splitext(f)[1] == ".log" and f[0].isdigit():
+ yield os.path.join(root, f)
+
+def read_data(log_file):
+ """
+ Read, parse and return the JSON data for the given log file name.
+ (As a side effect sets the global last_log_file to the log file name.)
+ """
+ global last_log_file
+ try:
+ with open(log_file, "r") as f:
+ # Skip past the date and GET parameters
+ current = last = ""
+ while last + current != '" ':
+ last, current = current, f.read(1)
+ if not current:
+ sys.exit("Unexpected EOF in log file %s" % log_file)
+
+ # Read the JSON
+ data = json.load(f)
+ # Keep track of the current log file in global variable in case we need to
+ # identify it later if there's a problem. (This works because the files are
+ # processed lazily.)
+ last_log_file = log_file
+ except IOError:
+ sys.exit("Could not read log file %s" % log_file)
+ return data
+
+if __name__ == "__main__":
+ if not len(sys.argv) == 2:
+ print "Usage: python -m sitescripts.filterhits.bin.process_logs /path/to/logs"
+ sys.exit(1)
+
+ interval = get_config().get("filterhitstats", "interval")
+
+ def read_update(f):
+ return geometrical_mean.update(interval, read_data(f))
+
+ if sys.argv[1].endswith(".log"):
+ sql = read_update(sys.argv[1])
+ else:
+ sql = itertools.chain.from_iterable(itertools.imap(read_update,
+ log_files(sys.argv[1])))
+
+ db_connection = db.connect()
+ try:
+ db.write(db_connection, sql)
+ except (KeyError, MySQLdb.Error), e:
+ if isinstance(e, KeyError):
+ message = "KeyError (%s)" % str(e.args[0])
+ elif isinstance(e, MySQLdb.Error):
+ message = "MySQL error (%d) \"%s\"" % (e.args[0], e.args[1])
+ sys.exit("Failed to process file %s, all changes rolled back. %s\n" % (
+ last_log_file, message
+ ))
+ finally:
+ db_connection.close()
« no previous file with comments | « sitescripts/filterhits/bin/__init__.py ('k') | sitescripts/filterhits/common.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld