Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/filterhits/bin/process_logs.py

Issue 4615801646612480: Issue 395 - Filter hits statistics backend (Closed)
Patch Set: Rebased. Created March 2, 2015, 8:41 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « sitescripts/filterhits/bin/__init__.py ('k') | sitescripts/filterhits/common.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 # coding: utf-8
2
3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH
5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation.
9 #
10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17
18 import MySQLdb, itertools, json, os, sys
19 from sitescripts.utils import get_config
20 from sitescripts.filterhits import common, db, geometrical_mean
21
22 last_log_file = None
23
24 def log_files(dir):
25 """
26 Provides a generator of filter hits log files for the given directory.
27 Works recursively, relative path of log file is returned.
28 """
29 for root, subdirs, files in os.walk(dir):
30 for f in files:
31 if os.path.splitext(f)[1] == ".log" and f[0].isdigit():
32 yield os.path.join(root, f)
33
34 def read_data(log_file):
35 """
36 Read, parse and return the JSON data for the given log file name.
37 (As a side effect sets the global last_log_file to the log file name.)
38 """
39 global last_log_file
40 try:
41 with open(log_file, "r") as f:
42 # Skip past the date and GET parameters
43 current = last = ""
44 while last + current != '" ':
45 last, current = current, f.read(1)
46 if not current:
47 sys.exit("Unexpected EOF in log file %s" % log_file)
48
49 # Read the JSON
50 data = json.load(f)
51 # Keep track of the current log file in global variable in case we need to
52 # identify it later if there's a problem. (This works because the files ar e
53 # processed lazily.)
54 last_log_file = log_file
55 except IOError:
56 sys.exit("Could not read log file %s" % log_file)
57 return data
58
59 if __name__ == "__main__":
60 if not len(sys.argv) == 2:
61 print "Usage: python -m sitescripts.filterhits.bin.process_logs /path/to/log s"
62 sys.exit(1)
63
64 interval = get_config().get("filterhitstats", "interval")
65
66 def read_update(f):
67 return geometrical_mean.update(interval, read_data(f))
68
69 if sys.argv[1].endswith(".log"):
70 sql = read_update(sys.argv[1])
71 else:
72 sql = itertools.chain.from_iterable(itertools.imap(read_update,
73 log_files(sys.argv[1])))
74
75 try:
76 db_connection = db.connect()
77 except MySQLdb.Error:
78 sys.exit("Failed to connect to the MySQL database.")
79
80 try:
81 db.write(db_connection, sql)
82 except (KeyError, MySQLdb.Error), e:
83 if isinstance(e, KeyError):
84 message = "KeyError (%s)" % str(e.args[0])
85 elif isinstance(e, MySQLdb.Error):
86 message = "MySQL error (%d) \"%s\"" % (e.args[0], e.args[1])
87 sys.exit("Failed to process file %s, all changes rolled back. %s\n" % (
88 last_log_file, message
89 ))
90 finally:
91 db_connection.close()
OLDNEW
« no previous file with comments | « sitescripts/filterhits/bin/__init__.py ('k') | sitescripts/filterhits/common.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld