Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/filterhits/bin/reprocess_logs.py

Issue 4615801646612480: Issue 395 - Filter hits statistics backend (Closed)
Patch Set: Addressed final nits. Created March 28, 2015, 1:10 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « sitescripts/filterhits/bin/__init__.py ('k') | sitescripts/filterhits/db.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 # coding: utf-8
2
3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH
5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation.
9 #
10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17
18 import logging
19 import MySQLdb
20 import itertools
21 import json
22 import os
23 import sys
24
25 from sitescripts.utils import get_config
26 from sitescripts.filterhits import db, geometrical_mean
27
28 _last_log_file = None
29
30 def log_files(dir):
31 """
32 Provides a generator of filter hits log files for the given directory.
33 Works recursively, relative path of log file is returned.
34 """
35 for root, subdirs, files in os.walk(dir):
36 for f in files:
37 if os.path.splitext(f)[1] == ".log" and f[0].isdigit():
38 yield os.path.join(root, f)
39
40 def read_data(log_file):
41 """
42 Read, parse and return the JSON data for the given log file name.
43 (As a side effect sets the global _last_log_file to the log file name.)
44 """
45 global _last_log_file
46 try:
47 with open(log_file, "r") as f:
48 f.readline()
49 data = json.load(f)
50 # Keep track of the current log file in global variable in case we need to
51 # identify it later if there's a problem. (This works because the files ar e
52 # processed lazily.)
53 _last_log_file = log_file
54 except IOError:
55 sys.exit("Could not read log file %s" % log_file)
56 return data
57
58 if __name__ == "__main__":
59 if not len(sys.argv) == 2:
60 print "Usage: python -m sitescripts.filterhits.bin.reprocess_logs /path/to/l ogs"
61 sys.exit(1)
62
63 interval = get_config().get("filterhitstats", "interval")
64
65 def read_update(f):
66 return geometrical_mean.update(interval, read_data(f))
67
68 if sys.argv[1].endswith(".log"):
69 sql = read_update(sys.argv[1])
70 else:
71 sql = itertools.chain.from_iterable(itertools.imap(read_update,
72 log_files(sys.argv[1])))
73
74 db_connection = db.connect()
75
76 try:
77 db.write(db_connection, sql)
78 except:
79 logging.error("Failed to process file %s, all changes rolled back." % _last_ log_file)
80 raise
81 finally:
82 db_connection.close()
OLDNEW
« no previous file with comments | « sitescripts/filterhits/bin/__init__.py ('k') | sitescripts/filterhits/db.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld