Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/filterhits/web/submit.py

Issue 4615801646612480: Issue 395 - Filter hits statistics backend (Closed)
Patch Set: Addressed more comments. Created March 2, 2015, 10:37 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « sitescripts/filterhits/web/query.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2014 Eyeo GmbH 4 # Copyright (C) 2006-2015 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 import MySQLdb, json 18 import MySQLdb, json, os
19 from sitescripts.web import url_handler, showError 19 from datetime import datetime
20 from sitescripts.web import url_handler
20 from sitescripts.utils import get_config, setupStderr 21 from sitescripts.utils import get_config, setupStderr
21 import sitescripts.filterhits.common as common 22 from sitescripts.filterhits import common, db, geometrical_mean
22
23 config = get_config()
24 23
25 @url_handler("/submit") 24 @url_handler("/submit")
26 def submit(environ, start_response): 25 def submit(environ, start_response):
27 global config
28 setupStderr(environ["wsgi.errors"]) 26 setupStderr(environ["wsgi.errors"])
27 config = get_config()
29 28
30 # Check that this is a POST request 29 # Check that this is a POST request
31 if environ["REQUEST_METHOD"].upper() != "POST": 30 if environ["REQUEST_METHOD"] != "POST":
32 return showError("Unsupported request method", start_response) 31 return common.show_error("Unsupported request method", start_response)
33 32
34 # Parse the submitted JSON 33 # Parse the submitted JSON
35 data = "{}"
36 try: 34 try:
37 data_length = int(environ.get("CONTENT_LENGTH", "0")) 35 data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])) )
38 except ValueError: 36 except (KeyError, IOError, ValueError):
39 data_length = 0 37 return common.show_error("Error while parsing JSON data.", start_response)
40 if data_length != 0:
41 data = environ["wsgi.input"].read(data_length)
42 try:
43 data = json.loads(data)
44 except json.decoder.JSONDecodeError:
45 return showError("Error while parsing JSON data.", start_response)
46 38
47 # Make sure it looks roughly valid 39 # Make sure the submitted data was contained within an object at least
48 if not common.valid_log_data(data): 40 if not isinstance(data, dict):
49 return showError("Data looks invalid.", start_response) 41 return common.show_error("Error, data must be contained within an object.", start_response)
50 42
51 # Log the data to a file 43 # Log the data to a file
52 log_dir = config.get("filterhitstats", "log_dir") 44 if not db.testing:
45 log_dir = config.get("filterhitstats", "log_dir")
46 try:
47 log_file = common.log_filterhits(data, log_dir,
48 environ.get("QUERY_STRING", ""))
49 except (OSError, IOError):
50 return common.show_error("Failed to write data to log file!", start_respon se,
51 "500 Logging error")
53 52
53 # Update the geometrical_mean aggregations in the database
54 interval = config.get("filterhitstats", "interval")
54 try: 55 try:
55 common.log_filterhits(data, log_dir) 56 db_connection = db.connect()
56 except OSError, IOError: 57 try:
57 return showError("Failed to save data!", start_response) 58 db.write(db_connection, geometrical_mean.update(interval, data))
59 finally:
60 db_connection.close()
61 except (KeyError, MySQLdb.Error), e:
62 # Updating the aggregations in the database failed for whatever reason,
63 # log the details but continue to return 200 OK to the client to avoid
64 # re-transmission of the data.
65 if not db.testing:
66 processing_error_log = os.path.join(config.get("filterhitstats", "log_dir" ),
67 "processing-errors.log")
68 with open(processing_error_log, "a+") as f:
69 if isinstance(e, KeyError):
70 message = "KeyError (%s) when processing data file %s\n" % (str(e.args [0]), log_file)
71 elif isinstance(e, MySQLdb.Error):
72 message = "MySQL error (%d) when processing data file %s: \"%s\"\n" % (e.args[0], log_file, e.args[1])
73 f.write("[%s] %s" % (datetime.now().strftime('%d/%b/%Y:%H:%M:%S %z'), me ssage))
58 74
59 # # TODO calculate the geometrical mean and update the database 75 # Send back a 200 OK response
60 # # Open DB connection set up cursor
61 # db = _get_db(config)
62 # cursor = db.cursor()
63 # for filter, filter_data in data['filters'].iteritems():
64 # for domain, domain_hits in itertools.chain(filter_data['thirdParty'].iteri tems(),
65 # filter_data['firstParty'].iteri tems()):
66 # 1 # UPDATE geometrical_mean SET hits=(??) timestamp=new_timestamp WHERE filter=filter AND domain=domain;
67 # # ^ SELECT current hits + timestamp + perform geometrical mean on combo
68 # # Commit / rollback and close
69 # # db.commit()
70 # db.close()
71
72 response_headers = [("Content-type", "text/plain")] 76 response_headers = [("Content-type", "text/plain")]
73 start_response("200 OK", response_headers) 77 start_response("200 OK", response_headers)
74 return [] 78 return []
75
76 # def _increment_entry(db, domain_id, status):
77 # cursor = db.cursor(MySQLdb.cursors.DictCursor)
78 # cursor.execute("INSERT INTO corrections(domain, status, curr_month, prev_mon th, curr_year, prev_year) VALUES (%s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDAT E curr_month = curr_month + 1, curr_year = curr_year + 1", (domain_id, status, 1 , 0, 1, 0))
OLDNEW
« no previous file with comments | « sitescripts/filterhits/web/query.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld