Issue 1390 - Record referrer for download stats

sitescripts/stats/bin/logprocessor.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2014 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 354 matching lines @@
   applicationVersion = re.sub(r"^(\d+\.\d+).*", r"\1", applicationVersion)
 
   return version, application, applicationVersion
 
 def parse_update_flag(query):
   return "update" if query == "update" else "install"
 
 def parse_record(line, ignored, geo, geov6):
   global log_regexp
   if log_regexp == None:
-    log_regexp = re.compile(r'(\S+) \S+ \S+ \[([^]\s]+) ([+\-]\d\d)(\d\d)\] "GET ([^"\s]+) [^"]+" (\d+) (\d+) "[^"]*" "([^"]*)"(?: "[^"]*" \S+ "[^"]*" "[^"]*" "([^"]*)")?')
+    log_regexp = re.compile(r'(\S+) \S+ \S+ \[([^]\s]+) ([+\-]\d\d)(\d\d)\] "GET ([^"\s]+) [^"]+" (\d+) (\d+) "([^"]*)" "([^"]*)"(?: "[^"]*" \S+ "[^"]*" "[^"]*" "([^"]*)")?')
 
   match = re.search(log_regexp, line)
   if not match:
     return None
 
   status = int(match.group(6))
   if status != 200:
     return None
 
   info = {
     "size": int(match.group(7)),
   }
 
   info["ip"], info["country"] = process_ip(match.group(1), geo, geov6)
   info["time"], info["month"], info["day"], info["weekday"], info["hour"] = parse_time(match.group(2), int(match.group(3)), int(match.group(4)))
   info["file"], info["query"] = parse_path(match.group(5))
-  info["ua"], info["uaversion"] = parse_ua(match.group(8))
+  info["referrer"] = match.group(8)
+  info["ua"], info["uaversion"] = parse_ua(match.group(9))
   info["fullua"] = "%s %s" % (info["ua"], info["uaversion"])
-  info["clientid"] = match.group(9)
+  info["clientid"] = match.group(10)
 
   # Additional metadata depends on file type
   filename = os.path.basename(info["file"])
   ext = os.path.splitext(filename)[1]
   if ext == ".txt" or filename == "update.json" or filename == "notification.json":
     # Subscription downloads, libadblockplus update checks and notification
     # checks are performed by the downloader
     parse_downloader_query(info)
   elif ext == ".tpl":
     # MSIE TPL download, no additional data here
@@ skipping 140 matching lines @@
   parser.add_argument("mirror_name", nargs="?", help="Name of the mirror server that the file belongs to")
   parser.add_argument("server_type", nargs="?", help="Server type like download, update or subscription")
   parser.add_argument("log_file", nargs="?", help="Log file path, can be a local file path, http:// or ssh:// URL")
   args = parser.parse_args()
 
   if args.mirror_name and args.server_type and args.log_file:
     sources = [(args.mirror_name, args.server_type, args.log_file)]
   else:
     sources = get_stats_files()
   parse_sources(sources, args.factor, args.verbose)