Issue 1426 - Don`t ignore redirects during stats processing

sitescripts/stats/test/logprocessor.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2014 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 323 matching lines @@
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "devbuilds/adblockpluschrome/updates.xml",
           "query": "os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&prodchannel=stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3.977%26uc",
           "size": 867,
           "referrer": "-",
+          "status": 200,
           "ua": "Chrome",
           "uaversion": "28.0",
           "fullua": "Chrome 28.0",
           "clientid": "-",
           "addonName": "adblockpluschrome",
           "addonVersion": "1.5.3.977",
           "fullAddon": "adblockpluschrome 1.5.3.977",
           "application": "chrome",
           "applicationVersion": "28.0",
           "fullApplication": "chrome 28.0",
         }
       ),
       (
+        '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcustomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion=25.0a1&updateType=97 HTTP/1.1" 404 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "downloads.adblockplus.org" "-"',
+        None
+      ),
+      (
+        '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcustomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion=25.0a1&updateType=97 HTTP/1.1" 301 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "downloads.adblockplus.org" "-"',
+        {
+          "ip": "1.2.3.4",
+          "country": "xy",
+          "time": datetime(2013, 7, 31, 17, 33, 8),
+          "day": 31,
+          "weekday": 2,
+          "hour": 17,
+          "month": "201307",
+          "file": "devbuilds/abpcustomization/update.rdf",
+          "query": "reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion=25.0a1&updateType=97",
+          "size": 867,
+          "referrer": "-",
+          "status": 301,
+          "ua": "Firefox",
+          "uaversion": "25.0",
+          "fullua": "Firefox 25.0",
+          "clientid": "-",
+          "addonName": "abpcustomization",
+          "addonVersion": "1.0.4a.74",
+          "fullAddon": "abpcustomization 1.0.4a.74",
+          "application": "firefox",
+          "applicationVersion": "25.0",
+          "fullApplication": "firefox 25.0",
+        }
+      ),
+      (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcustomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion=25.0a1&updateType=97 HTTP/1.1" 302 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "downloads.adblockplus.org" "-"',
-        None
+        {
+          "ip": "1.2.3.4",
+          "country": "xy",
+          "time": datetime(2013, 7, 31, 17, 33, 8),
+          "day": 31,
+          "weekday": 2,
+          "hour": 17,
+          "month": "201307",
+          "file": "devbuilds/abpcustomization/update.rdf",
+          "query": "reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion=25.0a1&updateType=97",
+          "size": 867,
+          "referrer": "-",
+          "status": 302,
+          "ua": "Firefox",
+          "uaversion": "25.0",
+          "fullua": "Firefox 25.0",
+          "clientid": "-",
+          "addonName": "abpcustomization",
+          "addonVersion": "1.0.4a.74",
+          "fullAddon": "abpcustomization 1.0.4a.74",
+          "application": "firefox",
+          "applicationVersion": "25.0",
+          "fullApplication": "firefox 25.0",
+        }
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcustomization/update.unknown?reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion=25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "downloads.adblockplus.org" "-"',
         None
       ),
       (
         '1.2.3.4 corrupted',
         None
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcustomization/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion=25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "downloads.adblockplus.org" "-"',
         {
           "ip": "1.2.3.4",
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "devbuilds/abpcustomization/update.rdf",
           "query": "reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion=25.0a1&updateType=97",
           "size": 867,
           "referrer": "-",
+          "status": 200,
           "ua": "Firefox",
           "uaversion": "25.0",
           "fullua": "Firefox 25.0",
           "clientid": "-",
           "addonName": "abpcustomization",
           "addonVersion": "1.0.4a.74",
           "fullAddon": "abpcustomization 1.0.4a.74",
           "application": "firefox",
           "applicationVersion": "25.0",
           "fullApplication": "firefox 25.0",
         }
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adblockplusie/update.json?addonName=adblockplusie&addonVersion=2.0&application=msie64&applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion=0 HTTP/1.1" 200 867 "-" "Adblock Plus" "-" https" "en-US" "downloads.adblockplus.org" "-"',
         {
           "ip": "1.2.3.4",
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "devbuilds/adblockplusie/update.json",
           "query": "addonName=adblockplusie&addonVersion=2.0&application=msie64&applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion=0",
           "size": 867,
           "referrer": "-",
+          "status": 200,
           "ua": "ABP",
           "uaversion": "",
           "fullua": "ABP ",
           "clientid": "-",
           "addonName": "adblockplusie",
           "addonVersion": "2.0",
           "fullAddon": "adblockplusie 2.0",
           "application": "msie64",
           "applicationVersion": "10.0",
           "fullApplication": "msie64 10.0",
@@ skipping 12 matching lines @@
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "adblockplusandroid-1.1.2.apk",
           "query": "",
           "size": 49152,
           "referrer": "https://adblockplus.org/en/android-install",
+          "status": 200,
           "ua": "Mobile Safari",
           "uaversion": "4.0",
           "fullua": "Mobile Safari 4.0",
           "clientid": "-",
           "installType": "install",
         }
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adblockplus/adblockplus-2.3.2.3712.xpi?update HTTP/1.1" 200 827261 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0" "-" https',
         {
           "ip": "1.2.3.4",
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "devbuilds/adblockplus/adblockplus-2.3.2.3712.xpi",
           "query": "update",
           "size": 827261,
           "referrer": "-",
+          "status": 200,
           "ua": "Firefox",
           "uaversion": "22.0",
           "fullua": "Firefox 22.0",
           "clientid": None,
           "installType": "update",
         }
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /exceptionrules.txt?addonName=adblockplus&addonVersion=2.3.2&application=firefox&applicationVersion=22.0&platform=gecko&platformVersion=22.0&lastVersion=201307311503 HTTP/1.1" 200 14303 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0" "-" https "en-US,en;q=0.5" "easylist-downloads.adblockplus.org" "-"',
         {
           "ip": "1.2.3.4",
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "exceptionrules.txt",
           "query": "addonName=adblockplus&addonVersion=2.3.2&application=firefox&applicationVersion=22.0&platform=gecko&platformVersion=22.0&lastVersion=201307311503",
           "size": 14303,
           "referrer": "-",
+          "status": 200,
           "ua": "Firefox",
           "uaversion": "22.0",
           "fullua": "Firefox 22.0",
           "clientid": "-",
           "addonName": "adblockplus",
           "addonVersion": "2.3.2",
           "fullAddon": "adblockplus 2.3.2",
           "application": "firefox",
           "applicationVersion": "22.0",
           "fullApplication": "firefox 22.0",
@@ skipping 11 matching lines @@
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "easylist.txt",
           "query": "_=1375446528229",
           "size": 326120,
           "referrer": "-",
+          "status": 200,
           "ua": "Chrome",
           "uaversion": "28.0",
           "fullua": "Chrome 28.0",
           "clientid": "AdBlock/2.6.2",
           "addonName": "chromeadblock",
           "addonVersion": "2.6.2",
           "fullAddon": "chromeadblock 2.6.2",
           "application": "unknown",
           "applicationVersion": "unknown",
           "fullApplication": "unknown unknown",
@@ skipping 11 matching lines @@
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "easylistitaly.txt",
           "query": "",
           "size": 85879,
           "referrer": "-",
+          "status": 200,
           "ua": "Other",
           "uaversion": "",
           "fullua": "Other ",
           "clientid": "-",
           "addonName": "unknown",
           "addonVersion": "unknown",
           "fullAddon": "unknown unknown",
           "application": "unknown",
           "applicationVersion": "unknown",
           "fullApplication": "unknown unknown",
@@ skipping 11 matching lines @@
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "easylistitaly.tpl",
           "query": "",
           "size": 85879,
           "referrer": "-",
+          "status": 200,
           "ua": "Other",
           "uaversion": "",
           "fullua": "Other ",
           "clientid": "-",
         }
       ),
       (
         '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /notification.json?addonName=adblockpluschrome&addonVersion=1.5.3&application=chrome&applicationVersion=28.0.1500.72&platform=chromium&platformVersion=28.0.1500.72&lastVersion=201307292310 HTTP/1.1" 200 299 "-" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https',
         {
           "ip": "1.2.3.4",
           "country": "xy",
           "time": datetime(2013, 7, 31, 17, 33, 8),
           "day": 31,
           "weekday": 2,
           "hour": 17,
           "month": "201307",
           "file": "notification.json",
           "query": "addonName=adblockpluschrome&addonVersion=1.5.3&application=chrome&applicationVersion=28.0.1500.72&platform=chromium&platformVersion=28.0.1500.72&lastVersion=201307292310",
           "size": 299,
           "referrer": "-",
+          "status": 200,
           "ua": "Chrome",
           "uaversion": "28.0",
           "fullua": "Chrome 28.0",
           "clientid": None,
           "addonName": "adblockpluschrome",
           "addonVersion": "1.5.3",
           "fullAddon": "adblockpluschrome 1.5.3",
           "application": "chrome",
           "applicationVersion": "28.0",
           "fullApplication": "chrome 28.0",
@@ skipping 61 matching lines @@
           "addonName": {"bar": {"hits": 1, "bandwidth": 200}}
         },
       ),
     ]
     for info, section, ignored_fields, expected_result in tests:
       logprocessor.add_record(info, section, ignored_fields)
       self.assertEqual(section, expected_result)
 
 if __name__ == '__main__':
   unittest.main()