Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/stats/test/logprocessor.py

Issue 11481051: Update stats processing (Closed)
Patch Set: Fixed two presentation issues Created Aug. 24, 2013, 1:11 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 # coding: utf-8
2
3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2013 Eyeo GmbH
5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation.
9 #
10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17
18 import unittest
19 import sitescripts.stats.bin.logprocessor as logprocessor
20 from datetime import datetime
21
22 class Test(unittest.TestCase):
23 longMessage = True
24 maxDiff = None
25
26 def test_uaparsing(self):
27 tests = [
28 ("Firefox", "25.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gec ko/20130730 Firefox/25.0"),
29 ("Firefox", "25.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:28.0) Gec ko/20130730 Firefox/25.0"),
30 ("Firefox", "25.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0.1) G ecko/20130730 Firefox/25.0.1"),
31 ("Firefox Mobile", "15.0", "Mozilla/5.0 (Maemo; Mobile; rv:15.0) Gecko/201 20829 Firefox/15.0 Fennec/15.0"),
32 ("Firefox Mobile", "14.0", "Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/1 4.0 Firefox/14.0a2"),
33 ("Thunderbird", "24.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Geck o/20100101 Thunderbird/24.0a2 Lightning/2.6a2"),
34 ("SeaMonkey", "2.19", "Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/2010010 1 Firefox/22.0 SeaMonkey/2.19"),
35 ("K-Meleon", "1.5", "Mozilla/5.0 (Windows; U; Windows NT 6.1; ru-RU; rv:1. 8.1.24pre) Gecko/20100228 K-Meleon/1.5.4"),
36 ("Prism", "1.0", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2 .20) Gecko/20110803 Prism/1.0b4"),
37 ("Gecko", "22.0", "Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/20100101 Fo oBar/1.0"),
38 ("Opera", "11.10", "Opera/9.80 (Android; Opera Mini/15.0.1162/30.3558; U; it) Presto/2.8.119 Version/11.10"),
39 ("Opera", "9.80", "Opera/9.80 (Android; Opera Mini/15.0.1162/30.3558; U; i t) Presto/2.8.119"),
40 ("Opera", "15.0", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36 OPR/15.0.1147.148"),
41 ("Chrome", "28.0", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36"),
42 ("Chrome", "28.0", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Iron/28.0.1550.0 Chrome/28.0.1550.0 Safari/537.36"),
43 ("Safari", "5.0", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us ) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16"),
44 ("Mobile Safari", "4.0", "Mozilla/5.0 (Linux; U; Android 4.0.4; pt-br; LG- E400 Build/IMM76L; CyanogenMod-9) AppleWebKit/534.30 (KHTML, like Gecko) Version /4.0 Mobile Safari/534.30"),
45 ("CoolNovo", "2.0.9", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537 .36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36 CoolNovo/2.0.9.11"),
46 ("WebKit", "", "Mozilla/5.0 (Linux; U; Android 4.2.1; zh-CN; P7 Build/JRO0 3C) AppleWebKit/534.31 (KHTML, like Gecko) UCBrowser/9.2.0.308 U3/0.8.0 Mobile S afari/534.31"),
47 ("MSIE", "10.0", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW6 4; Trident/6.0)"),
48 ("Trident", "7.0", "Mozilla/5.0 (Windows NT 6.3; ARM; Trident/7.0; Touch; rv:11.0) like Gecko"),
49 ("Android", "", "AndroidDownloadManager"),
50 ("Android", "4.1", "AndroidDownloadManager/4.1.1 (Linux; U; Android 4.1.1; A210 Build/JRO03H)"),
51 ("Android", "4.0", "Dalvik/1.6.0 (Linux; U; Android 4.0.3; KFOT Build/IML7 4K)"),
52 ("Android", "4.3", "Dalvik/1.6.0 (Linux; U; Android 4.3; Nexus 7 Build/JSS 15J)"),
53 ("Android", "", "Apache-HttpClient/UNAVAILABLE (java 1.4)"),
54 ("ABP", "", "Adblock Plus"),
55 ("Other", "", "-"),
56 ]
57 for expected_browser, expected_version, ua in tests:
58 self.assertEqual(logprocessor.parse_ua(ua), (expected_browser, expected_ve rsion), "Parsing user agent '%s'" % ua)
59
60 def test_ipprocessing(self):
61 country = None
62 class FakeGeo(object):
63 ip_checked = None
64 def country_code_by_addr(self, ip):
65 self.ip_checked = ip
66 return country
67
68 tests = [
69 ("1.2.3.4", "xy", "1.2.3.4", "xy"),
70 ("::ffff:1.2.3.4", "xy", "1.2.3.4", "xy"),
71 ("1.2.3.4", "--", "1.2.3.4", "unknown"),
72 ("1.2.3.4", "", "1.2.3.4", "unknown"),
73 ("::ffff:1.2.3.4", None, "1.2.3.4", "unknown"),
74 ]
75 for ip, country, expected_ip, expected_country in tests:
76 fake_geo = FakeGeo()
77 self.assertEqual(logprocessor.process_ip(ip, fake_geo), (expected_ip, expe cted_country), "Processing IP '%s'" % ip)
78 self.assertEqual(fake_geo.ip_checked, expected_ip, "GeoIP check for IP '%s '" % ip)
79
80 def test_timeparsing(self):
81 tests = [
82 ("31/Jul/2013:12:03:37", 0, 0, datetime(2013, 07, 31, 12, 03, 37), "201307 "),
83 ("31/Jul/2013:12:03:37", 5, 0, datetime(2013, 07, 31, 7, 03, 37), "201307" ),
84 ("31/Jul/2013:12:03:37", -5, 0, datetime(2013, 07, 31, 17, 03, 37), "20130 7"),
85 ("31/Jul/2013:12:03:37", 5, 30, datetime(2013, 07, 31, 6, 33, 37), "201307 "),
86 ("31/Jul/2013:12:03:37", -5, 30, datetime(2013, 07, 31, 17, 33, 37), "2013 07"),
87 ]
88 for timestr, tz_hours, tz_minutes, expected_time, expected_month in tests:
89 self.assertEqual(logprocessor.parse_time(timestr, tz_hours, tz_minutes),
90 (expected_time, expected_month, expected_time.day, expected_time.weekd ay(), expected_time.hour),
91 "Parsing time string '%s %+03i%02i'" % (timestr, tz_hours, tz_minutes) )
92
93 def test_pathparsing(self):
94 tests = [
95 ("/foo.txt", "foo.txt", ""),
96 ("/foo.txt?", "foo.txt", ""),
97 ("/foo.txt?asdf", "foo.txt", "asdf"),
98 ("http://example.com/foo.txt", "foo.txt", ""),
99 ("/xyz/foo.txt?asdf", "xyz/foo.txt", "asdf"),
100 ("/xyz/foo+bar.txt?asdf", "xyz/foo+bar.txt", "asdf"),
101 ("/xyz/foo%2Bbar%2etxt?asdf", "xyz/foo+bar.txt", "asdf"),
102 ("/xyz/%D1%82%D0%B5%D1%81%D1%82.txt", u"xyz/\u0442\u0435\u0441\u0442.txt", ""),
103 ("/xyz/foo%8Cbar.txt?asdf", "xyz/foo%8Cbar.txt", "asdf"),
104 ]
105 for path, expected_file, expected_query in tests:
106 self.assertEqual(logprocessor.parse_path(path), (expected_file, expected_q uery), "Parsing path '%s'" % path)
107
108 def test_downloaderdata(self):
109 tests = [
110 (
111 datetime(2013, 07, 31, 12, 03, 00),
112 "easylist.txt",
113 "",
114 "",
115 "unknown/unknown",
116 "unknown/unknown",
117 "unknown/unknown",
118 "unknown",
119 "",
120 ),
121 (
122 datetime(2013, 07, 31, 12, 03, 00),
123 "easylist.txt",
124 "addonName=adblockplus&addonVersion=2.3.1&application=firefox&applicatio nVersion=22.0a1&platform=gecko&platformVersion=23.0&lastVersion=0",
125 "-",
126 "adblockplus/2.3.1",
127 "firefox/22.0",
128 "gecko/23.0",
129 "unknown",
130 "firstDownload firstInMonth firstInWeek firstInDay",
131 ),
132 (
133 datetime(2013, 07, 31, 12, 03, 00),
134 "easylist.txt",
135 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion =23.0&lastVersion=201307311200",
136 "-",
137 "adblockplus/2.3.1",
138 "unknown/unknown",
139 "gecko/23.0",
140 "0 hour(s)",
141 "",
142 ),
143 (
144 datetime(2013, 07, 31, 12, 03, 00),
145 "easylist.txt",
146 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion =23.0&lastVersion=201307302200",
147 "-",
148 "adblockplus/2.3.1",
149 "unknown/unknown",
150 "gecko/23.0",
151 "14 hour(s)",
152 "firstInDay",
153 ),
154 (
155 datetime(2013, 07, 31, 12, 03, 00),
156 "easylist.txt",
157 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion =23.0&lastVersion=201307282200",
158 "-",
159 "adblockplus/2.3.1",
160 "unknown/unknown",
161 "gecko/23.0",
162 "2 day(s)",
163 "firstInWeek firstInDay",
164 ),
165 (
166 datetime(2013, 8, 2, 12, 03, 00),
167 "easylist.txt",
168 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion =23.0&lastVersion=201307311200",
169 "-",
170 "adblockplus/2.3.1",
171 "unknown/unknown",
172 "gecko/23.0",
173 "2 day(s)",
174 "firstInMonth firstInDay",
175 ),
176 (
177 datetime(2013, 07, 31, 12, 03, 00),
178 "easylist.txt",
179 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion =23.0&lastVersion=201306302200",
180 "-",
181 "adblockplus/2.3.1",
182 "unknown/unknown",
183 "gecko/23.0",
184 "1 month(s)",
185 "firstInMonth firstInWeek firstInDay",
186 ),
187 (
188 datetime(2013, 07, 31, 12, 03, 00),
189 "easylist.txt",
190 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion =23.0&lastVersion=201305302200",
191 "-",
192 "adblockplus/2.3.1",
193 "unknown/unknown",
194 "gecko/23.0",
195 "2 month(s)",
196 "firstInMonth firstInWeek firstInDay",
197 ),
198 (
199 datetime(2013, 07, 31, 12, 03, 00),
200 "easylist.txt",
201 "addonName=adblockplus&addonVersion=2.3.1&platform=gecko&platformVersion =23.0.1&lastVersion=201206302200",
202 "-",
203 "adblockplus/2.3.1",
204 "unknown/unknown",
205 "gecko/23.0",
206 "1 year(s)",
207 "firstInMonth firstInWeek firstInDay",
208 ),
209 (
210 datetime(2013, 07, 31, 12, 03, 00),
211 "easylist.txt",
212 "_=1375142394357",
213 "AdBlock/2.5.4",
214 "chromeadblock/2.5.4",
215 "unknown/unknown",
216 "unknown/unknown",
217 "unknown",
218 ""
219 ),
220 ]
221 for time, file, query, clientid, expected_addon, expected_application, expec ted_platform, expected_interval, expected_flags in tests:
222 info = {"time": time, "file": file, "query": query, "clientid": clientid}
223 logprocessor.parse_downloader_query(info)
224 self.assertEqual("%s/%s" % (info["addonName"], info["addonVersion"]), expe cted_addon, "Add-on for query '%s'" % query)
225 self.assertEqual("%s/%s" % (info["application"], info["applicationVersion" ]), expected_application, "Application for query '%s'" % query)
226 self.assertEqual("%s/%s" % (info["platform"], info["platformVersion"]), ex pected_platform, "Platform for query '%s'" % query)
227 self.assertEqual(info["downloadInterval"], expected_interval, "Download in terval for query '%s'" % query)
228
229 flags = []
230 for flag in ("firstDownload", "firstInMonth", "firstInWeek", "firstInDay") :
231 if flag in info:
232 flags.append(flag)
233 self.assertEqual(" ".join(flags), expected_flags, "Flags for query '%s'" % query)
234
235 def test_nameparsing(self):
236 tests = [
237 ("devbuilds/adblockplus/update.rdf", "adblockplus"),
238 ("adblockpluschrome-experimental/updates.xml", "adblockpluschrome-experime ntal"),
239 ("update.json", None),
240 ]
241 for file, expected_name in tests:
242 self.assertEqual(logprocessor.parse_addon_name(file), expected_name, "Gett ing add-on name for file '%s'" % file)
243
244 def test_geckoqueryparsing(self):
245 tests = [
246 (
247 "reqVersion=2&id={d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d}&version=2.3.1.37 07&maxAppVersion=25.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e9 7384}&appVersion=22.0&appOS=Darwin&appABI=x86_64-gcc3&locale=en-US&currentAppVer sion=22.0&updateType=112",
248 "2.3.1.3707", "firefox", "22.0"
249 ),
250 (
251 "reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&maxAppV ersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384}&appV ersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion=25.0 a1&updateType=97",
252 "1.0.4a.74", "firefox", "25.0"
253 ),
254 (
255 "reqVersion=2&id={d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d}&version=1.3a.201 00925&maxAppVersion=2.1b1&status=userEnabled,incompatible&appID={92650c4d-4b8e-4 d2a-b7eb-24ecf4f6b63a}&appVersion=2.19&appOS=WINNT&appABI=x86-msvc&locale=en-US& currentAppVersion=2.19&updateType=112",
256 "1.3a.20100925", "seamonkey", "2.19"
257 ),
258 ]
259 for query, expected_version, expected_application, expected_applicationversi on in tests:
260 self.assertEqual(logprocessor.parse_gecko_query(query), (expected_version, expected_application, expected_applicationversion), "Parsing Gecko query '%s'" % query)
261
262 def test_chromequeryparsing(self):
263 tests = [
264 (
265 "os=win&arch=x86&nacl_arch=x86-64&prod=chromecrx&prodchannel=stable&prod version=28.0.1500.72&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3.977%26u c",
266 "1.5.3.977", "chrome", "28.0"
267 ),
268 (
269 "x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3.977%26uc",
270 "1.5.3.977", "unknown", "unknown"
271 ),
272 (
273 "api=15&build=256&locale=ru_ru&device=LGE%20LG-P990",
274 "unknown", "unknown", "unknown"
275 ),
276 ]
277 for query, expected_version, expected_application, expected_applicationversi on in tests:
278 self.assertEqual(logprocessor.parse_chrome_query(query), (expected_version , expected_application, expected_applicationversion), "Parsing Chrome query '%s' " % query)
279
280 def test_updateflagparsing(self):
281 tests = [
282 ("update", "update"),
283 ("", "install"),
284 ("foo", "install"),
285 ("update&foo", "install"),
286 ]
287 for query, expected_result in tests:
288 self.assertEqual(logprocessor.parse_update_flag(query), expected_result, " Checking update flag for query '%s'" % query)
289
290 def test_recordparsing(self):
291 class FakeGeo(object):
292 def country_code_by_addr(self, ip):
293 return "xy"
294
295 tests = [
296 (
297 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adblockpluschr ome/updates.xml?os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&prodchannel=stab le&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1.5.3. 977%26uc HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) A ppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36" "-" htt ps" "en-US" "downloads.adblockplus.org" "-"',
298 {
299 "mirror": "mirror123",
300 "ip": "1.2.3.4",
301 "country": "xy",
302 "time": datetime(2013, 7, 31, 17, 33, 8),
303 "day": 31,
304 "weekday": 2,
305 "hour": 17,
306 "month": "201307",
307 "file": "devbuilds/adblockpluschrome/updates.xml",
308 "query": "os=mac&arch=x86&nacl_arch=x86-32&prod=chromecrx&prodchannel= stable&prodversion=28.0.1500.71&x=id%3Dldcecbkkoecffmfljeihcmifjjdoepkn%26v%3D1. 5.3.977%26uc",
309 "size": 867,
310 "ua": "Chrome",
311 "uaversion": "28.0",
312 "fullua": "Chrome 28.0",
313 "clientid": "-",
314 "addonName": "adblockpluschrome",
315 "addonVersion": "1.5.3.977",
316 "fullAddon": "adblockpluschrome 1.5.3.977",
317 "application": "chrome",
318 "applicationVersion": "28.0",
319 "fullApplication": "chrome 28.0",
320 }
321 ),
322 (
323 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcustomizati on/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&ma xAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384} &appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion =25.0a1&updateType=97 HTTP/1.1" 302 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "downloads.adbloc kplus.org" "-"',
324 None
325 ),
326 (
327 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcustomizati on/update.unknown?reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.7 4&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97 384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVer sion=25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1; Wi n64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "downloads.ad blockplus.org" "-"',
328 None
329 ),
330 (
331 '1.2.3.4 corrupted',
332 None
333 ),
334 (
335 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/abpcustomizati on/update.rdf?reqVersion=2&id=customization@adblockplus.org&version=1.0.4a.74&ma xAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e97384} &appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppVersion =25.0a1&updateType=97 HTTP/1.1" 200 867 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20130730 Firefox/25.0" "-" https" "en-US" "downloads.adbloc kplus.org" "-"',
336 {
337 "mirror": "mirror123",
338 "ip": "1.2.3.4",
339 "country": "xy",
340 "time": datetime(2013, 7, 31, 17, 33, 8),
341 "day": 31,
342 "weekday": 2,
343 "hour": 17,
344 "month": "201307",
345 "file": "devbuilds/abpcustomization/update.rdf",
346 "query": "reqVersion=2&id=customization@adblockplus.org&version=1.0.4a .74&maxAppVersion=26.0&status=userEnabled&appID={ec8030f7-c20a-464f-9b0e-13a3a9e 97384}&appVersion=25.0a1&appOS=WINNT&appABI=x86_64-msvc&locale=en-US&currentAppV ersion=25.0a1&updateType=97",
347 "size": 867,
348 "ua": "Firefox",
349 "uaversion": "25.0",
350 "fullua": "Firefox 25.0",
351 "clientid": "-",
352 "addonName": "abpcustomization",
353 "addonVersion": "1.0.4a.74",
354 "fullAddon": "abpcustomization 1.0.4a.74",
355 "application": "firefox",
356 "applicationVersion": "25.0",
357 "fullApplication": "firefox 25.0",
358 }
359 ),
360 (
361 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adblockplusie/ update.json?addonName=adblockplusie&addonVersion=2.0&application=msie64&applicat ionVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion=0 HTTP/1 .1" 200 867 "-" "Adblock Plus" "-" https" "en-US" "downloads.adblockplus.org" "- "',
362 {
363 "mirror": "mirror123",
364 "ip": "1.2.3.4",
365 "country": "xy",
366 "time": datetime(2013, 7, 31, 17, 33, 8),
367 "day": 31,
368 "weekday": 2,
369 "hour": 17,
370 "month": "201307",
371 "file": "devbuilds/adblockplusie/update.json",
372 "query": "addonName=adblockplusie&addonVersion=2.0&application=msie64& applicationVersion=10.0&platform=libadblockplus&platformVersion=1.0&lastVersion= 0",
373 "size": 867,
374 "ua": "ABP",
375 "uaversion": "",
376 "fullua": "ABP ",
377 "clientid": "-",
378 "addonName": "adblockplusie",
379 "addonVersion": "2.0",
380 "fullAddon": "adblockplusie 2.0",
381 "application": "msie64",
382 "applicationVersion": "10.0",
383 "fullApplication": "msie64 10.0",
384 "platform": "libadblockplus",
385 "platformVersion": "1.0",
386 "fullPlatform": "libadblockplus 1.0",
387 "downloadInterval": "unknown",
388 "firstDownload": True,
389 "firstInDay": True,
390 "firstInWeek": True,
391 "firstInMonth": True,
392 }
393 ),
394 (
395 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /adblockplusandroid-1.1.2 .apk HTTP/1.1" 200 49152 "https://adblockplus.org/en/android-install" "Mozilla/5 .0 (Linux; U; Android 4.1.2; es-es; GT-I9100 Build/JZO54K) AppleWebKit/534.30 (K HTML, like Gecko) Version/4.0 Mobile Safari/534.30" "-" https "en-US" "downloads .adblockplus.org" "-"',
396 {
397 "mirror": "mirror123",
398 "ip": "1.2.3.4",
399 "country": "xy",
400 "time": datetime(2013, 7, 31, 17, 33, 8),
401 "day": 31,
402 "weekday": 2,
403 "hour": 17,
404 "month": "201307",
405 "file": "adblockplusandroid-1.1.2.apk",
406 "query": "",
407 "size": 49152,
408 "ua": "Mobile Safari",
409 "uaversion": "4.0",
410 "fullua": "Mobile Safari 4.0",
411 "clientid": "-",
412 "installType": "install",
413 }
414 ),
415 (
416 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /devbuilds/adblockplus/ad blockplus-2.3.2.3712.xpi?update HTTP/1.1" 200 827261 "-" "Mozilla/5.0 (Macintosh ; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0" "-" https',
417 {
418 "mirror": "mirror123",
419 "ip": "1.2.3.4",
420 "country": "xy",
421 "time": datetime(2013, 7, 31, 17, 33, 8),
422 "day": 31,
423 "weekday": 2,
424 "hour": 17,
425 "month": "201307",
426 "file": "devbuilds/adblockplus/adblockplus-2.3.2.3712.xpi",
427 "query": "update",
428 "size": 827261,
429 "ua": "Firefox",
430 "uaversion": "22.0",
431 "fullua": "Firefox 22.0",
432 "clientid": None,
433 "installType": "update",
434 }
435 ),
436 (
437 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /exceptionrules.txt?addon Name=adblockplus&addonVersion=2.3.2&application=firefox&applicationVersion=22.0& platform=gecko&platformVersion=22.0&lastVersion=201307311503 HTTP/1.1" 200 14303 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firef ox/22.0" "-" https "en-US,en;q=0.5" "easylist-downloads.adblockplus.org" "-"',
438 {
439 "mirror": "mirror123",
440 "ip": "1.2.3.4",
441 "country": "xy",
442 "time": datetime(2013, 7, 31, 17, 33, 8),
443 "day": 31,
444 "weekday": 2,
445 "hour": 17,
446 "month": "201307",
447 "file": "exceptionrules.txt",
448 "query": "addonName=adblockplus&addonVersion=2.3.2&application=firefox &applicationVersion=22.0&platform=gecko&platformVersion=22.0&lastVersion=2013073 11503",
449 "size": 14303,
450 "ua": "Firefox",
451 "uaversion": "22.0",
452 "fullua": "Firefox 22.0",
453 "clientid": "-",
454 "addonName": "adblockplus",
455 "addonVersion": "2.3.2",
456 "fullAddon": "adblockplus 2.3.2",
457 "application": "firefox",
458 "applicationVersion": "22.0",
459 "fullApplication": "firefox 22.0",
460 "platform": "gecko",
461 "platformVersion": "22.0",
462 "fullPlatform": "gecko 22.0",
463 "downloadInterval": "2 hour(s)",
464 }
465 ),
466 (
467 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylist.txt?_=137544652 8229 HTTP/1.1" 200 326120 "-" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ( KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https "nl-NL,nl;q=0.8, en-US;q=0.6,en;q=0.4" "easylist-downloads.adblockplus.org" "AdBlock/2.6.2"',
468 {
469 "mirror": "mirror123",
470 "ip": "1.2.3.4",
471 "country": "xy",
472 "time": datetime(2013, 7, 31, 17, 33, 8),
473 "day": 31,
474 "weekday": 2,
475 "hour": 17,
476 "month": "201307",
477 "file": "easylist.txt",
478 "query": "_=1375446528229",
479 "size": 326120,
480 "ua": "Chrome",
481 "uaversion": "28.0",
482 "fullua": "Chrome 28.0",
483 "clientid": "AdBlock/2.6.2",
484 "addonName": "chromeadblock",
485 "addonVersion": "2.6.2",
486 "fullAddon": "chromeadblock 2.6.2",
487 "application": "unknown",
488 "applicationVersion": "unknown",
489 "fullApplication": "unknown unknown",
490 "platform": "unknown",
491 "platformVersion": "unknown",
492 "fullPlatform": "unknown unknown",
493 "downloadInterval": "unknown",
494 }
495 ),
496 (
497 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.txt HTTP/1 .1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"',
498 {
499 "mirror": "mirror123",
500 "ip": "1.2.3.4",
501 "country": "xy",
502 "time": datetime(2013, 7, 31, 17, 33, 8),
503 "day": 31,
504 "weekday": 2,
505 "hour": 17,
506 "month": "201307",
507 "file": "easylistitaly.txt",
508 "query": "",
509 "size": 85879,
510 "ua": "Other",
511 "uaversion": "",
512 "fullua": "Other ",
513 "clientid": "-",
514 "addonName": "unknown",
515 "addonVersion": "unknown",
516 "fullAddon": "unknown unknown",
517 "application": "unknown",
518 "applicationVersion": "unknown",
519 "fullApplication": "unknown unknown",
520 "platform": "unknown",
521 "platformVersion": "unknown",
522 "fullPlatform": "unknown unknown",
523 "downloadInterval": "unknown",
524 }
525 ),
526 (
527 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /easylistitaly.tpl HTTP/1 .1" 200 85879 "-" "-" "-" https "-" "easylist-downloads.adblockplus.org" "-"',
528 {
529 "mirror": "mirror123",
530 "ip": "1.2.3.4",
531 "country": "xy",
532 "time": datetime(2013, 7, 31, 17, 33, 8),
533 "day": 31,
534 "weekday": 2,
535 "hour": 17,
536 "month": "201307",
537 "file": "easylistitaly.tpl",
538 "query": "",
539 "size": 85879,
540 "ua": "Other",
541 "uaversion": "",
542 "fullua": "Other ",
543 "clientid": "-",
544 }
545 ),
546 (
547 '1.2.3.4 - - [31/Jul/2013:12:03:08 -0530] "GET /notification.json?addonN ame=adblockpluschrome&addonVersion=1.5.3&application=chrome&applicationVersion=2 8.0.1500.72&platform=chromium&platformVersion=28.0.1500.72&lastVersion=201307292 310 HTTP/1.1" 200 299 "-" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTM L, like Gecko) Chrome/28.0.1500.72 Safari/537.36" "-" https',
548 {
549 "mirror": "mirror123",
550 "ip": "1.2.3.4",
551 "country": "xy",
552 "time": datetime(2013, 7, 31, 17, 33, 8),
553 "day": 31,
554 "weekday": 2,
555 "hour": 17,
556 "month": "201307",
557 "file": "notification.json",
558 "query": "addonName=adblockpluschrome&addonVersion=1.5.3&application=c hrome&applicationVersion=28.0.1500.72&platform=chromium&platformVersion=28.0.150 0.72&lastVersion=201307292310",
559 "size": 299,
560 "ua": "Chrome",
561 "uaversion": "28.0",
562 "fullua": "Chrome 28.0",
563 "clientid": None,
564 "addonName": "adblockpluschrome",
565 "addonVersion": "1.5.3",
566 "fullAddon": "adblockpluschrome 1.5.3",
567 "application": "chrome",
568 "applicationVersion": "28.0",
569 "fullApplication": "chrome 28.0",
570 "platform": "chromium",
571 "platformVersion": "28.0",
572 "fullPlatform": "chromium 28.0",
573 "downloadInterval": "1 day(s)",
574 "firstInDay": True,
575 }
576 ),
577 ]
578 for line, expected_record in tests:
579 logprocessor.mirror_name = "mirror123"
580 self.assertEqual(logprocessor.parse_record(line, set(), FakeGeo()), expect ed_record, "Parsing log line '%s'" % line)
581
582 def test_record_adding(self):
583 tests = [
584 (
585 {"size": 200},
586 {},
587 (),
588 {"hits": 1, "bandwidth": 200},
589 ),
590 (
591 {"size": 200},
592 {"hits": 12, "bandwidth": 30},
593 (),
594 {"hits": 13, "bandwidth": 230},
595 ),
596 (
597 {"size": 200, "ua": "Foo"},
598 {"hits": 12, "bandwidth": 30},
599 (),
600 {"hits": 13, "bandwidth": 230, "ua": {"Foo": {"hits": 1, "bandwidth": 20 0}}},
601 ),
602 (
603 {"size": 200, "ua": "Foo"},
604 {"hits": 12, "bandwidth": 30, "ua": {"Bar": {"hits": 21, "bandwidth": 12 00}}},
605 (),
606 {"hits": 13, "bandwidth": 230, "ua": {"Bar": {"hits": 21, "bandwidth": 1 200}, "Foo": {"hits": 1, "bandwidth": 200}}},
607 ),
608 (
609 {"size": 200, "ua": "Foo", "bar": "xyz"},
610 {"hits": 12, "bandwidth": 30, "ua": {"Foo": {"hits": 21, "bandwidth": 12 00}}},
611 (),
612 {"hits": 13, "bandwidth": 230, "ua": {"Foo": {"hits": 22, "bandwidth": 1 400}}},
613 ),
614 (
615 {"size": 200, "ua": "Foo", "addonName": "bar"},
616 {},
617 (),
618 {
619 "hits": 1, "bandwidth": 200,
620 "ua": {"Foo": {"hits": 1, "bandwidth": 200, "addonName": {"bar": {"hit s": 1, "bandwidth": 200}}}},
621 "addonName": {"bar": {"hits": 1, "bandwidth": 200, "ua": {"Foo": {"hit s": 1, "bandwidth": 200}}}}
622 },
623 ),
624 (
625 {"size": 200, "ua": "Foo", "addonName": "bar", "platform": "xyz"},
626 {},
627 ("platform",),
628 {
629 "hits": 1, "bandwidth": 200,
630 "ua": {"Foo": {"hits": 1, "bandwidth": 200}},
631 "addonName": {"bar": {"hits": 1, "bandwidth": 200}}
632 },
633 ),
634 ]
635 for info, section, ignored_fields, expected_result in tests:
636 logprocessor.add_record(info, section, ignored_fields)
637 self.assertEqual(section, expected_result)
638
639 if __name__ == '__main__':
640 unittest.main()
OLDNEW

Powered by Google App Engine
This is Rietveld