Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: lib/filterHits.js

Issue 6337686776315904: Issue 394 - hit statistics tool data collection (Closed)
Left Patch Set: Created April 6, 2014, 3:16 p.m.
Right Patch Set: Use Downloader to send the data to server Created April 6, 2016, 3 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « lib/contentPolicy.js ('k') | lib/prefs.json » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 /* 1 /*
2 * This file is part of Adblock Plus <http://adblockplus.org/>, 2 * This file is part of Adblock Plus <https://adblockplus.org/>,
3 * Copyright (C) 2006-2014 Eyeo GmbH 3 * Copyright (C) 2006-2016 Eyeo GmbH
4 * 4 *
5 * Adblock Plus is free software: you can redistribute it and/or modify 5 * Adblock Plus is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 3 as 6 * it under the terms of the GNU General Public License version 3 as
7 * published by the Free Software Foundation. 7 * published by the Free Software Foundation.
8 * 8 *
9 * Adblock Plus is distributed in the hope that it will be useful, 9 * Adblock Plus is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16 */ 16 */
17 17
18 let {Services} = Cu.import("resource://gre/modules/Services.jsm", null);
19 let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null);
20 let {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm", n ull);
21
18 let {IO} = require("io"); 22 let {IO} = require("io");
19 let {Utils} = require("utils");
20 let {Prefs} = require("prefs"); 23 let {Prefs} = require("prefs");
21 let {TimeLine} = require("timeline"); 24 let {Downloader, Downloadable, MILLIS_IN_SECOND, MILLIS_IN_MINUTE,
22 let {Filter} = require("filterClasses"); 25 MILLIS_IN_HOUR, MILLIS_IN_DAY} = require("downloader");
26 let {FilterNotifier} = require("filterNotifier");
27 let {DownloadableSubscription} = require("subscriptionClasses");
28
29 const CHECK_INTERVAL = MILLIS_IN_HOUR;
30 const INITIAL_DELAY = MILLIS_IN_MINUTE * 2;
31 const PUSH_INTERVAL = MILLIS_IN_DAY * 7;
23 32
24 /** 33 /**
25 * This class reads filter hits statistics from disk, manages them in memory and writes them back.W 34 * The value of storage statement normal execution constant
35 * @type Number
36 */
37 const REASON_FINISHED = Components.interfaces.mozIStorageStatementCallback.REASO N_FINISHED;
38
39 /**
40 * This class collects filter hits statistics in a SQLite database
41 * and sends them to the server when user opt-in for that
26 * @class 42 * @class
27 */ 43 */
28 let FilterHits = exports.FilterHits = 44 let FilterHits = exports.FilterHits =
29 { 45 {
30 statistics: {}, 46 /**
31 47 * Used to prevent timers running in parallel
32 _loading: false, 48 * @type Number
33 _saving: false, 49 */
34 _needsSave: false, 50 _prefToggleTimeout: 0,
35 51
36 /** 52 get connection()
37 * File that the filter hits statistics has been loaded from and should be sav ed to 53 {
38 * @type nsIFile 54 if (!this.storageFile)
39 */ 55 return null;
40 get filterHitsFile() 56
41 { 57 let connection = Services.storage.openDatabase(this.storageFile);
42 let file = null; 58 if (!connection.tableExists("filtersubscriptions"))
43 if (!file) 59 {
44 { 60 connection.executeSimpleSQL("CREATE TABLE filtersubscriptions " +
45 // Place the file in the data dir 61 "(id INTEGER, subscriptions TEXT, PRIMARY KEY" +
46 file = IO.resolveFilePath(Prefs.data_directory); 62 "(id), UNIQUE(subscriptions))");
47 if (file) 63 }
48 file.append("filter-hits.ini"); 64 if (!connection.tableExists("filterhits"))
49 } 65 {
50 if (!file) 66 connection.executeSimpleSQL("CREATE TABLE filterhits (filter TEXT, " +
51 { 67 "host TEXT, thirdParty INTEGER, hitCount INTEGER, lastHit INTEGER, " +
52 // Data directory pref misconfigured? Try the default value 68 "subscriptions INTEGER NOT NULL, PRIMARY KEY(filter, host, " +
53 try 69 "thirdParty), FOREIGN KEY(subscriptions) REFERENCES "+
54 { 70 "filtersubscriptions(id))");
55 file = IO.resolveFilePath(Services.prefs.getDefaultBranch("extensions.ad blockplus.").getCharPref("data_directory")); 71 }
56 if (file) 72 Object.defineProperty(this, "connection",
57 file.append("filter-hits.ini"); 73 {
58 } catch(e) {} 74 value: connection
59 } 75 });
60 76 return connection;
61 if (!file) 77 },
62 Cu.reportError("Adblock Plus: Failed to resolve filter-hits file"); 78
63 79 /**
64 this.__defineGetter__("filterHitsFile", function() file); 80 * @return nsIFile SQLite database file with filter hits data
65 return this.filterHitsFile; 81 */
66 }, 82 get storageFile()
67 83 {
68 /** 84 let file = IO.resolveFilePath(Prefs.data_directory);
69 * Increases the filter hit count by one 85 if (file)
86 file.append("adblockplus.sqlite");
87
88 Object.defineProperty(this, "storageFile",
89 {
90 value: file
91 });
92 return file;
93 },
94
95 /**
96 * Called on module startup.
97 */
98 init: function()
99 {
100 if (!Prefs.sendstats_status.lastPush)
101 {
102 Prefs.sendstats_status.lastPush = Date.now();
103 this._saveFilterHitPrefs();
104 }
105 Prefs.addListener(name =>
106 {
107 if (name == "sendstats")
108 {
109 if (this._prefToggleTimeout)
110 clearTimeout(this._prefToggleTimeout);
111 this._prefToggleTimeout = setTimeout(() =>
112 {
113 if (!Prefs.sendstats)
114 this.resetFilterHits();
115 }, 1000);
116 }
117 });
118
119 let downloader = new Downloader(this._getDownloadables.bind(this),
120 INITIAL_DELAY, CHECK_INTERVAL);
121 downloader.onExpirationChange = this._onExpirationChange.bind(this);
122 downloader.generateRequestData = this._generateFilterHitsData.bind(this);
123 downloader.onDownloadSuccess = this._onDownloadSuccess.bind(this);
124 downloader.validResponses.push(204);
125 onShutdown.add(() => downloader.cancel());
126 },
127
128 /**
129 * Yields a Downloadable instance for the filter hits push.
130 */
131 _getDownloadables: function*()
132 {
133 let downloadable = new Downloadable(Prefs.sendstats_url);
134 if (Prefs.sendstats_status.lastError)
135 downloadable.lastError = Prefs.sendstats_status.lastError;
136 if (Prefs.sendstats_status.lastCheck)
137 downloadable.lastCheck = Prefs.sendstats_status.lastCheck;
138 if (Prefs.sendstats_status.lastPush)
139 {
140 downloadable.softExpiration = Prefs.sendstats_status.lastPush + PUSH_INTER VAL;
saroyanm 2016/04/06 15:15:45 I'm not sure regarding this, should we specify sof
141 downloadable.hardExpiration = Prefs.sendstats_status.lastPush + PUSH_INTER VAL;
142 }
143 yield downloadable;
144 },
145
146 _onExpirationChange: function(downloadable)
147 {
148 Prefs.sendstats_status.lastCheck = downloadable.lastCheck;
149 this._saveFilterHitPrefs();
150 },
151
152 _saveFilterHitPrefs: function()
153 {
154 Prefs.sendstats_status = JSON.parse(JSON.stringify(Prefs.sendstats_status));
155 },
156
157 _onDownloadSuccess: function(downloadable, responseText, errorCallback, redire ctCallback)
158 {
159 Prefs.sendstats_status.lastError = 0;
160 Prefs.sendstats_status.lastPush = Date.now();
161 this._saveFilterHitPrefs();
162 },
163
164 /**
165 * Increases the filter hit count
70 * @param {Filter} filter 166 * @param {Filter} filter
71 * @param {Window} window Window that the match originated in (required to ge t host name) 167 * @param {String} host
72 */ 168 */
73 increaseFilterHits: function(filter, wnd) 169 increaseFilterHits: function(filter, host, thirdParty)
74 { 170 {
75 if(!filter.text) 171 let subscriptions = filter.subscriptions;
172 let downloadableSubscriptions = [];
173 for (let i = 0; i < subscriptions.length; i++)
174 {
175 if (subscriptions[i] instanceof DownloadableSubscription)
176 downloadableSubscriptions.push(subscriptions[i].url);
177 }
178
179 if (downloadableSubscriptions.length == 0)
76 return; 180 return;
77 181
78 if(filter.text in this.statistics) 182 if (!this.connection)
79 this.statistics[filter.text].statsHitCount++; 183 return;
80 else 184
81 this.statistics[filter.text] = {statsHitCount:1, lastHit: 0, domainsCount: {}}; 185 let statements = [];
82 186 let filterSubscriptions = JSON.stringify(downloadableSubscriptions);
83 if(filter.lastHit) 187 let subscriptionStatement = this.connection.createStatement("INSERT OR " +
84 this.statistics[filter.text].lastHit = filter.lastHit; 188 "IGNORE INTO filtersubscriptions (subscriptions) VALUES " +
85 189 "(:subscriptions)");
86 // Get hostname from window object 190 subscriptionStatement.params.subscriptions = filterSubscriptions;
87 let wndLocation = Utils.getOriginWindow(wnd).location.href; 191 statements.push(subscriptionStatement);
88 let host = Utils.unwrapURL(wndLocation).host; 192 let filterHitStatement = this.connection.createStatement("INSERT OR " +
89 193 "REPLACE INTO filterhits (filter, host, thirdParty, hitCount, lastHit, " +
90 this._increaseDomainHit(this.statistics[filter.text], host); 194 "subscriptions) VALUES (:filter, :host, :thirdParty, COALESCE ((SELECT " +
91 }, 195 "hitCount FROM filterhits WHERE filter=:filter AND host=:host AND " +
92 196 "thirdParty=:thirdParty), 0) + 1, :lastHit, (SELECT id " +
93 /** 197 "FROM filtersubscriptions WHERE subscriptions=:subscriptions))");
94 * Increases Domain hit count by one 198 filterHitStatement.params.filter = filter.text;
95 * @param {statisticFilterObj} statisticFilterObj statistic's filter object 199 filterHitStatement.params.host = host;
96 * @param {host} host String 200 filterHitStatement.params.lastHit = roundToHours(filter.lastHit);
97 */ 201 filterHitStatement.params.thirdParty = thirdParty;
98 _increaseDomainHit: function(statisticFilterObj, host) 202 filterHitStatement.params.subscriptions = filterSubscriptions;
99 { 203 statements.push(filterHitStatement);
100 if(host in statisticFilterObj.domainsCount) 204
101 statisticFilterObj.domainsCount[host]++; 205 this.connection.executeAsync(statements, statements.length,
102 else 206 {
103 statisticFilterObj.domainsCount[host] = 1; 207 handleError: (error) =>
104 }, 208 {
105 209 Cu.reportError("Error updating filter hits: " + error.message);
210 },
211 handleCompletion: (reason) =>
212 {
213 if (reason != REASON_FINISHED)
214 Cu.reportError("Updating filter hits canceled or aborted: " + reason);
215 }
216 });
217 },
218
219 /**
220 * Remove all local collected filter hits data
221 */
106 resetFilterHits: function() 222 resetFilterHits: function()
107 { 223 {
108 this.statistics = {}; 224 if (!this.connection)
109 },
110
111 /**
112 * send filter hits statistics to ABP sever and reset filter hits count
113 */
114 sendFilterHitsToServer: function()
115 {
116 //TODO implement ajax request to server sending this.statistics
117 //append user subscriptions and browser info before sending
118 //and reset filterhits on 200 response
119 this.resetFilterHits();
120 this.saveFilterHitsToDisk();
121 },
122
123 /**
124 * Loads Filter hit statistics from the disk
125 */
126 loadFilterHitsFromDisk: function()
127 {
128 if (this._loading)
129 return; 225 return;
130 226
131 let sourceFile = FilterHits.filterHitsFile; 227 this.connection.executeSimpleSQL("DELETE FROM filterhits");
132 if(!sourceFile) 228 this.connection.executeSimpleSQL("DELETE FROM filtersubscriptions");
133 return; 229 },
134 230
135 TimeLine.enter("Entered FilterHits.loadFilterHitsFromDisk()"); 231 _generateFilterHitsData: function(downloadable)
136 this._loading = true; 232 {
137 233 return {
138 let readFile = function() 234 then: (resolve) =>
139 { 235 {
140 TimeLine.enter("FilterHits.loadFilterHitsFromDisk() -> readFile()"); 236 if (!this.connection)
141 let parser = new FilterHitsParser(); 237 return;
142 IO.readFromFile(sourceFile, true, parser, function(e) 238
143 { 239 let statement = this.connection.createStatement("SELECT filterhits.*, " +
144 TimeLine.enter("FilterHits.loadFilterHitsFromDisk() read callback"); 240 "filtersubscriptions.subscriptions FROM filterhits LEFT JOIN " +
145 if (e) 241 "filtersubscriptions filtersubscriptions ON " +
146 Cu.reportError(e); 242 "filterhits.subscriptions=filtersubscriptions.id");
147 243 statement.executeAsync(
148 doneReading(parser);
149 }.bind(this), "FilterHitsRead");
150
151 TimeLine.leave("FilterHits.loadFilterHitsFromDisk() <- readFile()", "Filte rHitsRead");
152 }.bind(this);
153
154 var doneReading = function(parser)
155 {
156 FilterHits.statistics = parser.statistics;
157 this._loading = false;
158 TimeLine.leave("FilterHits.loadFilterHitsFromDisk() read callback done");
159 }.bind(this);
160
161 readFile(sourceFile);
162 TimeLine.leave("FilterHits.loadFilterHitsFromDisk() done");
163 },
164
165 /**
166 * Stringify filter hits statistics object
167 */
168 _generateFilterHitData: function()
169 {
170 yield JSON.stringify(FilterHits.statistics);
171 },
172
173 /**
174 * Save Filters hit statistics to the disk
175 */
176 saveFilterHitsToDisk: function()
177 {
178 let targetFile = FilterHits.filterHitsFile;
179 if (!targetFile)
180 return;
181
182 if (this._saving)
183 {
184 this._needsSave = true;
185 return;
186 }
187 TimeLine.enter("Entered FilterHits.saveFilterHitsToDisk()");
188 // Make sure the file's parent directory exists
189 try {
190 targetFile.parent.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECT ORY);
191 } catch (e) {}
192
193 let writeStats = function()
194 {
195 TimeLine.enter("FilterHits.saveFilterHitsToDisk() -> writeStats()");
196 IO.writeToFile(targetFile, true, this._generateFilterHitData(), function(e )
197 {
198 TimeLine.enter("FilterHits.saveFilterHitsToDisk() write callback");
199 this._saving = false;
200
201 if (e)
202 Cu.reportError(e);
203
204 if (this._needsSave)
205 { 244 {
206 this._needsSave = false; 245 handleResult: (result) =>
207 this.saveFilterHitsToDisk(); 246 {
208 } 247 let filters = Object.create(null);
209 TimeLine.leave("FilterHits.saveFilterHitsToDisk() write callback done"); 248 for (let row = result.getNextRow(); row;
210 }.bind(this), "FilterHitsWriteStats"); 249 row = result.getNextRow())
211 TimeLine.leave("FilterHits.saveFilterHitsToDisk() -> writeStats()", "Filte rStorageWriteStats"); 250 {
212 }.bind(this); 251 let filterText = row.getResultByName("filter");
213 252 let host = row.getResultByName("host");
214 this._saving = true; 253 let matchType = (row.getResultByName("thirdParty") ?
215 writeStats(); 254 "thirdParty" : "firstParty");
216 TimeLine.leave("FilterHits.saveFilterHitsToDisk() done"); 255 let hitCount = row.getResultByName("hitCount");
217 } 256 let lastHit = row.getResultByName("lastHit");
257 let subscriptions = row.getResultByName("subscriptions");
258
259 if (!(filterText in filters))
260 filters[filterText] = Object.create(null);
261
262 let filter = filters[filterText];
263 filter.subscriptions = subscriptions;
264 filter[matchType] = Object.create(null);
265 filter[matchType][host] = {"hits": hitCount, latest: lastHit};
266 }
267 resolve({timeSincePush: roundToHours(Date.now() -
268 Prefs.sendstats_status.lastPush), "filters": filters});
269 },
270 handleError: (error) =>
271 {
272 Prefs.sendstats_status.lastError = Date.now();
273 this._saveFilterHitPrefs();
274 Cu.reportError("Error loading filter hits:" + error.message);
275 },
276 handleCompletion: (reason) =>
277 {
278 if (reason != REASON_FINISHED)
279 {
280 Prefs.sendstats_status.lastError = Date.now();
281 this._saveFilterHitPrefs();
282 Cu.reportError("Loading of filter hits canceled or aborted: " +
283 reason);
284 }
285 }
286 });
287 }
288 }
289 },
218 }; 290 };
219 291
220 /** 292 function roundToHours(timestamp)
221 * IO.readFromFile() listener to parse filter-hits data.
222 * @constructor
223 */
224 function FilterHitsParser()
225 { 293 {
226 this.statistics = {__proto__: null}; 294 return Math.round(timestamp / MILLIS_IN_HOUR) *
295 (MILLIS_IN_HOUR / MILLIS_IN_SECOND);
227 } 296 }
228 FilterHitsParser.prototype = 297
229 { 298 FilterHits.init();
230 process: function(val)
231 {
232 if (val === null)
233 return;
234 this.statistics = JSON.parse(val);
235 }
236 };
LEFTRIGHT

Powered by Google App Engine
This is Rietveld