Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: lib/filterHits.js

Issue 6337686776315904: Issue 394 - hit statistics tool data collection (Closed)
Left Patch Set: Created Feb. 28, 2015, 3:01 p.m.
Right Patch Set: Use Downloader to send the data to server Created April 6, 2016, 3 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « lib/contentPolicy.js ('k') | lib/prefs.json » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 /* 1 /*
2 * This file is part of Adblock Plus <https://adblockplus.org/>, 2 * This file is part of Adblock Plus <https://adblockplus.org/>,
3 * Copyright (C) 2006-2015 Eyeo GmbH 3 * Copyright (C) 2006-2016 Eyeo GmbH
4 * 4 *
5 * Adblock Plus is free software: you can redistribute it and/or modify 5 * Adblock Plus is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 3 as 6 * it under the terms of the GNU General Public License version 3 as
7 * published by the Free Software Foundation. 7 * published by the Free Software Foundation.
8 * 8 *
9 * Adblock Plus is distributed in the hope that it will be useful, 9 * Adblock Plus is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16 */ 16 */
17 17
18 let {Services} = Cu.import("resource://gre/modules/Services.jsm", null); 18 let {Services} = Cu.import("resource://gre/modules/Services.jsm", null);
19 let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null); 19 let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null);
20 20 let {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm", n ull);
21
22 let {IO} = require("io");
21 let {Prefs} = require("prefs"); 23 let {Prefs} = require("prefs");
22 let {Utils} = require("utils"); 24 let {Downloader, Downloadable, MILLIS_IN_SECOND, MILLIS_IN_MINUTE,
23 let {MILLIS_IN_DAY} = require("downloader"); 25 MILLIS_IN_HOUR, MILLIS_IN_DAY} = require("downloader");
24 let {FilterNotifier} = require("filterNotifier"); 26 let {FilterNotifier} = require("filterNotifier");
25 let {DownloadableSubscription} = require("subscriptionClasses"); 27 let {DownloadableSubscription} = require("subscriptionClasses");
26 28
29 const CHECK_INTERVAL = MILLIS_IN_HOUR;
30 const INITIAL_DELAY = MILLIS_IN_MINUTE * 2;
31 const PUSH_INTERVAL = MILLIS_IN_DAY * 7;
32
27 /** 33 /**
28 * This class reads filter hits statistics from SQLite database, 34 * The value of storage statement normal execution constant
29 * manages them in memory and writes them back. 35 * @type Number
36 */
37 const REASON_FINISHED = Components.interfaces.mozIStorageStatementCallback.REASO N_FINISHED;
38
39 /**
40 * This class collects filter hits statistics in a SQLite database
41 * and sends them to the server when user opt-in for that
30 * @class 42 * @class
31 */ 43 */
32 let FilterHits = exports.FilterHits = 44 let FilterHits = exports.FilterHits =
33 { 45 {
34 /** 46 /**
35 * Data that shoud be sent to the server 47 * Used to prevent timers running in parallel
36 * @type Object
37 */
38 filters: Object.create(null),
39
40 /**
41 * The Service URL that should handle the data
42 * @type String
43 */
44 _serviceURL: "",
45
46 /**
47 * Time since last push
48 * @type Number 48 * @type Number
49 */ 49 */
50 _lastPush: 0, 50 _prefToggleTimeout: 0,
51 51
52 /** 52 get connection()
53 * Indicates the timeframe between pushes 53 {
54 * @type Number 54 if (!this.storageFile)
55 */ 55 return null;
56 _pushInterval: MILLIS_IN_DAY * 7, 56
57 57 let connection = Services.storage.openDatabase(this.storageFile);
58 /** 58 if (!connection.tableExists("filtersubscriptions"))
59 * Indicates whether the data are being loaded from storage 59 {
60 * @type Boolean 60 connection.executeSimpleSQL("CREATE TABLE filtersubscriptions " +
61 */ 61 "(id INTEGER, subscriptions TEXT, PRIMARY KEY" +
62 _loading: false, 62 "(id), UNIQUE(subscriptions))");
63 63 }
64 /** 64 if (!connection.tableExists("filterhits"))
65 * Indicates whether the data are being saved to storage 65 {
66 * @type Boolean 66 connection.executeSimpleSQL("CREATE TABLE filterhits (filter TEXT, " +
67 */ 67 "host TEXT, thirdParty INTEGER, hitCount INTEGER, lastHit INTEGER, " +
68 _saving: false, 68 "subscriptions INTEGER NOT NULL, PRIMARY KEY(filter, host, " +
69 69 "thirdParty), FOREIGN KEY(subscriptions) REFERENCES "+
70 /** 70 "filtersubscriptions(id))");
71 * Indicates whether the data are being sent to the server 71 }
72 * @type Boolean 72 Object.defineProperty(this, "connection",
73 */ 73 {
74 _sending: false, 74 value: connection
75 });
76 return connection;
77 },
78
79 /**
80 * @return nsIFile SQLite database file with filter hits data
81 */
82 get storageFile()
83 {
84 let file = IO.resolveFilePath(Prefs.data_directory);
85 if (file)
86 file.append("adblockplus.sqlite");
87
88 Object.defineProperty(this, "storageFile",
89 {
90 value: file
91 });
92 return file;
93 },
94
95 /**
96 * Called on module startup.
97 */
98 init: function()
99 {
100 if (!Prefs.sendstats_status.lastPush)
101 {
102 Prefs.sendstats_status.lastPush = Date.now();
103 this._saveFilterHitPrefs();
104 }
105 Prefs.addListener(name =>
106 {
107 if (name == "sendstats")
108 {
109 if (this._prefToggleTimeout)
110 clearTimeout(this._prefToggleTimeout);
111 this._prefToggleTimeout = setTimeout(() =>
112 {
113 if (!Prefs.sendstats)
114 this.resetFilterHits();
115 }, 1000);
116 }
117 });
118
119 let downloader = new Downloader(this._getDownloadables.bind(this),
120 INITIAL_DELAY, CHECK_INTERVAL);
121 downloader.onExpirationChange = this._onExpirationChange.bind(this);
122 downloader.generateRequestData = this._generateFilterHitsData.bind(this);
123 downloader.onDownloadSuccess = this._onDownloadSuccess.bind(this);
124 downloader.validResponses.push(204);
125 onShutdown.add(() => downloader.cancel());
126 },
127
128 /**
129 * Yields a Downloadable instance for the filter hits push.
130 */
131 _getDownloadables: function*()
132 {
133 let downloadable = new Downloadable(Prefs.sendstats_url);
134 if (Prefs.sendstats_status.lastError)
135 downloadable.lastError = Prefs.sendstats_status.lastError;
136 if (Prefs.sendstats_status.lastCheck)
137 downloadable.lastCheck = Prefs.sendstats_status.lastCheck;
138 if (Prefs.sendstats_status.lastPush)
139 {
140 downloadable.softExpiration = Prefs.sendstats_status.lastPush + PUSH_INTER VAL;
saroyanm 2016/04/06 15:15:45 I'm not sure regarding this, should we specify sof
141 downloadable.hardExpiration = Prefs.sendstats_status.lastPush + PUSH_INTER VAL;
142 }
143 yield downloadable;
144 },
145
146 _onExpirationChange: function(downloadable)
147 {
148 Prefs.sendstats_status.lastCheck = downloadable.lastCheck;
149 this._saveFilterHitPrefs();
150 },
151
152 _saveFilterHitPrefs: function()
153 {
154 Prefs.sendstats_status = JSON.parse(JSON.stringify(Prefs.sendstats_status));
155 },
156
157 _onDownloadSuccess: function(downloadable, responseText, errorCallback, redire ctCallback)
158 {
159 Prefs.sendstats_status.lastError = 0;
160 Prefs.sendstats_status.lastPush = Date.now();
161 this._saveFilterHitPrefs();
162 },
75 163
76 /** 164 /**
77 * Increases the filter hit count 165 * Increases the filter hit count
78 * @param {Filter} filter 166 * @param {Filter} filter
79 * @param {Window} window Window that the match originated in (required to ge t host) 167 * @param {String} host
80 */ 168 */
81 increaseFilterHits: function(filter, wnd) 169 increaseFilterHits: function(filter, host, thirdParty)
82 { 170 {
83 let subscriptionTitles = []; 171 let subscriptions = filter.subscriptions;
84 for (let i = 0; i < filter.subscriptions.length; i++) 172 let downloadableSubscriptions = [];
85 { 173 for (let i = 0; i < subscriptions.length; i++)
86 if (filter.subscriptions[i] instanceof DownloadableSubscription) 174 {
87 subscriptionTitles.push(filter.subscriptions[i]._title) 175 if (subscriptions[i] instanceof DownloadableSubscription)
Thomas Greiner 2015/03/06 11:29:19 See https://issues.adblockplus.org/ticket/394#comm
saroyanm 2015/03/06 16:54:04 Ohh, so stupid from my side :/ Done.
88 } 176 downloadableSubscriptions.push(subscriptions[i].url);
89 177 }
90 if (subscriptionTitles.length == 0) 178
179 if (downloadableSubscriptions.length == 0)
91 return; 180 return;
Thomas Greiner 2015/03/06 11:29:19 Rather than constructing an array on each function
saroyanm 2015/03/06 16:54:04 Done.
92 181
93 if (!(filter.text in this.filters)) 182 if (!this.connection)
94 this.filters[filter.text] = Object.create(null); 183 return;
95 184
96 let statFilter = this.filters[filter.text]; 185 let statements = [];
97 let filterType = filter.thirdParty ? "thirdParty" : "firstParty"; 186 let filterSubscriptions = JSON.stringify(downloadableSubscriptions);
98 187 let subscriptionStatement = this.connection.createStatement("INSERT OR " +
99 if (!(filterType in statFilter)) 188 "IGNORE INTO filtersubscriptions (subscriptions) VALUES " +
100 statFilter[filterType] = Object.create(null); 189 "(:subscriptions)");
101 190 subscriptionStatement.params.subscriptions = filterSubscriptions;
102 if (!("subscriptions" in statFilter)) 191 statements.push(subscriptionStatement);
103 statFilter.subscriptions = []; 192 let filterHitStatement = this.connection.createStatement("INSERT OR " +
104 193 "REPLACE INTO filterhits (filter, host, thirdParty, hitCount, lastHit, " +
105 for (let i = 0; i < subscriptionTitles.length; i++) 194 "subscriptions) VALUES (:filter, :host, :thirdParty, COALESCE ((SELECT " +
106 { 195 "hitCount FROM filterhits WHERE filter=:filter AND host=:host AND " +
107 if (statFilter.subscriptions.indexOf(subscriptionTitles[i]) == -1) 196 "thirdParty=:thirdParty), 0) + 1, :lastHit, (SELECT id " +
108 statFilter.subscriptions.push(subscriptionTitles[i]); 197 "FROM filtersubscriptions WHERE subscriptions=:subscriptions))");
109 } 198 filterHitStatement.params.filter = filter.text;
110 199 filterHitStatement.params.host = host;
111 let wndLocation = Utils.getOriginWindow(wnd).location.href; 200 filterHitStatement.params.lastHit = roundToHours(filter.lastHit);
112 let host = Utils.unwrapURL(wndLocation).host; 201 filterHitStatement.params.thirdParty = thirdParty;
113 202 filterHitStatement.params.subscriptions = filterSubscriptions;
114 if (!(host in statFilter[filterType])) 203 statements.push(filterHitStatement);
115 statFilter[filterType][host] = {hits: 1, latest: filter.lastHit}; 204
116 else 205 this.connection.executeAsync(statements, statements.length,
117 { 206 {
118 statFilter[filterType][host].hits++; 207 handleError: (error) =>
119 statFilter[filterType][host].latest = filter.lastHit; 208 {
120 } 209 Cu.reportError("Error updating filter hits: " + error.message);
121 }, 210 },
122 211 handleCompletion: (reason) =>
212 {
213 if (reason != REASON_FINISHED)
214 Cu.reportError("Updating filter hits canceled or aborted: " + reason);
215 }
216 });
217 },
218
219 /**
220 * Remove all local collected filter hits data
221 */
123 resetFilterHits: function() 222 resetFilterHits: function()
124 { 223 {
125 this.filters = Object.create(null); 224 if (!this.connection)
126 this.saveFilterHitsToDatabase(); 225 return;
127 }, 226
128 227 this.connection.executeSimpleSQL("DELETE FROM filterhits");
129 sendFilterHitsToServer: function() 228 this.connection.executeSimpleSQL("DELETE FROM filtersubscriptions");
130 { 229 },
131 let request = new XMLHttpRequest(); 230
132 request.open("POST", this._serviceURL); 231 _generateFilterHitsData: function(downloadable)
133 request.setRequestHeader("Content-Type", "application/json"); 232 {
134 request.addEventListener("load", function(event) 233 return {
135 { 234 then: (resolve) =>
136 FilterHits._sending = false; 235 {
137 if (request.status == 200) 236 if (!this.connection)
138 { 237 return;
139 FilterHits._lastPush = new Date().getTime(); 238
140 FilterHits.resetFilterHits(); 239 let statement = this.connection.createStatement("SELECT filterhits.*, " +
240 "filtersubscriptions.subscriptions FROM filterhits LEFT JOIN " +
241 "filtersubscriptions filtersubscriptions ON " +
242 "filterhits.subscriptions=filtersubscriptions.id");
243 statement.executeAsync(
244 {
245 handleResult: (result) =>
246 {
247 let filters = Object.create(null);
248 for (let row = result.getNextRow(); row;
249 row = result.getNextRow())
250 {
251 let filterText = row.getResultByName("filter");
252 let host = row.getResultByName("host");
253 let matchType = (row.getResultByName("thirdParty") ?
254 "thirdParty" : "firstParty");
255 let hitCount = row.getResultByName("hitCount");
256 let lastHit = row.getResultByName("lastHit");
257 let subscriptions = row.getResultByName("subscriptions");
258
259 if (!(filterText in filters))
260 filters[filterText] = Object.create(null);
261
262 let filter = filters[filterText];
263 filter.subscriptions = subscriptions;
264 filter[matchType] = Object.create(null);
265 filter[matchType][host] = {"hits": hitCount, latest: lastHit};
266 }
267 resolve({timeSincePush: roundToHours(Date.now() -
268 Prefs.sendstats_status.lastPush), "filters": filters});
269 },
270 handleError: (error) =>
271 {
272 Prefs.sendstats_status.lastError = Date.now();
273 this._saveFilterHitPrefs();
274 Cu.reportError("Error loading filter hits:" + error.message);
275 },
276 handleCompletion: (reason) =>
277 {
278 if (reason != REASON_FINISHED)
279 {
280 Prefs.sendstats_status.lastError = Date.now();
281 this._saveFilterHitPrefs();
282 Cu.reportError("Loading of filter hits canceled or aborted: " +
283 reason);
284 }
285 }
286 });
141 } 287 }
142 else 288 }
143 Cu.reportError("Could not send filter hit statistics to Adblock Plus ser ver."); 289 },
144 }, false);
145
146 let {addonName, addonVersion, application, applicationVersion, platform, pla tformVersion} = require("info");
147 let data = {
148 version: 1,
149 timeSincePush: this._lastPush,
150 addonName: addonName,
151 addonVersion: addonVersion,
152 application: application,
153 applicationVersion: applicationVersion,
154 platform: platform,
155 platformVersion: platformVersion,
156 filters: this.filters
157 };
158
159 this._sending = true;
160 request.send(JSON.stringify(data));
161 },
162
163 getStorageFile: function()
164 {
165 return FileUtils.getFile("ProfD", ["adblockplus.sqlite"]);
166 },
167
168 checkCreateTable: function(connection)
169 {
170 if (!connection.tableExists("filterhits"))
171 connection.executeSimpleSQL("CREATE TABLE filterhits (id INTEGER PRIMARY K EY, filters TEXT, date INTEGER)");
172 },
173
174 /**
175 * Load Filter hits from database
176 */
177 loadFilterHitsFromDatabase: function()
178 {
179 let storageFile = this.getStorageFile();
180 if (!storageFile)
181 return;
182
183 let connection = Services.storage.openDatabase(storageFile);
184 this.checkCreateTable(connection);
185
186 let statement = connection.createStatement("SELECT * FROM filterhits");
187 if (!this._loading)
188 {
189 this._loading = true;
190 statement.executeAsync(
191 {
192 handleResult: function(results)
193 {
194 let row = results.getNextRow();
195 if (row)
196 {
197 let filters = row.getResultByName("filters");
198 let lastDate = row.getResultByName("date");
199 FilterHits.filters = JSON.parse(filters);
200 FilterHits._lastPush = lastDate;
201 }
202 },
203
204 handleError: function(error)
205 {
206 Cu.reportError(error.message);
207 },
208
209 handleCompletion: function(reason)
210 {
211 if (reason != Ci.mozIStorageStatementCallback.REASON_FINISHED)
212 Cu.reportError("Loading of filter hits canceled or aborted.");
213 FilterHits._loading = false;
214 }
215 });
216 }
217
218 connection.asyncClose();
219 },
220
221 /**
222 * Save Filter hits to database
223 */
224 saveFilterHitsToDatabase: function()
225 {
226 let now = new Date().getTime();
227 if (!this._lastPush)
228 this._lastPush = now;
229
230 if (!this._sending && now - this._lastPush > this._pushInterval)
231 {
232 this.sendFilterHitsToServer();
233 return;
234 }
235
236 let storageFile = this.getStorageFile();
237 if (!storageFile)
238 return;
239
240 let connection = Services.storage.openDatabase(storageFile);
241 this.checkCreateTable(connection);
242
243 let statement = connection.createStatement("INSERT OR REPLACE INTO filterhit s (id, filters, date) VALUES(0, :filters, :date)");
244 statement.params.filters = JSON.stringify(this.filters);
245 statement.params.date = this._lastPush;
246 if (!this._saving)
247 {
248 this._saving = true;
249 statement.executeAsync(
250 {
251 handleError: function(aError)
252 {
253 Cu.reportError(aError.message);
254 },
255
256 handleCompletion: function(aReason)
257 {
258 if (aReason != Components.interfaces.mozIStorageStatementCallback.REAS ON_FINISHED)
259 Cu.reportError("Writing of filter hits canceled or aborted.");
260 FilterHits._saving = false;
261 }
262 });
263 }
264
265 connection.asyncClose();
266 }
267 }; 290 };
268 291
269 FilterNotifier.addListener(function(action) 292 function roundToHours(timestamp)
270 { 293 {
271 if (action == "load" && Prefs.sendstats) 294 return Math.round(timestamp / MILLIS_IN_HOUR) *
272 FilterHits.loadFilterHitsFromDatabase(); 295 (MILLIS_IN_HOUR / MILLIS_IN_SECOND);
273 }); 296 }
297
298 FilterHits.init();
LEFTRIGHT

Powered by Google App Engine
This is Rietveld