Left: | ||
Right: |
LEFT | RIGHT |
---|---|
1 /* | 1 /* |
2 * This file is part of Adblock Plus <https://adblockplus.org/>, | 2 * This file is part of Adblock Plus <https://adblockplus.org/>, |
3 * Copyright (C) 2006-2015 Eyeo GmbH | 3 * Copyright (C) 2006-2016 Eyeo GmbH |
4 * | 4 * |
5 * Adblock Plus is free software: you can redistribute it and/or modify | 5 * Adblock Plus is free software: you can redistribute it and/or modify |
6 * it under the terms of the GNU General Public License version 3 as | 6 * it under the terms of the GNU General Public License version 3 as |
7 * published by the Free Software Foundation. | 7 * published by the Free Software Foundation. |
8 * | 8 * |
9 * Adblock Plus is distributed in the hope that it will be useful, | 9 * Adblock Plus is distributed in the hope that it will be useful, |
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of | 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 * GNU General Public License for more details. | 12 * GNU General Public License for more details. |
13 * | 13 * |
14 * You should have received a copy of the GNU General Public License | 14 * You should have received a copy of the GNU General Public License |
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. | 15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. |
16 */ | 16 */ |
17 | 17 |
18 let {Services} = Cu.import("resource://gre/modules/Services.jsm", null); | 18 let {Services} = Cu.import("resource://gre/modules/Services.jsm", null); |
19 let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null); | 19 let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null); |
20 | 20 let {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm", n ull); |
21 | |
22 let {IO} = require("io"); | |
21 let {Prefs} = require("prefs"); | 23 let {Prefs} = require("prefs"); |
22 let {Utils} = require("utils"); | 24 let {Downloader, Downloadable, MILLIS_IN_SECOND, MILLIS_IN_MINUTE, |
23 let {MILLIS_IN_DAY} = require("downloader"); | 25 MILLIS_IN_HOUR, MILLIS_IN_DAY} = require("downloader"); |
24 let {FilterNotifier} = require("filterNotifier"); | 26 let {FilterNotifier} = require("filterNotifier"); |
25 let {DownloadableSubscription} = require("subscriptionClasses"); | 27 let {DownloadableSubscription} = require("subscriptionClasses"); |
26 | 28 |
29 const CHECK_INTERVAL = MILLIS_IN_HOUR; | |
30 const INITIAL_DELAY = MILLIS_IN_MINUTE * 2; | |
31 const PUSH_INTERVAL = MILLIS_IN_DAY * 7; | |
32 | |
27 /** | 33 /** |
28 * This class reads filter hits statistics from SQLite database, | 34 * The value of storage statement normal execution constant |
29 * manages them in memory and writes them back. | 35 * @type Number |
36 */ | |
37 const REASON_FINISHED = Components.interfaces.mozIStorageStatementCallback.REASO N_FINISHED; | |
38 | |
39 /** | |
40 * This class collects filter hits statistics in a SQLite database | |
41 * and sends them to the server when user opt-in for that | |
30 * @class | 42 * @class |
31 */ | 43 */ |
32 let FilterHits = exports.FilterHits = | 44 let FilterHits = exports.FilterHits = |
33 { | 45 { |
34 /** | 46 /** |
35 * Data that shoud be sent to the server | 47 * Used to prevent timers running in parallel |
36 * @type Object | |
37 */ | |
38 filters: Object.create(null), | |
39 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
saroyanm
2015/04/07 15:23:09
Done.
| |
40 /** | |
41 * Time since last push | |
42 * @type Number | 48 * @type Number |
43 */ | 49 */ |
44 _lastPush: 0, | 50 _prefToggleTimeout: 0, |
45 | 51 |
46 /** | 52 get connection() |
47 * Indicates the timeframe between pushes | 53 { |
48 * @type Number | 54 if (!this.storageFile) |
49 */ | 55 return null; |
50 _pushInterval: MILLIS_IN_DAY * 7, | 56 |
kzar
2015/03/23 13:11:05
I set this to 60000 (number of milliseconds in one
saroyanm
2015/03/24 10:42:40
Yes because we are saving the data to database and
kzar
2015/03/24 10:51:14
If the data submission interval is too variable it
| |
51 | 57 let connection = Services.storage.openDatabase(this.storageFile); |
52 /** | 58 if (!connection.tableExists("filtersubscriptions")) |
53 * Indicates whether the data are being loaded from storage | 59 { |
kzar
2015/03/23 13:11:05
Nit: These three comments should start with "Indic
saroyanm
2015/04/07 15:23:09
Done.
| |
54 * @type Boolean | 60 connection.executeSimpleSQL("CREATE TABLE filtersubscriptions " + |
55 */ | 61 "(id INTEGER, subscriptions TEXT, PRIMARY KEY" + |
56 _loading: false, | 62 "(id), UNIQUE(subscriptions))"); |
57 | 63 } |
58 /** | 64 if (!connection.tableExists("filterhits")) |
59 * Indicates whether the data are being saved to storage | 65 { |
60 * @type Boolean | 66 connection.executeSimpleSQL("CREATE TABLE filterhits (filter TEXT, " + |
61 */ | 67 "host TEXT, thirdParty INTEGER, hitCount INTEGER, lastHit INTEGER, " + |
62 _saving: false, | 68 "subscriptions INTEGER NOT NULL, PRIMARY KEY(filter, host, " + |
63 | 69 "thirdParty), FOREIGN KEY(subscriptions) REFERENCES "+ |
64 /** | 70 "filtersubscriptions(id))"); |
65 * Indicates whether the data are being sent to the server | 71 } |
66 * @type Boolean | 72 Object.defineProperty(this, "connection", |
67 */ | 73 { |
68 _sending: false, | 74 value: connection |
75 }); | |
76 return connection; | |
77 }, | |
78 | |
79 /** | |
80 * @return nsIFile SQLite database file with filter hits data | |
81 */ | |
82 get storageFile() | |
83 { | |
84 let file = IO.resolveFilePath(Prefs.data_directory); | |
85 if (file) | |
86 file.append("adblockplus.sqlite"); | |
87 | |
88 Object.defineProperty(this, "storageFile", | |
89 { | |
90 value: file | |
91 }); | |
92 return file; | |
93 }, | |
94 | |
95 /** | |
96 * Called on module startup. | |
97 */ | |
98 init: function() | |
99 { | |
100 if (!Prefs.sendstats_status.lastPush) | |
101 { | |
102 Prefs.sendstats_status.lastPush = Date.now(); | |
103 this._saveFilterHitPrefs(); | |
104 } | |
105 Prefs.addListener(name => | |
106 { | |
107 if (name == "sendstats") | |
108 { | |
109 if (this._prefToggleTimeout) | |
110 clearTimeout(this._prefToggleTimeout); | |
111 this._prefToggleTimeout = setTimeout(() => | |
112 { | |
113 if (!Prefs.sendstats) | |
114 this.resetFilterHits(); | |
115 }, 1000); | |
116 } | |
117 }); | |
118 | |
119 let downloader = new Downloader(this._getDownloadables.bind(this), | |
120 INITIAL_DELAY, CHECK_INTERVAL); | |
121 downloader.onExpirationChange = this._onExpirationChange.bind(this); | |
122 downloader.generateRequestData = this._generateFilterHitsData.bind(this); | |
123 downloader.onDownloadSuccess = this._onDownloadSuccess.bind(this); | |
124 downloader.validResponses.push(204); | |
125 onShutdown.add(() => downloader.cancel()); | |
126 }, | |
127 | |
128 /** | |
129 * Yields a Downloadable instance for the filter hits push. | |
130 */ | |
131 _getDownloadables: function*() | |
132 { | |
133 let downloadable = new Downloadable(Prefs.sendstats_url); | |
134 if (Prefs.sendstats_status.lastError) | |
135 downloadable.lastError = Prefs.sendstats_status.lastError; | |
136 if (Prefs.sendstats_status.lastCheck) | |
137 downloadable.lastCheck = Prefs.sendstats_status.lastCheck; | |
138 if (Prefs.sendstats_status.lastPush) | |
139 { | |
140 downloadable.softExpiration = Prefs.sendstats_status.lastPush + PUSH_INTER VAL; | |
saroyanm
2016/04/06 15:15:45
I'm not sure regarding this, should we specify sof
| |
141 downloadable.hardExpiration = Prefs.sendstats_status.lastPush + PUSH_INTER VAL; | |
142 } | |
143 yield downloadable; | |
144 }, | |
145 | |
146 _onExpirationChange: function(downloadable) | |
147 { | |
148 Prefs.sendstats_status.lastCheck = downloadable.lastCheck; | |
149 this._saveFilterHitPrefs(); | |
150 }, | |
151 | |
152 _saveFilterHitPrefs: function() | |
153 { | |
154 Prefs.sendstats_status = JSON.parse(JSON.stringify(Prefs.sendstats_status)); | |
155 }, | |
156 | |
157 _onDownloadSuccess: function(downloadable, responseText, errorCallback, redire ctCallback) | |
158 { | |
159 Prefs.sendstats_status.lastError = 0; | |
160 Prefs.sendstats_status.lastPush = Date.now(); | |
161 this._saveFilterHitPrefs(); | |
162 }, | |
69 | 163 |
70 /** | 164 /** |
71 * Increases the filter hit count | 165 * Increases the filter hit count |
72 * @param {Filter} filter | 166 * @param {Filter} filter |
73 * @param {Window} window Window that the match originated in (required to ge t host) | 167 * @param {String} host |
74 */ | 168 */ |
75 increaseFilterHits: function(filter, wnd) | 169 increaseFilterHits: function(filter, host, thirdParty) |
76 { | 170 { |
77 let subscriptions = filter.subscriptions; | 171 let subscriptions = filter.subscriptions; |
78 let inDownloadableSubscription = false; | 172 let downloadableSubscriptions = []; |
79 for (let i = 0; i < subscriptions.length; i++) | 173 for (let i = 0; i < subscriptions.length; i++) |
80 { | 174 { |
81 if (subscriptions[i] instanceof DownloadableSubscription) | 175 if (subscriptions[i] instanceof DownloadableSubscription) |
82 { | 176 downloadableSubscriptions.push(subscriptions[i].url); |
83 inDownloadableSubscription = true; | 177 } |
84 break; | 178 |
179 if (downloadableSubscriptions.length == 0) | |
180 return; | |
181 | |
182 if (!this.connection) | |
183 return; | |
184 | |
185 let statements = []; | |
186 let filterSubscriptions = JSON.stringify(downloadableSubscriptions); | |
187 let subscriptionStatement = this.connection.createStatement("INSERT OR " + | |
188 "IGNORE INTO filtersubscriptions (subscriptions) VALUES " + | |
189 "(:subscriptions)"); | |
190 subscriptionStatement.params.subscriptions = filterSubscriptions; | |
191 statements.push(subscriptionStatement); | |
192 let filterHitStatement = this.connection.createStatement("INSERT OR " + | |
193 "REPLACE INTO filterhits (filter, host, thirdParty, hitCount, lastHit, " + | |
194 "subscriptions) VALUES (:filter, :host, :thirdParty, COALESCE ((SELECT " + | |
195 "hitCount FROM filterhits WHERE filter=:filter AND host=:host AND " + | |
196 "thirdParty=:thirdParty), 0) + 1, :lastHit, (SELECT id " + | |
197 "FROM filtersubscriptions WHERE subscriptions=:subscriptions))"); | |
198 filterHitStatement.params.filter = filter.text; | |
199 filterHitStatement.params.host = host; | |
200 filterHitStatement.params.lastHit = roundToHours(filter.lastHit); | |
201 filterHitStatement.params.thirdParty = thirdParty; | |
202 filterHitStatement.params.subscriptions = filterSubscriptions; | |
203 statements.push(filterHitStatement); | |
204 | |
205 this.connection.executeAsync(statements, statements.length, | |
206 { | |
207 handleError: (error) => | |
208 { | |
209 Cu.reportError("Error updating filter hits: " + error.message); | |
210 }, | |
211 handleCompletion: (reason) => | |
212 { | |
213 if (reason != REASON_FINISHED) | |
214 Cu.reportError("Updating filter hits canceled or aborted: " + reason); | |
85 } | 215 } |
86 } | 216 }); |
87 | 217 }, |
88 if (!inDownloadableSubscription) | 218 |
219 /** | |
220 * Remove all local collected filter hits data | |
221 */ | |
222 resetFilterHits: function() | |
223 { | |
224 if (!this.connection) | |
89 return; | 225 return; |
90 | 226 |
91 if (!(filter.text in this.filters)) | 227 this.connection.executeSimpleSQL("DELETE FROM filterhits"); |
92 this.filters[filter.text] = Object.create(null); | 228 this.connection.executeSimpleSQL("DELETE FROM filtersubscriptions"); |
93 | 229 }, |
94 let statFilter = this.filters[filter.text]; | 230 |
95 let filterType = filter.thirdParty ? "thirdParty" : "firstParty"; | 231 _generateFilterHitsData: function(downloadable) |
96 | 232 { |
97 if (!(filterType in statFilter)) | 233 return { |
98 statFilter[filterType] = Object.create(null); | 234 then: (resolve) => |
99 | 235 { |
100 if (!("subscriptions" in statFilter)) | 236 if (!this.connection) |
101 statFilter.subscriptions = []; | 237 return; |
102 | 238 |
103 for (let i = 0; i < subscriptions.length; i++) | 239 let statement = this.connection.createStatement("SELECT filterhits.*, " + |
104 { | 240 "filtersubscriptions.subscriptions FROM filterhits LEFT JOIN " + |
105 if (subscriptions[i] instanceof DownloadableSubscription | 241 "filtersubscriptions filtersubscriptions ON " + |
106 && statFilter.subscriptions.indexOf(subscriptions[i].url) == -1) | 242 "filterhits.subscriptions=filtersubscriptions.id"); |
107 statFilter.subscriptions.push(subscriptions[i].url); | 243 statement.executeAsync( |
108 } | 244 { |
109 | 245 handleResult: (result) => |
110 let wndLocation = Utils.getOriginWindow(wnd).location.href; | 246 { |
111 let host = Utils.unwrapURL(wndLocation).host; | 247 let filters = Object.create(null); |
112 | 248 for (let row = result.getNextRow(); row; |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
113 if (!(host in statFilter[filterType])) | 249 row = result.getNextRow()) |
114 statFilter[filterType][host] = {hits: 1, latest: filter.lastHit}; | 250 { |
115 else | 251 let filterText = row.getResultByName("filter"); |
116 { | 252 let host = row.getResultByName("host"); |
117 statFilter[filterType][host].hits++; | 253 let matchType = (row.getResultByName("thirdParty") ? |
118 statFilter[filterType][host].latest = filter.lastHit; | 254 "thirdParty" : "firstParty"); |
119 } | 255 let hitCount = row.getResultByName("hitCount"); |
120 }, | 256 let lastHit = row.getResultByName("lastHit"); |
121 | 257 let subscriptions = row.getResultByName("subscriptions"); |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
122 resetFilterHits: function() | 258 |
123 { | 259 if (!(filterText in filters)) |
124 this.filters = Object.create(null); | 260 filters[filterText] = Object.create(null); |
125 this.saveFilterHitsToDatabase(); | 261 |
126 }, | 262 let filter = filters[filterText]; |
127 | 263 filter.subscriptions = subscriptions; |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
128 sendFilterHitsToServer: function() | 264 filter[matchType] = Object.create(null); |
129 { | 265 filter[matchType][host] = {"hits": hitCount, latest: lastHit}; |
130 if (!Prefs.sendstats) | 266 } |
131 return; | 267 resolve({timeSincePush: roundToHours(Date.now() - |
132 | 268 Prefs.sendstats_status.lastPush), "filters": filters}); |
133 let request = new XMLHttpRequest(); | 269 }, |
134 request.open("POST", Prefs.sendstats_url); | 270 handleError: (error) => |
135 request.setRequestHeader("Content-Type", "application/json"); | 271 { |
136 request.addEventListener("load", function(event) | 272 Prefs.sendstats_status.lastError = Date.now(); |
137 { | 273 this._saveFilterHitPrefs(); |
138 FilterHits._sending = false; | 274 Cu.reportError("Error loading filter hits:" + error.message); |
139 if (request.status == 200) | 275 }, |
140 { | 276 handleCompletion: (reason) => |
141 FilterHits._lastPush = new Date().getTime(); | 277 { |
142 FilterHits.resetFilterHits(); | 278 if (reason != REASON_FINISHED) |
279 { | |
280 Prefs.sendstats_status.lastError = Date.now(); | |
281 this._saveFilterHitPrefs(); | |
282 Cu.reportError("Loading of filter hits canceled or aborted: " + | |
283 reason); | |
284 } | |
285 } | |
286 }); | |
143 } | 287 } |
144 else | 288 } |
145 Cu.reportError("Could not send filter hit statistics to Adblock Plus ser ver."); | 289 }, |
146 }, false); | |
147 | |
148 let {addonName, addonVersion, application, applicationVersion, platform, pla tformVersion} = require("info"); | |
149 let data = { | |
150 version: 1, | |
151 timeSincePush: this._lastPush, | |
152 addonName: addonName, | |
153 addonVersion: addonVersion, | |
154 application: application, | |
155 applicationVersion: applicationVersion, | |
156 platform: platform, | |
157 platformVersion: platformVersion, | |
158 filters: this.filters | |
159 }; | |
160 | |
161 this._sending = true; | |
162 request.send(JSON.stringify(data)); | |
163 }, | |
164 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
165 getStorageFile: function() | |
166 { | |
167 return FileUtils.getFile("ProfD", ["adblockplus.sqlite"]); | |
168 }, | |
169 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
170 checkCreateTable: function(connection) | |
171 { | |
172 if (!connection.tableExists("filterhits")) | |
173 connection.executeSimpleSQL("CREATE TABLE filterhits (id INTEGER PRIMARY K EY, filters TEXT, date INTEGER)"); | |
174 }, | |
175 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
176 /** | |
177 * Load Filter hits from database | |
178 */ | |
179 loadFilterHitsFromDatabase: function() | |
180 { | |
181 let storageFile = this.getStorageFile(); | |
182 if (!storageFile) | |
183 return; | |
184 | |
185 let connection = Services.storage.openDatabase(storageFile); | |
186 this.checkCreateTable(connection); | |
187 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
188 let statement = connection.createStatement("SELECT * FROM filterhits"); | |
189 if (!this._loading) | |
190 { | |
191 this._loading = true; | |
192 statement.executeAsync( | |
193 { | |
194 handleResult: function(results) | |
195 { | |
196 let row = results.getNextRow(); | |
197 if (row) | |
198 { | |
199 let filters = row.getResultByName("filters"); | |
200 let lastDate = row.getResultByName("date"); | |
201 FilterHits.filters = JSON.parse(filters); | |
202 FilterHits._lastPush = lastDate; | |
203 } | |
204 }, | |
205 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
206 handleError: function(error) | |
207 { | |
208 Cu.reportError(error.message); | |
209 }, | |
210 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
211 handleCompletion: function(reason) | |
212 { | |
213 if (reason != Ci.mozIStorageStatementCallback.REASON_FINISHED) | |
214 Cu.reportError("Loading of filter hits canceled or aborted."); | |
215 FilterHits._loading = false; | |
216 } | |
217 }); | |
218 } | |
219 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
220 connection.asyncClose(); | |
221 }, | |
222 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
223 /** | |
224 * Save Filter hits to database | |
225 */ | |
226 saveFilterHitsToDatabase: function() | |
227 { | |
228 let now = new Date().getTime(); | |
229 if (!this._lastPush) | |
230 this._lastPush = now; | |
231 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
232 if (!this._sending && now - this._lastPush > this._pushInterval) | |
233 { | |
234 this.sendFilterHitsToServer(); | |
235 return; | |
236 } | |
237 | |
238 let storageFile = this.getStorageFile(); | |
239 if (!storageFile) | |
240 return; | |
241 | |
242 let connection = Services.storage.openDatabase(storageFile); | |
243 this.checkCreateTable(connection); | |
244 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
245 let statement = connection.createStatement("INSERT OR REPLACE INTO filterhit s (id, filters, date) VALUES(0, :filters, :date)"); | |
246 statement.params.filters = JSON.stringify(this.filters); | |
247 statement.params.date = this._lastPush; | |
248 if (!this._saving) | |
249 { | |
250 this._saving = true; | |
251 statement.executeAsync( | |
252 { | |
253 handleError: function(aError) | |
254 { | |
255 Cu.reportError(aError.message); | |
256 }, | |
257 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
258 handleCompletion: function(aReason) | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
259 { | |
260 if (aReason != Components.interfaces.mozIStorageStatementCallback.REAS ON_FINISHED) | |
261 Cu.reportError("Writing of filter hits canceled or aborted."); | |
262 FilterHits._saving = false; | |
263 } | |
264 }); | |
265 } | |
266 | |
kzar
2015/03/23 13:11:05
Nit: Trailing whitespace
| |
267 connection.asyncClose(); | |
268 } | |
269 }; | 290 }; |
270 | 291 |
271 FilterNotifier.addListener(function(action) | 292 function roundToHours(timestamp) |
272 { | 293 { |
273 if (action == "load" && Prefs.sendstats) | 294 return Math.round(timestamp / MILLIS_IN_HOUR) * |
274 FilterHits.loadFilterHitsFromDatabase(); | 295 (MILLIS_IN_HOUR / MILLIS_IN_SECOND); |
275 }); | 296 } |
297 | |
298 FilterHits.init(); | |
LEFT | RIGHT |