Left: | ||
Right: |
LEFT | RIGHT |
---|---|
1 /* | 1 /* |
2 * This file is part of Adblock Plus <https://adblockplus.org/>, | 2 * This file is part of Adblock Plus <https://adblockplus.org/>, |
3 * Copyright (C) 2006-2015 Eyeo GmbH | 3 * Copyright (C) 2006-2016 Eyeo GmbH |
4 * | 4 * |
5 * Adblock Plus is free software: you can redistribute it and/or modify | 5 * Adblock Plus is free software: you can redistribute it and/or modify |
6 * it under the terms of the GNU General Public License version 3 as | 6 * it under the terms of the GNU General Public License version 3 as |
7 * published by the Free Software Foundation. | 7 * published by the Free Software Foundation. |
8 * | 8 * |
9 * Adblock Plus is distributed in the hope that it will be useful, | 9 * Adblock Plus is distributed in the hope that it will be useful, |
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of | 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 * GNU General Public License for more details. | 12 * GNU General Public License for more details. |
13 * | 13 * |
14 * You should have received a copy of the GNU General Public License | 14 * You should have received a copy of the GNU General Public License |
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. | 15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. |
16 */ | 16 */ |
17 | 17 |
18 let {Services} = Cu.import("resource://gre/modules/Services.jsm", null); | 18 let {Services} = Cu.import("resource://gre/modules/Services.jsm", null); |
19 let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null); | 19 let {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", null); |
20 | 20 let {setTimeout, clearTimeout} = Cu.import("resource://gre/modules/Timer.jsm", n ull); |
21 | |
22 let {IO} = require("io"); | |
21 let {Prefs} = require("prefs"); | 23 let {Prefs} = require("prefs"); |
22 let {Utils} = require("utils"); | 24 let {Downloader, Downloadable, MILLIS_IN_SECOND, MILLIS_IN_MINUTE, |
23 let {MILLIS_IN_DAY} = require("downloader"); | 25 MILLIS_IN_HOUR, MILLIS_IN_DAY} = require("downloader"); |
24 let {FilterNotifier} = require("filterNotifier"); | 26 let {FilterNotifier} = require("filterNotifier"); |
25 let {DownloadableSubscription} = require("subscriptionClasses"); | 27 let {DownloadableSubscription} = require("subscriptionClasses"); |
26 | 28 |
29 const CHECK_INTERVAL = MILLIS_IN_HOUR; | |
30 const INITIAL_DELAY = MILLIS_IN_MINUTE * 2; | |
31 const PUSH_INTERVAL = MILLIS_IN_DAY * 7; | |
32 | |
27 /** | 33 /** |
28 * This class reads filter hits statistics from SQLite database, | 34 * The value of storage statement normal execution constant |
29 * manages them in memory and writes them back. | 35 * @type Number |
36 */ | |
37 const REASON_FINISHED = Components.interfaces.mozIStorageStatementCallback.REASO N_FINISHED; | |
38 | |
39 /** | |
40 * This class collects filter hits statistics in a SQLite database | |
41 * and sends them to the server when user opt-in for that | |
30 * @class | 42 * @class |
31 */ | 43 */ |
32 let FilterHits = exports.FilterHits = | 44 let FilterHits = exports.FilterHits = |
33 { | 45 { |
34 /** | 46 /** |
35 * Data that shoud be sent to the server | 47 * Used to prevent timers running in parallel |
36 * @type Object | |
37 */ | |
38 filters: Object.create(null), | |
39 | |
40 /** | |
41 * The Service URL that should handle the data | |
42 * @type String | |
43 */ | |
44 _serviceURL: "", | |
45 | |
46 /** | |
47 * Time since last push | |
48 * @type Number | 48 * @type Number |
49 */ | 49 */ |
50 _lastPush: 0, | 50 _prefToggleTimeout: 0, |
51 | 51 |
52 /** | 52 get connection() |
53 * Indicates the timeframe between pushes | 53 { |
54 * @type Number | 54 if (!this.storageFile) |
55 */ | 55 return null; |
56 _pushInterval: MILLIS_IN_DAY * 7, | 56 |
57 | 57 let connection = Services.storage.openDatabase(this.storageFile); |
58 /** | 58 if (!connection.tableExists("filtersubscriptions")) |
59 * Indicates whether the data are being loaded from storage | 59 { |
60 * @type Boolean | 60 connection.executeSimpleSQL("CREATE TABLE filtersubscriptions " + |
61 */ | 61 "(id INTEGER, subscriptions TEXT, PRIMARY KEY" + |
62 _loading: false, | 62 "(id), UNIQUE(subscriptions))"); |
63 | 63 } |
64 /** | 64 if (!connection.tableExists("filterhits")) |
65 * Indicates whether the data are being saved to storage | 65 { |
66 * @type Boolean | 66 connection.executeSimpleSQL("CREATE TABLE filterhits (filter TEXT, " + |
67 */ | 67 "host TEXT, thirdParty INTEGER, hitCount INTEGER, lastHit INTEGER, " + |
68 _saving: false, | 68 "subscriptions INTEGER NOT NULL, PRIMARY KEY(filter, host, " + |
69 | 69 "thirdParty), FOREIGN KEY(subscriptions) REFERENCES "+ |
70 /** | 70 "filtersubscriptions(id))"); |
71 * Indicates whether the data are being sent to the server | 71 } |
72 * @type Boolean | 72 Object.defineProperty(this, "connection", |
73 */ | 73 { |
74 _sending: false, | 74 value: connection |
75 }); | |
76 return connection; | |
77 }, | |
78 | |
79 /** | |
80 * @return nsIFile SQLite database file with filter hits data | |
81 */ | |
82 get storageFile() | |
83 { | |
84 let file = IO.resolveFilePath(Prefs.data_directory); | |
85 if (file) | |
86 file.append("adblockplus.sqlite"); | |
87 | |
88 Object.defineProperty(this, "storageFile", | |
89 { | |
90 value: file | |
91 }); | |
92 return file; | |
93 }, | |
94 | |
95 /** | |
96 * Called on module startup. | |
97 */ | |
98 init: function() | |
99 { | |
100 if (!Prefs.sendstats_status.lastPush) | |
101 { | |
102 Prefs.sendstats_status.lastPush = Date.now(); | |
103 this._saveFilterHitPrefs(); | |
104 } | |
105 Prefs.addListener(name => | |
106 { | |
107 if (name == "sendstats") | |
108 { | |
109 if (this._prefToggleTimeout) | |
110 clearTimeout(this._prefToggleTimeout); | |
111 this._prefToggleTimeout = setTimeout(() => | |
112 { | |
113 if (!Prefs.sendstats) | |
114 this.resetFilterHits(); | |
115 }, 1000); | |
116 } | |
117 }); | |
118 | |
119 let downloader = new Downloader(this._getDownloadables.bind(this), | |
120 INITIAL_DELAY, CHECK_INTERVAL); | |
121 downloader.onExpirationChange = this._onExpirationChange.bind(this); | |
122 downloader.generateRequestData = this._generateFilterHitsData.bind(this); | |
123 downloader.onDownloadSuccess = this._onDownloadSuccess.bind(this); | |
124 downloader.validResponses.push(204); | |
125 onShutdown.add(() => downloader.cancel()); | |
126 }, | |
127 | |
128 /** | |
129 * Yields a Downloadable instance for the filter hits push. | |
130 */ | |
131 _getDownloadables: function*() | |
132 { | |
133 let downloadable = new Downloadable(Prefs.sendstats_url); | |
134 if (Prefs.sendstats_status.lastError) | |
135 downloadable.lastError = Prefs.sendstats_status.lastError; | |
136 if (Prefs.sendstats_status.lastCheck) | |
137 downloadable.lastCheck = Prefs.sendstats_status.lastCheck; | |
138 if (Prefs.sendstats_status.lastPush) | |
139 { | |
140 downloadable.softExpiration = Prefs.sendstats_status.lastPush + PUSH_INTER VAL; | |
saroyanm
2016/04/06 15:15:45
I'm not sure regarding this, should we specify sof
| |
141 downloadable.hardExpiration = Prefs.sendstats_status.lastPush + PUSH_INTER VAL; | |
142 } | |
143 yield downloadable; | |
144 }, | |
145 | |
146 _onExpirationChange: function(downloadable) | |
147 { | |
148 Prefs.sendstats_status.lastCheck = downloadable.lastCheck; | |
149 this._saveFilterHitPrefs(); | |
150 }, | |
151 | |
152 _saveFilterHitPrefs: function() | |
153 { | |
154 Prefs.sendstats_status = JSON.parse(JSON.stringify(Prefs.sendstats_status)); | |
155 }, | |
156 | |
157 _onDownloadSuccess: function(downloadable, responseText, errorCallback, redire ctCallback) | |
158 { | |
159 Prefs.sendstats_status.lastError = 0; | |
160 Prefs.sendstats_status.lastPush = Date.now(); | |
161 this._saveFilterHitPrefs(); | |
162 }, | |
75 | 163 |
76 /** | 164 /** |
77 * Increases the filter hit count | 165 * Increases the filter hit count |
78 * @param {Filter} filter | 166 * @param {Filter} filter |
79 * @param {Window} window Window that the match originated in (required to ge t host) | 167 * @param {String} host |
80 */ | 168 */ |
81 increaseFilterHits: function(filter, wnd) | 169 increaseFilterHits: function(filter, host, thirdParty) |
82 { | 170 { |
83 let subscriptionTitles = []; | 171 let subscriptions = filter.subscriptions; |
84 for (let i = 0; i < filter.subscriptions.length; i++) | 172 let downloadableSubscriptions = []; |
85 { | 173 for (let i = 0; i < subscriptions.length; i++) |
86 if (filter.subscriptions[i] instanceof DownloadableSubscription) | 174 { |
87 subscriptionTitles.push(filter.subscriptions[i]._title) | 175 if (subscriptions[i] instanceof DownloadableSubscription) |
Thomas Greiner
2015/03/06 11:29:19
See https://issues.adblockplus.org/ticket/394#comm
saroyanm
2015/03/06 16:54:04
Ohh, so stupid from my side :/
Done.
| |
88 } | 176 downloadableSubscriptions.push(subscriptions[i].url); |
89 | 177 } |
90 if (subscriptionTitles.length == 0) | 178 |
179 if (downloadableSubscriptions.length == 0) | |
91 return; | 180 return; |
Thomas Greiner
2015/03/06 11:29:19
Rather than constructing an array on each function
saroyanm
2015/03/06 16:54:04
Done.
| |
92 | 181 |
93 if (!(filter.text in this.filters)) | 182 if (!this.connection) |
94 this.filters[filter.text] = Object.create(null); | 183 return; |
95 | 184 |
96 let statFilter = this.filters[filter.text]; | 185 let statements = []; |
97 let filterType = filter.thirdParty ? "thirdParty" : "firstParty"; | 186 let filterSubscriptions = JSON.stringify(downloadableSubscriptions); |
98 | 187 let subscriptionStatement = this.connection.createStatement("INSERT OR " + |
99 if (!(filterType in statFilter)) | 188 "IGNORE INTO filtersubscriptions (subscriptions) VALUES " + |
100 statFilter[filterType] = Object.create(null); | 189 "(:subscriptions)"); |
101 | 190 subscriptionStatement.params.subscriptions = filterSubscriptions; |
102 if (!("subscriptions" in statFilter)) | 191 statements.push(subscriptionStatement); |
103 statFilter.subscriptions = []; | 192 let filterHitStatement = this.connection.createStatement("INSERT OR " + |
104 | 193 "REPLACE INTO filterhits (filter, host, thirdParty, hitCount, lastHit, " + |
105 for (let i = 0; i < subscriptionTitles.length; i++) | 194 "subscriptions) VALUES (:filter, :host, :thirdParty, COALESCE ((SELECT " + |
106 { | 195 "hitCount FROM filterhits WHERE filter=:filter AND host=:host AND " + |
107 if (statFilter.subscriptions.indexOf(subscriptionTitles[i]) == -1) | 196 "thirdParty=:thirdParty), 0) + 1, :lastHit, (SELECT id " + |
108 statFilter.subscriptions.push(subscriptionTitles[i]); | 197 "FROM filtersubscriptions WHERE subscriptions=:subscriptions))"); |
109 } | 198 filterHitStatement.params.filter = filter.text; |
110 | 199 filterHitStatement.params.host = host; |
111 let wndLocation = Utils.getOriginWindow(wnd).location.href; | 200 filterHitStatement.params.lastHit = roundToHours(filter.lastHit); |
112 let host = Utils.unwrapURL(wndLocation).host; | 201 filterHitStatement.params.thirdParty = thirdParty; |
113 | 202 filterHitStatement.params.subscriptions = filterSubscriptions; |
114 if (!(host in statFilter[filterType])) | 203 statements.push(filterHitStatement); |
115 statFilter[filterType][host] = {hits: 1, latest: filter.lastHit}; | 204 |
116 else | 205 this.connection.executeAsync(statements, statements.length, |
117 { | 206 { |
118 statFilter[filterType][host].hits++; | 207 handleError: (error) => |
119 statFilter[filterType][host].latest = filter.lastHit; | 208 { |
120 } | 209 Cu.reportError("Error updating filter hits: " + error.message); |
121 }, | 210 }, |
122 | 211 handleCompletion: (reason) => |
212 { | |
213 if (reason != REASON_FINISHED) | |
214 Cu.reportError("Updating filter hits canceled or aborted: " + reason); | |
215 } | |
216 }); | |
217 }, | |
218 | |
219 /** | |
220 * Remove all local collected filter hits data | |
221 */ | |
123 resetFilterHits: function() | 222 resetFilterHits: function() |
124 { | 223 { |
125 this.filters = Object.create(null); | 224 if (!this.connection) |
126 this.saveFilterHitsToDatabase(); | 225 return; |
127 }, | 226 |
128 | 227 this.connection.executeSimpleSQL("DELETE FROM filterhits"); |
129 sendFilterHitsToServer: function() | 228 this.connection.executeSimpleSQL("DELETE FROM filtersubscriptions"); |
130 { | 229 }, |
131 let request = new XMLHttpRequest(); | 230 |
132 request.open("POST", this._serviceURL); | 231 _generateFilterHitsData: function(downloadable) |
133 request.setRequestHeader("Content-Type", "application/json"); | 232 { |
134 request.addEventListener("load", function(event) | 233 return { |
135 { | 234 then: (resolve) => |
136 FilterHits._sending = false; | 235 { |
137 if (request.status == 200) | 236 if (!this.connection) |
138 { | 237 return; |
139 FilterHits._lastPush = new Date().getTime(); | 238 |
140 FilterHits.resetFilterHits(); | 239 let statement = this.connection.createStatement("SELECT filterhits.*, " + |
240 "filtersubscriptions.subscriptions FROM filterhits LEFT JOIN " + | |
241 "filtersubscriptions filtersubscriptions ON " + | |
242 "filterhits.subscriptions=filtersubscriptions.id"); | |
243 statement.executeAsync( | |
244 { | |
245 handleResult: (result) => | |
246 { | |
247 let filters = Object.create(null); | |
248 for (let row = result.getNextRow(); row; | |
249 row = result.getNextRow()) | |
250 { | |
251 let filterText = row.getResultByName("filter"); | |
252 let host = row.getResultByName("host"); | |
253 let matchType = (row.getResultByName("thirdParty") ? | |
254 "thirdParty" : "firstParty"); | |
255 let hitCount = row.getResultByName("hitCount"); | |
256 let lastHit = row.getResultByName("lastHit"); | |
257 let subscriptions = row.getResultByName("subscriptions"); | |
258 | |
259 if (!(filterText in filters)) | |
260 filters[filterText] = Object.create(null); | |
261 | |
262 let filter = filters[filterText]; | |
263 filter.subscriptions = subscriptions; | |
264 filter[matchType] = Object.create(null); | |
265 filter[matchType][host] = {"hits": hitCount, latest: lastHit}; | |
266 } | |
267 resolve({timeSincePush: roundToHours(Date.now() - | |
268 Prefs.sendstats_status.lastPush), "filters": filters}); | |
269 }, | |
270 handleError: (error) => | |
271 { | |
272 Prefs.sendstats_status.lastError = Date.now(); | |
273 this._saveFilterHitPrefs(); | |
274 Cu.reportError("Error loading filter hits:" + error.message); | |
275 }, | |
276 handleCompletion: (reason) => | |
277 { | |
278 if (reason != REASON_FINISHED) | |
279 { | |
280 Prefs.sendstats_status.lastError = Date.now(); | |
281 this._saveFilterHitPrefs(); | |
282 Cu.reportError("Loading of filter hits canceled or aborted: " + | |
283 reason); | |
284 } | |
285 } | |
286 }); | |
141 } | 287 } |
142 else | 288 } |
143 Cu.reportError("Could not send filter hit statistics to Adblock Plus ser ver."); | 289 }, |
144 }, false); | |
145 | |
146 let {addonName, addonVersion, application, applicationVersion, platform, pla tformVersion} = require("info"); | |
147 let data = { | |
148 version: 1, | |
149 timeSincePush: this._lastPush, | |
150 addonName: addonName, | |
151 addonVersion: addonVersion, | |
152 application: application, | |
153 applicationVersion: applicationVersion, | |
154 platform: platform, | |
155 platformVersion: platformVersion, | |
156 filters: this.filters | |
157 }; | |
158 | |
159 this._sending = true; | |
160 request.send(JSON.stringify(data)); | |
161 }, | |
162 | |
163 getStorageFile: function() | |
164 { | |
165 return FileUtils.getFile("ProfD", ["adblockplus.sqlite"]); | |
166 }, | |
167 | |
168 checkCreateTable: function(connection) | |
169 { | |
170 if (!connection.tableExists("filterhits")) | |
171 connection.executeSimpleSQL("CREATE TABLE filterhits (id INTEGER PRIMARY K EY, filters TEXT, date INTEGER)"); | |
172 }, | |
173 | |
174 /** | |
175 * Load Filter hits from database | |
176 */ | |
177 loadFilterHitsFromDatabase: function() | |
178 { | |
179 let storageFile = this.getStorageFile(); | |
180 if (!storageFile) | |
181 return; | |
182 | |
183 let connection = Services.storage.openDatabase(storageFile); | |
184 this.checkCreateTable(connection); | |
185 | |
186 let statement = connection.createStatement("SELECT * FROM filterhits"); | |
187 if (!this._loading) | |
188 { | |
189 this._loading = true; | |
190 statement.executeAsync( | |
191 { | |
192 handleResult: function(results) | |
193 { | |
194 let row = results.getNextRow(); | |
195 if (row) | |
196 { | |
197 let filters = row.getResultByName("filters"); | |
198 let lastDate = row.getResultByName("date"); | |
199 FilterHits.filters = JSON.parse(filters); | |
200 FilterHits._lastPush = lastDate; | |
201 } | |
202 }, | |
203 | |
204 handleError: function(error) | |
205 { | |
206 Cu.reportError(error.message); | |
207 }, | |
208 | |
209 handleCompletion: function(reason) | |
210 { | |
211 if (reason != Ci.mozIStorageStatementCallback.REASON_FINISHED) | |
212 Cu.reportError("Loading of filter hits canceled or aborted."); | |
213 FilterHits._loading = false; | |
214 } | |
215 }); | |
216 } | |
217 | |
218 connection.asyncClose(); | |
219 }, | |
220 | |
221 /** | |
222 * Save Filter hits to database | |
223 */ | |
224 saveFilterHitsToDatabase: function() | |
225 { | |
226 let now = new Date().getTime(); | |
227 if (!this._lastPush) | |
228 this._lastPush = now; | |
229 | |
230 if (!this._sending && now - this._lastPush > this._pushInterval) | |
231 { | |
232 this.sendFilterHitsToServer(); | |
233 return; | |
234 } | |
235 | |
236 let storageFile = this.getStorageFile(); | |
237 if (!storageFile) | |
238 return; | |
239 | |
240 let connection = Services.storage.openDatabase(storageFile); | |
241 this.checkCreateTable(connection); | |
242 | |
243 let statement = connection.createStatement("INSERT OR REPLACE INTO filterhit s (id, filters, date) VALUES(0, :filters, :date)"); | |
244 statement.params.filters = JSON.stringify(this.filters); | |
245 statement.params.date = this._lastPush; | |
246 if (!this._saving) | |
247 { | |
248 this._saving = true; | |
249 statement.executeAsync( | |
250 { | |
251 handleError: function(aError) | |
252 { | |
253 Cu.reportError(aError.message); | |
254 }, | |
255 | |
256 handleCompletion: function(aReason) | |
257 { | |
258 if (aReason != Components.interfaces.mozIStorageStatementCallback.REAS ON_FINISHED) | |
259 Cu.reportError("Writing of filter hits canceled or aborted."); | |
260 FilterHits._saving = false; | |
261 } | |
262 }); | |
263 } | |
264 | |
265 connection.asyncClose(); | |
266 } | |
267 }; | 290 }; |
268 | 291 |
269 FilterNotifier.addListener(function(action) | 292 function roundToHours(timestamp) |
270 { | 293 { |
271 if (action == "load" && Prefs.sendstats) | 294 return Math.round(timestamp / MILLIS_IN_HOUR) * |
272 FilterHits.loadFilterHitsFromDatabase(); | 295 (MILLIS_IN_HOUR / MILLIS_IN_SECOND); |
273 }); | 296 } |
297 | |
298 FilterHits.init(); | |
LEFT | RIGHT |