Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: lib/synchronizer.js

Issue 29606600: Issue 5146 - Implement DownloadableSubscription parsing in C++ (Closed) Base URL: https://hg.adblockplus.org/adblockpluscore/
Patch Set: Improved logic for Process() Created Dec. 5, 2017, 7:16 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « compiled/subscription/Subscription.h ('k') | meson.build » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* 1 /*
2 * This file is part of Adblock Plus <https://adblockplus.org/>, 2 * This file is part of Adblock Plus <https://adblockplus.org/>,
3 * Copyright (C) 2006-present eyeo GmbH 3 * Copyright (C) 2006-present eyeo GmbH
4 * 4 *
5 * Adblock Plus is free software: you can redistribute it and/or modify 5 * Adblock Plus is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 3 as 6 * it under the terms of the GNU General Public License version 3 as
7 * published by the Free Software Foundation. 7 * published by the Free Software Foundation.
8 * 8 *
9 * Adblock Plus is distributed in the hope that it will be useful, 9 * Adblock Plus is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16 */ 16 */
17 17
18 "use strict"; 18 "use strict";
19 19
20 /** 20 /**
21 * @fileOverview Manages synchronization of filter subscriptions. 21 * @fileOverview Manages synchronization of filter subscriptions.
22 */ 22 */
23 23
24 const {Downloader, Downloadable, 24 const {Downloader, Downloadable,
25 MILLIS_IN_SECOND, MILLIS_IN_MINUTE, 25 MILLIS_IN_SECOND, MILLIS_IN_MINUTE,
26 MILLIS_IN_HOUR, MILLIS_IN_DAY} = require("downloader"); 26 MILLIS_IN_HOUR, MILLIS_IN_DAY} = require("downloader");
27 const {Filter} = require("filterClasses");
28 const {FilterStorage} = require("filterStorage"); 27 const {FilterStorage} = require("filterStorage");
29 const {FilterNotifier} = require("filterNotifier"); 28 const {FilterNotifier} = require("filterNotifier");
30 const {Prefs} = require("prefs"); 29 const {Prefs} = require("prefs");
31 const {Subscription, DownloadableSubscription} = require("subscriptionClasses"); 30 const {Subscription, DownloadableSubscription} = require("subscriptionClasses");
32 const {Utils} = require("utils");
33 31
34 const INITIAL_DELAY = 1 * MILLIS_IN_MINUTE; 32 const INITIAL_DELAY = 1 * MILLIS_IN_MINUTE;
35 const CHECK_INTERVAL = 1 * MILLIS_IN_HOUR; 33 const CHECK_INTERVAL = 1 * MILLIS_IN_HOUR;
36 const DEFAULT_EXPIRATION_INTERVAL = 5 * MILLIS_IN_DAY; 34 const DEFAULT_EXPIRATION_INTERVAL = 5 * MILLIS_IN_DAY;
37 35
38 /** 36 /**
39 * The object providing actual downloading functionality. 37 * The object providing actual downloading functionality.
40 * @type {Downloader} 38 * @type {Downloader}
41 */ 39 */
42 let downloader = null; 40 let downloader = null;
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after
146 144
147 _onDownloadSuccess(downloadable, responseText, errorCallback, 145 _onDownloadSuccess(downloadable, responseText, errorCallback,
148 redirectCallback) 146 redirectCallback)
149 { 147 {
150 let lines = responseText.split(/[\r\n]+/); 148 let lines = responseText.split(/[\r\n]+/);
151 let headerMatch = /\[Adblock(?:\s*Plus\s*([\d.]+)?)?\]/i.exec(lines[0]); 149 let headerMatch = /\[Adblock(?:\s*Plus\s*([\d.]+)?)?\]/i.exec(lines[0]);
152 if (!headerMatch) 150 if (!headerMatch)
153 return errorCallback("synchronize_invalid_data"); 151 return errorCallback("synchronize_invalid_data");
154 let minVersion = headerMatch[1]; 152 let minVersion = headerMatch[1];
155 153
156 // Don't remove parameter comments immediately but add them to a list first, 154 let parser = DownloadableSubscription.parseDownload();
157 // they need to be considered in the checksum calculation. 155
158 let remove = []; 156 // Process filters
159 let params = { 157 for (let line of lines)
160 redirect: null, 158 parser.process(line);
161 homepage: null, 159
162 title: null, 160 if (!parser.verifyChecksum())
163 version: null, 161 return errorCallback("synchronize_checksum_mismatch");
164 expires: null 162
165 }; 163 if (parser.redirect)
166 for (let i = 0; i < lines.length; i++)
167 { 164 {
168 let match = /^\s*!\s*(\w+)\s*:\s*(.*)/.exec(lines[i]); 165 let {redirect} = parser;
169 if (match) 166 parser.delete();
170 { 167 return redirectCallback(redirect);
171 let keyword = match[1].toLowerCase();
172 let value = match[2];
173 if (keyword in params)
174 {
175 params[keyword] = value;
176 remove.push(i);
177 }
178 else if (keyword == "checksum")
179 {
180 lines.splice(i--, 1);
181 let checksum = Utils.generateChecksum(lines);
182 if (checksum && checksum != value.replace(/=+$/, ""))
183 return errorCallback("synchronize_checksum_mismatch");
184 }
185 }
186 } 168 }
187 169
188 if (params.redirect)
189 return redirectCallback(params.redirect);
190
191 // Handle redirects 170 // Handle redirects
192 let subscription = Subscription.fromURL(downloadable.redirectURL || 171 let subscription = Subscription.fromURL(downloadable.redirectURL ||
193 downloadable.url); 172 downloadable.url);
194 if (downloadable.redirectURL && 173 if (downloadable.redirectURL &&
195 downloadable.redirectURL != downloadable.url) 174 downloadable.redirectURL != downloadable.url)
196 { 175 {
197 let oldSubscription = Subscription.fromURL(downloadable.url); 176 let oldSubscription = Subscription.fromURL(downloadable.url);
198 subscription.title = oldSubscription.title; 177 subscription.title = oldSubscription.title;
199 subscription.disabled = oldSubscription.disabled; 178 subscription.disabled = oldSubscription.disabled;
200 subscription.lastCheck = oldSubscription.lastCheck; 179 subscription.lastCheck = oldSubscription.lastCheck;
201 180
202 let listed = (oldSubscription.url in FilterStorage.knownSubscriptions); 181 let {listed} = oldSubscription;
203 if (listed) 182 if (listed)
204 FilterStorage.removeSubscription(oldSubscription); 183 FilterStorage.removeSubscription(oldSubscription);
205 184
206 delete Subscription.knownSubscriptions[oldSubscription.url];
207
208 if (listed) 185 if (listed)
209 FilterStorage.addSubscription(subscription); 186 FilterStorage.addSubscription(subscription);
210 } 187 }
211 188
212 // The download actually succeeded 189 // The download actually succeeded
213 subscription.lastSuccess = subscription.lastDownload = Math.round( 190 subscription.lastSuccess = subscription.lastDownload = Math.round(
214 Date.now() / MILLIS_IN_SECOND 191 Date.now() / MILLIS_IN_SECOND
215 ); 192 );
216 subscription.downloadStatus = "synchronize_ok"; 193 subscription.downloadStatus = "synchronize_ok";
217 subscription.downloadCount = downloadable.downloadCount; 194 subscription.downloadCount = downloadable.downloadCount;
218 subscription.errors = 0; 195 subscription.errors = 0;
219 196
220 // Remove lines containing parameters
221 for (let i = remove.length - 1; i >= 0; i--)
222 lines.splice(remove[i], 1);
223
224 // Process parameters 197 // Process parameters
225 if (params.homepage) 198 if (parser.homepage)
226 { 199 {
227 let url; 200 let url;
228 try 201 try
229 { 202 {
230 url = new URL(params.homepage); 203 url = new URL(parser.homepage);
231 } 204 }
232 catch (e) 205 catch (e)
233 { 206 {
234 url = null; 207 url = null;
235 } 208 }
236 209
237 if (url && (url.protocol == "http:" || url.protocol == "https:")) 210 if (url && (url.protocol == "http:" || url.protocol == "https:"))
238 subscription.homepage = url.href; 211 subscription.homepage = url.href;
239 } 212 }
240 213
241 if (params.title) 214 if (minVersion)
242 { 215 subscription.requiredVersion = minVersion;
243 subscription.title = params.title;
244 subscription.fixedTitle = true;
245 }
246 else 216 else
247 subscription.fixedTitle = false; 217 delete subscription.requiredVersion;
248
249 subscription.version = (params.version ? parseInt(params.version, 10) : 0);
250 218
251 let expirationInterval = DEFAULT_EXPIRATION_INTERVAL; 219 let expirationInterval = DEFAULT_EXPIRATION_INTERVAL;
252 if (params.expires) 220 let expiration = parser.finalize(subscription);
253 { 221 if (expiration != 0)
254 let match = /^(\d+)\s*(h)?/.exec(params.expires); 222 expirationInterval = expiration;
255 if (match)
256 {
257 let interval = parseInt(match[1], 10);
258 if (match[2])
259 expirationInterval = interval * MILLIS_IN_HOUR;
260 else
261 expirationInterval = interval * MILLIS_IN_DAY;
262 }
263 }
264 223
265 let [ 224 let [
266 softExpiration, 225 softExpiration,
267 hardExpiration 226 hardExpiration
268 ] = downloader.processExpirationInterval(expirationInterval); 227 ] = downloader.processExpirationInterval(expirationInterval);
269 subscription.softExpiration = Math.round(softExpiration / MILLIS_IN_SECOND); 228 subscription.softExpiration = Math.round(softExpiration / MILLIS_IN_SECOND);
270 subscription.expires = Math.round(hardExpiration / MILLIS_IN_SECOND); 229 subscription.expires = Math.round(hardExpiration / MILLIS_IN_SECOND);
271 230
272 if (minVersion) 231 parser.delete();
273 subscription.requiredVersion = minVersion;
274 else
275 delete subscription.requiredVersion;
276
277 // Process filters
278 lines.shift();
279 let filters = [];
280 for (let line of lines)
281 {
282 line = Filter.normalize(line);
283 if (line)
284 filters.push(Filter.fromText(line));
285 }
286
287 FilterStorage.updateSubscriptionFilters(subscription, filters);
288 232
289 return undefined; 233 return undefined;
290 }, 234 },
291 235
292 _onDownloadError(downloadable, downloadURL, error, channelStatus, 236 _onDownloadError(downloadable, downloadURL, error, channelStatus,
293 responseStatus, redirectCallback) 237 responseStatus, redirectCallback)
294 { 238 {
295 let subscription = Subscription.fromURL(downloadable.url); 239 let subscription = Subscription.fromURL(downloadable.url);
296 subscription.lastDownload = Math.round(Date.now() / MILLIS_IN_SECOND); 240 subscription.lastDownload = Math.round(Date.now() / MILLIS_IN_SECOND);
297 subscription.downloadStatus = error; 241 subscription.downloadStatus = error;
(...skipping 29 matching lines...) Expand all
327 request.open("GET", fallbackURL); 271 request.open("GET", fallbackURL);
328 request.overrideMimeType("text/plain"); 272 request.overrideMimeType("text/plain");
329 request.channel.loadFlags = request.channel.loadFlags | 273 request.channel.loadFlags = request.channel.loadFlags |
330 request.channel.INHIBIT_CACHING | 274 request.channel.INHIBIT_CACHING |
331 request.channel.VALIDATE_ALWAYS; 275 request.channel.VALIDATE_ALWAYS;
332 request.addEventListener("load", ev => 276 request.addEventListener("load", ev =>
333 { 277 {
334 if (onShutdown.done) 278 if (onShutdown.done)
335 return; 279 return;
336 280
337 if (!(subscription.url in FilterStorage.knownSubscriptions)) 281 if (!subscription.listed)
338 return; 282 return;
339 283
340 let match = /^(\d+)(?:\s+(\S+))?$/.exec(request.responseText); 284 let match = /^(\d+)(?:\s+(\S+))?$/.exec(request.responseText);
341 if (match && match[1] == "301" && // Moved permanently 285 if (match && match[1] == "301" && // Moved permanently
342 match[2] && /^https?:\/\//i.test(match[2])) 286 match[2] && /^https?:\/\//i.test(match[2]))
343 { 287 {
344 redirectCallback(match[2]); 288 redirectCallback(match[2]);
345 } 289 }
346 else if (match && match[1] == "410") // Gone 290 else if (match && match[1] == "410") // Gone
347 { 291 {
348 let data = "[Adblock]\n" + 292 let data = "[Adblock]\n" +
349 subscription.filters.map(f => f.text).join("\n"); 293 Array.from(subscription.filters, f => f.text).join("\n");
350 redirectCallback("data:text/plain," + encodeURIComponent(data)); 294 redirectCallback("data:text/plain," + encodeURIComponent(data));
351 } 295 }
352 }, false); 296 }, false);
353 request.send(null); 297 request.send(null);
354 } 298 }
355 } 299 }
356 } 300 }
357 }; 301 };
358 Synchronizer.init(); 302 Synchronizer.init();
OLDNEW
« no previous file with comments | « compiled/subscription/Subscription.h ('k') | meson.build » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld