Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: lib/abp2blocklist.js

Issue 29340694: Issue 3956 - Convert domain whitelisting filters (Closed)
Left Patch Set: Addressed feedback Created May 16, 2016, 4:21 p.m.
Right Patch Set: Fix whitelisting request type logic Created May 17, 2016, 11:22 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « no previous file | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 /* 1 /*
2 * This file is part of Adblock Plus <https://adblockplus.org/>, 2 * This file is part of Adblock Plus <https://adblockplus.org/>,
3 * Copyright (C) 2006-2016 Eyeo GmbH 3 * Copyright (C) 2006-2016 Eyeo GmbH
4 * 4 *
5 * Adblock Plus is free software: you can redistribute it and/or modify 5 * Adblock Plus is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 3 as 6 * it under the terms of the GNU General Public License version 3 as
7 * published by the Free Software Foundation. 7 * published by the Free Software Foundation.
8 * 8 *
9 * Adblock Plus is distributed in the hope that it will be useful, 9 * Adblock Plus is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16 */ 16 */
17 17
18 /** @module abp2blocklist */ 18 /** @module abp2blocklist */
19 19
20 "use strict"; 20 "use strict";
21 21
22 let filterClasses = require("filterClasses"); 22 let filterClasses = require("filterClasses");
23 let tldjs = require("tldjs"); 23 let tldjs = require("tldjs");
24 let punycode = require("punycode"); 24 let punycode = require("punycode");
25 25
26 const selectorLimit = 5000; 26 const selectorLimit = 5000;
27 const typeMap = filterClasses.RegExpFilter.typeMap; 27 const typeMap = filterClasses.RegExpFilter.typeMap;
28 const whitelistableRequestTypes = (typeMap.IMAGE
29 | typeMap.STYLESHEET
30 | typeMap.SCRIPT
31 | typeMap.FONT
32 | typeMap.MEDIA
33 | typeMap.POPUP
34 | typeMap.OBJECT
35 | typeMap.OBJECT_SUBREQUEST
36 | typeMap.XMLHTTPREQUEST
37 | typeMap.PING
38 | typeMap.SUBDOCUMENT
39 | typeMap.OTHER);
28 40
29 function parseDomains(domains, included, excluded) 41 function parseDomains(domains, included, excluded)
30 { 42 {
31 for (let domain in domains) 43 for (let domain in domains)
32 { 44 {
33 if (domain != "") 45 if (domain != "")
34 { 46 {
35 let enabled = domains[domain]; 47 let enabled = domains[domain];
36 domain = punycode.toASCII(domain.toLowerCase()); 48 domain = punycode.toASCII(domain.toLowerCase());
37 49
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after
211 { 223 {
212 result.push(domain); 224 result.push(domain);
213 225
214 if (tldjs.getDomain(domain) == domain) 226 if (tldjs.getDomain(domain) == domain)
215 result.push("www." + domain); 227 result.push("www." + domain);
216 } 228 }
217 229
218 return result; 230 return result;
219 } 231 }
220 232
221 function convertFilter(filter, action, withResourceTypes) 233 function convertFilterAddRules(rules, filter, action, withResourceTypes)
222 { 234 {
223 let rules = [];
224 let parsed = parseFilterRegexpSource(filter.regexpSource); 235 let parsed = parseFilterRegexpSource(filter.regexpSource);
225 236
226 // For the special case of $document whitelisting filters with just a domain 237 // For the special case of $document whitelisting filters with just a domain
227 // we can generate an equivalent blocking rule exception using if-domain. 238 // we can generate an equivalent blocking rule exception using if-domain.
228 if (filter instanceof filterClasses.WhitelistFilter && 239 if (filter instanceof filterClasses.WhitelistFilter &&
229 filter.contentType & typeMap.DOCUMENT && 240 filter.contentType & typeMap.DOCUMENT &&
230 parsed.justHostname) 241 parsed.justHostname)
231 { 242 {
232 rules.push({ 243 rules.push({
233 trigger: { 244 trigger: {
234 "url-filter": ".*", 245 "url-filter": ".*",
235 "if-domain": addDomainPrefix([parsed.hostname]) 246 "if-domain": addDomainPrefix([parsed.hostname])
236 }, 247 },
237 action: {type: "ignore-previous-rules"} 248 action: {type: "ignore-previous-rules"}
238 }); 249 });
239 // If the filter contains multiple options we'll need to generate further 250 // If the filter contains other supported options we'll need to generate
240 // rules for it, but if not we can simply return now. 251 // further rules for it, but if not we can simply return now.
241 if (filter.contentType == typeMap.DOCUMENT) 252 if (!(filter.contentType | whitelistableRequestTypes))
242 return rules; 253 return;
243 } 254 }
244 255
245 let trigger = {"url-filter": parsed.regexp}; 256 let trigger = {"url-filter": parsed.regexp};
246 257
247 // Limit rules to HTTP(S) URLs 258 // Limit rules to HTTP(S) URLs
248 if (!/^(\^|http)/i.test(trigger["url-filter"])) 259 if (!/^(\^|http)/i.test(trigger["url-filter"]))
249 trigger["url-filter"] = "^https?://.*" + trigger["url-filter"]; 260 trigger["url-filter"] = "^https?://.*" + trigger["url-filter"];
250 261
251 // For rules containing only a hostname we know that we're matching against 262 // For rules containing only a hostname we know that we're matching against
252 // a lowercase string unless the matchCase option was passed. 263 // a lowercase string unless the matchCase option was passed.
(...skipping 12 matching lines...) Expand all
265 trigger["resource-type"] = getResourceTypes(filter); 276 trigger["resource-type"] = getResourceTypes(filter);
266 if (filter.thirdParty != null) 277 if (filter.thirdParty != null)
267 trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"]; 278 trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"];
268 279
269 if (included.length > 0) 280 if (included.length > 0)
270 trigger["if-domain"] = addDomainPrefix(included); 281 trigger["if-domain"] = addDomainPrefix(included);
271 else if (excluded.length > 0) 282 else if (excluded.length > 0)
272 trigger["unless-domain"] = addDomainPrefix(excluded); 283 trigger["unless-domain"] = addDomainPrefix(excluded);
273 284
274 rules.push({trigger: trigger, action: {type: action}}); 285 rules.push({trigger: trigger, action: {type: action}});
275
276 return rules;
277 } 286 }
278 287
279 function hasNonASCI(obj) 288 function hasNonASCI(obj)
280 { 289 {
281 if (typeof obj == "string") 290 if (typeof obj == "string")
282 { 291 {
283 if (/[^\x00-\x7F]/.test(obj)) 292 if (/[^\x00-\x7F]/.test(obj))
284 return true; 293 return true;
285 } 294 }
286 295
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after
377 return; 386 return;
378 if (filter instanceof filterClasses.RegExpFilter && 387 if (filter instanceof filterClasses.RegExpFilter &&
379 filter.regexpSource == null) 388 filter.regexpSource == null)
380 return; 389 return;
381 390
382 if (filter instanceof filterClasses.BlockingFilter) 391 if (filter instanceof filterClasses.BlockingFilter)
383 this.requestFilters.push(filter); 392 this.requestFilters.push(filter);
384 393
385 if (filter instanceof filterClasses.WhitelistFilter) 394 if (filter instanceof filterClasses.WhitelistFilter)
386 { 395 {
387 if (filter.contentType & (typeMap.DOCUMENT 396 if (filter.contentType & (typeMap.DOCUMENT | whitelistableRequestTypes))
388 | typeMap.IMAGE
389 | typeMap.STYLESHEET
390 | typeMap.SCRIPT
391 | typeMap.FONT
392 | typeMap.MEDIA
393 | typeMap.POPUP
394 | typeMap.OBJECT
395 | typeMap.OBJECT_SUBREQUEST
396 | typeMap.XMLHTTPREQUEST
397 | typeMap.PING
398 | typeMap.SUBDOCUMENT
399 | typeMap.OTHER))
400 this.requestExceptions.push(filter); 397 this.requestExceptions.push(filter);
401 398
402 if (filter.contentType & typeMap.ELEMHIDE) 399 if (filter.contentType & typeMap.ELEMHIDE)
403 this.elemhideExceptions.push(filter); 400 this.elemhideExceptions.push(filter);
404 } 401 }
405 402
406 if (filter instanceof filterClasses.ElemHideFilter) 403 if (filter instanceof filterClasses.ElemHideFilter)
407 this.elemhideFilters.push(filter); 404 this.elemhideFilters.push(filter);
408 405
409 if (filter instanceof filterClasses.ElemHideException) 406 if (filter instanceof filterClasses.ElemHideException)
410 { 407 {
411 let domains = this.elemhideSelectorExceptions[filter.selector]; 408 let domains = this.elemhideSelectorExceptions[filter.selector];
412 if (!domains) 409 if (!domains)
413 domains = this.elemhideSelectorExceptions[filter.selector] = []; 410 domains = this.elemhideSelectorExceptions[filter.selector] = [];
414 411
415 parseDomains(filter.domains, domains, []); 412 parseDomains(filter.domains, domains, []);
416 } 413 }
417 }; 414 };
418 415
419 /** 416 /**
420 * Generate content blocker list for all filters that were added 417 * Generate content blocker list for all filters that were added
421 * 418 *
422 * @returns {Filter} filter Filter to convert 419 * @returns {Filter} filter Filter to convert
423 */ 420 */
424 ContentBlockerList.prototype.generateRules = function(filter) 421 ContentBlockerList.prototype.generateRules = function(filter)
425 { 422 {
426 let rules = []; 423 let rules = [];
427 424
428 function addRules(newRules)
429 {
430 for (let rule of newRules)
431 if (!hasNonASCI(rule))
432 rules.push(rule);
433 }
434
435 let groupedElemhideFilters = new Map(); 425 let groupedElemhideFilters = new Map();
436 for (let filter of this.elemhideFilters) 426 for (let filter of this.elemhideFilters)
437 { 427 {
438 let result = convertElemHideFilter(filter, this.elemhideSelectorExceptions); 428 let result = convertElemHideFilter(filter, this.elemhideSelectorExceptions);
439 if (!result) 429 if (!result)
440 continue; 430 continue;
441 431
442 if (result.matchDomains.length == 0) 432 if (result.matchDomains.length == 0)
443 result.matchDomains = ["^https?://"]; 433 result.matchDomains = ["^https?://"];
444 434
445 for (let matchDomain of result.matchDomains) 435 for (let matchDomain of result.matchDomains)
446 { 436 {
447 let group = groupedElemhideFilters.get(matchDomain) || []; 437 let group = groupedElemhideFilters.get(matchDomain) || [];
448 group.push(result.selector); 438 group.push(result.selector);
449 groupedElemhideFilters.set(matchDomain, group); 439 groupedElemhideFilters.set(matchDomain, group);
450 } 440 }
451 } 441 }
452 442
453 groupedElemhideFilters.forEach((selectors, matchDomain) => 443 groupedElemhideFilters.forEach((selectors, matchDomain) =>
454 { 444 {
455 while (selectors.length) 445 while (selectors.length)
456 { 446 {
457 let selector = selectors.splice(0, selectorLimit).join(", "); 447 let selector = selectors.splice(0, selectorLimit).join(", ");
458 448
459 // As of Safari 9.0 element IDs are matched as lowercase. We work around 449 // As of Safari 9.0 element IDs are matched as lowercase. We work around
460 // this by converting to the attribute format [id="elementID"] 450 // this by converting to the attribute format [id="elementID"]
461 selector = convertIDSelectorsToAttributeSelectors(selector); 451 selector = convertIDSelectorsToAttributeSelectors(selector);
462 452
463 addRules([{ 453 rules.push({
Sebastian Noack 2016/05/17 10:17:54 We are creating quite a few temporary arrays now.
kzar 2016/05/17 10:38:02 Done.
464 trigger: {"url-filter": matchDomain, 454 trigger: {"url-filter": matchDomain,
465 "url-filter-is-case-sensitive": true}, 455 "url-filter-is-case-sensitive": true},
466 action: {type: "css-display-none", 456 action: {type: "css-display-none",
467 selector: selector} 457 selector: selector}
468 }]); 458 });
469 } 459 }
470 }); 460 });
471 461
472 for (let filter of this.elemhideExceptions) 462 for (let filter of this.elemhideExceptions)
473 addRules(convertFilter(filter, "ignore-previous-rules", false)); 463 convertFilterAddRules(rules, filter, "ignore-previous-rules", false);
474 for (let filter of this.requestFilters) 464 for (let filter of this.requestFilters)
475 addRules(convertFilter(filter, "block", true)); 465 convertFilterAddRules(rules, filter, "block", true);
476 for (let filter of this.requestExceptions) 466 for (let filter of this.requestExceptions)
477 addRules(convertFilter(filter, "ignore-previous-rules", true)); 467 convertFilterAddRules(rules, filter, "ignore-previous-rules", true);
478 468
479 return rules; 469 return rules.filter(rule => !hasNonASCI(rule));
480 }; 470 };
LEFTRIGHT
« no previous file | no next file » | Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Toggle Comments ('s')

Powered by Google App Engine
This is Rietveld