| OLD | NEW |
| 1 var readline = require("readline"); | 1 "use strict"; |
| 2 var punycode = require("punycode"); | |
| 3 var tldjs = require("tldjs"); | |
| 4 var filterClasses = require("./adblockplus.js"); | |
| 5 | 2 |
| 6 var typeMap = filterClasses.RegExpFilter.typeMap; | 3 let readline = require("readline"); |
| 4 let punycode = require("punycode"); |
| 5 let tldjs = require("tldjs"); |
| 6 let filterClasses = require("./adblockplus.js"); |
| 7 | 7 |
| 8 var requestFilters = []; | 8 let typeMap = filterClasses.RegExpFilter.typeMap; |
| 9 var requestExceptions = []; | 9 |
| 10 var elemhideFilters = []; | 10 let requestFilters = []; |
| 11 var elemhideExceptions = []; | 11 let requestExceptions = []; |
| 12 var elemhideSelectorExceptions = Object.create(null); | 12 let elemhideFilters = []; |
| 13 let elemhideExceptions = []; |
| 14 let elemhideSelectorExceptions = new Map(); |
| 13 | 15 |
| 14 function recordException(filter) | 16 function recordException(filter) |
| 15 { | 17 { |
| 16 if (filter.contentType & (typeMap.IMAGE | 18 if (filter.contentType & (typeMap.IMAGE |
| 17 | typeMap.STYLESHEET | 19 | typeMap.STYLESHEET |
| 18 | typeMap.SCRIPT | 20 | typeMap.SCRIPT |
| 19 | typeMap.FONT | 21 | typeMap.FONT |
| 20 | typeMap.MEDIA | 22 | typeMap.MEDIA |
| 21 | typeMap.POPUP | 23 | typeMap.POPUP |
| 22 | typeMap.OBJECT | 24 | typeMap.OBJECT |
| 23 | typeMap.OBJECT_SUBREQUEST | 25 | typeMap.OBJECT_SUBREQUEST |
| 24 | typeMap.XMLHTTPREQUEST | 26 | typeMap.XMLHTTPREQUEST |
| 25 | typeMap.PING | 27 | typeMap.PING |
| 26 | typeMap.SUBDOCUMENT | 28 | typeMap.SUBDOCUMENT |
| 27 | typeMap.OTHER)) | 29 | typeMap.OTHER)) |
| 28 requestExceptions.push(filter); | 30 requestExceptions.push(filter); |
| 29 | 31 |
| 30 if (filter.contentType & typeMap.ELEMHIDE) | 32 if (filter.contentType & typeMap.ELEMHIDE) |
| 31 elemhideExceptions.push(filter); | 33 elemhideExceptions.push(filter); |
| 32 } | 34 } |
| 33 | 35 |
| 34 function parseDomains(domains, included, excluded) | 36 function parseDomains(domains, included, excluded) |
| 35 { | 37 { |
| 36 for (var domain in domains) | 38 for (let domain in domains) |
| 37 { | 39 { |
| 38 if (domain != "") | 40 if (domain != "") |
| 39 { | 41 { |
| 40 var enabled = domains[domain]; | 42 let enabled = domains[domain]; |
| 41 domain = punycode.toASCII(domain.toLowerCase()); | 43 domain = punycode.toASCII(domain.toLowerCase()); |
| 42 | 44 |
| 43 if (!enabled) | 45 if (!enabled) |
| 44 excluded.push(domain); | 46 excluded.push(domain); |
| 45 else if (!domains[""]) | 47 else if (!domains[""]) |
| 46 included.push(domain); | 48 included.push(domain); |
| 47 } | 49 } |
| 48 } | 50 } |
| 49 } | 51 } |
| 50 | 52 |
| 51 function recordSelectorException(filter) | 53 function recordSelectorException(filter) |
| 52 { | 54 { |
| 53 var domains = elemhideSelectorExceptions[filter.selector]; | 55 let domains = elemhideSelectorExceptions[filter.selector]; |
| 54 if (!domains) | 56 if (!domains) |
| 55 domains = elemhideSelectorExceptions[filter.selector] = []; | 57 domains = elemhideSelectorExceptions[filter.selector] = []; |
| 56 | 58 |
| 57 parseDomains(filter.domains, domains, []); | 59 parseDomains(filter.domains, domains, []); |
| 58 } | 60 } |
| 59 | 61 |
| 60 function parseFilter(line) | 62 function parseFilter(line) |
| 61 { | 63 { |
| 62 if (line.charAt(0) == "[") | 64 if (line.charAt(0) == "[") |
| 63 return; | 65 return; |
| 64 | 66 |
| 65 var filter = filterClasses.Filter.fromText(line); | 67 let filter = filterClasses.Filter.fromText(line); |
| 66 | 68 |
| 67 if (filter.sitekeys) | 69 if (filter.sitekeys) |
| 68 return; | 70 return; |
| 69 if (filter instanceof filterClasses.RegExpFilter && !filter.regexpSource) | 71 if (filter instanceof filterClasses.RegExpFilter && !filter.regexpSource) |
| 70 return; | 72 return; |
| 71 | 73 |
| 72 if (filter instanceof filterClasses.BlockingFilter) | 74 if (filter instanceof filterClasses.BlockingFilter) |
| 73 requestFilters.push(filter); | 75 requestFilters.push(filter); |
| 74 if (filter instanceof filterClasses.WhitelistFilter) | 76 if (filter instanceof filterClasses.WhitelistFilter) |
| 75 recordException(filter); | 77 recordException(filter); |
| 76 if (filter instanceof filterClasses.ElemHideFilter) | 78 if (filter instanceof filterClasses.ElemHideFilter) |
| 77 elemhideFilters.push(filter); | 79 elemhideFilters.push(filter); |
| 78 if (filter instanceof filterClasses.ElemHideException) | 80 if (filter instanceof filterClasses.ElemHideException) |
| 79 recordSelectorException(filter); | 81 recordSelectorException(filter); |
| 80 } | 82 } |
| 81 | 83 |
| 82 function escapeRegExp(s) | 84 function escapeRegExp(s) |
| 83 { | 85 { |
| 84 return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); | 86 return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); |
| 85 } | 87 } |
| 86 | 88 |
| 87 function matchDomain(domain) | 89 function matchDomain(domain) |
| 88 { | 90 { |
| 89 return "^https?://([^/:]*\\.)?" + escapeRegExp(domain) + "[/:]"; | 91 return "^https?://([^/:]*\\.)?" + escapeRegExp(domain) + "[/:]"; |
| 90 } | 92 } |
| 91 | 93 |
| 92 function convertElemHideFilter(filter) | 94 function convertElemHideFilter(filter) |
| 93 { | 95 { |
| 94 var included = []; | 96 let included = []; |
| 95 var excluded = []; | 97 let excluded = []; |
| 96 var rules = []; | 98 let rules = []; |
| 97 | 99 |
| 98 parseDomains(filter.domains, included, excluded); | 100 parseDomains(filter.domains, included, excluded); |
| 99 | 101 |
| 100 if (excluded.length == 0 && !(filter.selector in elemhideSelectorExceptions)) | 102 if (excluded.length == 0 && !(filter.selector in elemhideSelectorExceptions)) |
| 101 { | 103 { |
| 102 var action = { | 104 let action = { |
| 103 type: "css-display-none", | 105 type: "css-display-none", |
| 104 selector: filter.selector | 106 selector: filter.selector |
| 105 }; | 107 }; |
| 106 | 108 |
| 107 for (var i = 0; i < included.length; i++) | 109 for (let domain of included) |
| 108 rules.push({ | 110 rules.push({ |
| 109 trigger: {"url-filter": matchDomain(included[i])}, | 111 trigger: {"url-filter": matchDomain(domain)}, |
| 110 action: action | 112 action: action |
| 111 }); | 113 }); |
| 112 | 114 |
| 113 if (included.length == 0) | 115 if (included.length == 0) |
| 114 rules.push({ | 116 rules.push({ |
| 115 trigger: {"url-filter": "^https?://"}, | 117 trigger: {"url-filter": "^https?://"}, |
| 116 action: action | 118 action: action |
| 117 }); | 119 }); |
| 118 } | 120 } |
| 119 | 121 |
| 120 return rules; | 122 return rules; |
| 121 } | 123 } |
| 122 | 124 |
| 123 function toRegExp(text) | 125 function toRegExp(text) |
| 124 { | 126 { |
| 125 var result = ""; | 127 let result = ""; |
| 126 var lastIndex = text.length - 1; | 128 let lastIndex = text.length - 1; |
| 127 | 129 |
| 128 for (var i = 0; i < text.length; i++) | 130 for (let i = 0; i < text.length; i++) |
| 129 { | 131 { |
| 130 var c = text[i]; | 132 let c = text[i]; |
| 131 | 133 |
| 132 switch (c) | 134 switch (c) |
| 133 { | 135 { |
| 134 case "*": | 136 case "*": |
| 135 if (result.length > 0 && i < lastIndex && text[i + 1] != "*") | 137 if (result.length > 0 && i < lastIndex && text[i + 1] != "*") |
| 136 result += ".*"; | 138 result += ".*"; |
| 137 break; | 139 break; |
| 138 case "^": | 140 case "^": |
| 139 if (i < lastIndex) | 141 if (i < lastIndex) |
| 140 result += "."; | 142 result += "."; |
| (...skipping 21 matching lines...) Expand all Loading... |
| 162 default: | 164 default: |
| 163 result += c; | 165 result += c; |
| 164 } | 166 } |
| 165 } | 167 } |
| 166 | 168 |
| 167 return result; | 169 return result; |
| 168 } | 170 } |
| 169 | 171 |
| 170 function getRegExpSource(filter) | 172 function getRegExpSource(filter) |
| 171 { | 173 { |
| 172 var source = toRegExp(filter.regexpSource.replace( | 174 let source = toRegExp(filter.regexpSource.replace( |
| 173 // Safari expects punycode, filter lists use unicode | 175 // Safari expects punycode, filter lists use unicode |
| 174 /^(\|\||\|?https?:\/\/)([\w\-.*\u0080-\uFFFF]+)/i, | 176 /^(\|\||\|?https?:\/\/)([\w\-.*\u0080-\uFFFF]+)/i, |
| 175 function (match, prefix, domain) | 177 function (match, prefix, domain) |
| 176 { | 178 { |
| 177 return prefix + punycode.toASCII(domain); | 179 return prefix + punycode.toASCII(domain); |
| 178 } | 180 } |
| 179 )); | 181 )); |
| 180 | 182 |
| 181 // Limit rules to to HTTP(S) URLs | 183 // Limit rules to to HTTP(S) URLs |
| 182 if (!/^(\^|http)/i.test(source)) | 184 if (!/^(\^|http)/i.test(source)) |
| 183 source = "^https?://.*" + source; | 185 source = "^https?://.*" + source; |
| 184 | 186 |
| 185 return source; | 187 return source; |
| 186 } | 188 } |
| 187 | 189 |
| 188 function getResourceTypes(filter) | 190 function getResourceTypes(filter) |
| 189 { | 191 { |
| 190 var types = []; | 192 let types = []; |
| 191 | 193 |
| 192 if (filter.contentType & typeMap.IMAGE) | 194 if (filter.contentType & typeMap.IMAGE) |
| 193 types.push("image"); | 195 types.push("image"); |
| 194 if (filter.contentType & typeMap.STYLESHEET) | 196 if (filter.contentType & typeMap.STYLESHEET) |
| 195 types.push("style-sheet"); | 197 types.push("style-sheet"); |
| 196 if (filter.contentType & typeMap.SCRIPT) | 198 if (filter.contentType & typeMap.SCRIPT) |
| 197 types.push("script"); | 199 types.push("script"); |
| 198 if (filter.contentType & typeMap.FONT) | 200 if (filter.contentType & typeMap.FONT) |
| 199 types.push("font"); | 201 types.push("font"); |
| 200 if (filter.contentType & (typeMap.MEDIA | typeMap.OBJECT)) | 202 if (filter.contentType & (typeMap.MEDIA | typeMap.OBJECT)) |
| 201 types.push("media"); | 203 types.push("media"); |
| 202 if (filter.contentType & typeMap.POPUP) | 204 if (filter.contentType & typeMap.POPUP) |
| 203 types.push("popup"); | 205 types.push("popup"); |
| 204 if (filter.contentType & (typeMap.XMLHTTPREQUEST | typeMap.OBJECT_SUBREQUEST | 206 if (filter.contentType & (typeMap.XMLHTTPREQUEST | typeMap.OBJECT_SUBREQUEST |
| 205 | typeMap.PING | typeMap.OTHER)) | 207 | typeMap.PING | typeMap.OTHER)) |
| 206 types.push("raw"); | 208 types.push("raw"); |
| 207 if (filter.contentType & typeMap.SUBDOCUMENT) | 209 if (filter.contentType & typeMap.SUBDOCUMENT) |
| 208 types.push("document"); | 210 types.push("document"); |
| 209 | 211 |
| 210 return types; | 212 return types; |
| 211 } | 213 } |
| 212 | 214 |
| 213 function addDomainPrefix(domains) | 215 function addDomainPrefix(domains) |
| 214 { | 216 { |
| 215 var result = []; | 217 let result = []; |
| 216 | 218 |
| 217 for (var i = 0; i < domains.length; i++) | 219 for (let domain of domains) |
| 218 { | 220 { |
| 219 var domain = domains[i]; | |
| 220 result.push(domain); | 221 result.push(domain); |
| 221 | 222 |
| 222 if (tldjs.getSubdomain(domain) == "") | 223 if (tldjs.getSubdomain(domain) == "") |
| 223 result.push("www." + domain); | 224 result.push("www." + domain); |
| 224 } | 225 } |
| 225 | 226 |
| 226 return result; | 227 return result; |
| 227 } | 228 } |
| 228 | 229 |
| 229 function convertFilter(filter, action, withResourceTypes) | 230 function convertFilter(filter, action, withResourceTypes) |
| 230 { | 231 { |
| 231 var trigger = {"url-filter": getRegExpSource(filter)}; | 232 let trigger = {"url-filter": getRegExpSource(filter)}; |
| 232 var included = []; | 233 let included = []; |
| 233 var excluded = []; | 234 let excluded = []; |
| 234 | 235 |
| 235 parseDomains(filter.domains, included, excluded); | 236 parseDomains(filter.domains, included, excluded); |
| 236 | 237 |
| 237 if (withResourceTypes) | 238 if (withResourceTypes) |
| 238 trigger["resource-type"] = getResourceTypes(filter); | 239 trigger["resource-type"] = getResourceTypes(filter); |
| 239 if (filter.thirdParty != null) | 240 if (filter.thirdParty != null) |
| 240 trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"]; | 241 trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"]; |
| 241 | 242 |
| 242 if (included.length > 0) | 243 if (included.length > 0) |
| 243 trigger["if-domain"] = addDomainPrefix(included); | 244 trigger["if-domain"] = addDomainPrefix(included); |
| 244 else if (excluded.length > 0) | 245 else if (excluded.length > 0) |
| 245 trigger["unless-domain"] = addDomainPrefix(excluded); | 246 trigger["unless-domain"] = addDomainPrefix(excluded); |
| 246 | 247 |
| 247 return {trigger: trigger, action: {type: action}}; | 248 return {trigger: trigger, action: {type: action}}; |
| 248 } | 249 } |
| 249 | 250 |
| 250 function hasNonASCI(obj) | 251 function hasNonASCI(obj) |
| 251 { | 252 { |
| 252 if (typeof obj == "string") | 253 if (typeof obj == "string") |
| 253 { | 254 { |
| 254 if (/[^\x00-\x7F]/.test(obj)) | 255 if (/[^\x00-\x7F]/.test(obj)) |
| 255 return true; | 256 return true; |
| 256 } | 257 } |
| 257 | 258 |
| 258 if (typeof obj == "object") | 259 if (typeof obj == "object") |
| 259 { | 260 { |
| 260 var i; | |
| 261 if (obj instanceof Array) | 261 if (obj instanceof Array) |
| 262 for (i = 0; i < obj.length; i++) | 262 for (let item of obj) |
| 263 if (hasNonASCI(obj[i])) | 263 if (hasNonASCI(item)) |
| 264 return true; | 264 return true; |
| 265 | 265 |
| 266 var names = Object.getOwnPropertyNames(obj); | 266 for (let name of Object.getOwnPropertyNames(obj)) |
| 267 for (i = 0; i < names.length; i++) | 267 if (hasNonASCI(obj[name])) |
| 268 if (hasNonASCI(obj[names[i]])) | |
| 269 return true; | 268 return true; |
| 270 } | 269 } |
| 271 | 270 |
| 272 return false; | 271 return false; |
| 273 } | 272 } |
| 274 | 273 |
| 275 function logRules() | 274 function logRules() |
| 276 { | 275 { |
| 277 var rules = []; | 276 let rules = []; |
| 278 var i; | |
| 279 | 277 |
| 280 function addRule(rule) | 278 function addRule(rule) |
| 281 { | 279 { |
| 282 if (!hasNonASCI(rule)) | 280 if (!hasNonASCI(rule)) |
| 283 rules.push(rule); | 281 rules.push(rule); |
| 284 } | 282 } |
| 285 | 283 |
| 286 // HACK: We ignore element hiding filter for now to get the list of | 284 // HACK: We ignore element hiding filter for now to get the list of |
| 287 // rules down below 50K. This limit is enforced by iOS and Safari. | 285 // rules down below 50K. This limit is enforced by iOS and Safari. |
| 288 // To be undone with https://issues.adblockplus.org/ticket/3585 | 286 // To be undone with https://issues.adblockplus.org/ticket/3585 |
| 289 | 287 |
| 290 //for (i = 0; i < elemhideFilters.length; i++) | 288 //for (let filter of elemhideFilters) |
| 291 // convertElemHideFilter(elemhideFilters[i]).forEach(addRule); | 289 // convertElemHideFilter(filter).forEach(addRule); |
| 292 //for (i = 0; i < elemhideExceptions.length; i++) | 290 //for (let filter of elemhideExceptions) |
| 293 // addRule(convertFilter(elemhideExceptions[i], "ignore-previous-rules", fals
e)); | 291 // addRule(convertFilter(filter, "ignore-previous-rules", false)); |
| 294 | 292 |
| 295 for (i = 0; i < requestFilters.length; i++) | 293 for (let filter of requestFilters) |
| 296 addRule(convertFilter(requestFilters[i], "block", true)); | 294 addRule(convertFilter(filter, "block", true)); |
| 297 for (i = 0; i < requestExceptions.length; i++) | 295 for (let filter of requestExceptions) |
| 298 addRule(convertFilter(requestExceptions[i], "ignore-previous-rules", true)); | 296 addRule(convertFilter(filter, "ignore-previous-rules", true)); |
| 299 | 297 |
| 300 console.log(JSON.stringify(rules, null, "\t")); | 298 console.log(JSON.stringify(rules, null, "\t")); |
| 301 } | 299 } |
| 302 | 300 |
| 303 var rl = readline.createInterface({input: process.stdin, terminal: false}); | 301 let rl = readline.createInterface({input: process.stdin, terminal: false}); |
| 304 rl.on("line", parseFilter); | 302 rl.on("line", parseFilter); |
| 305 rl.on("close", logRules); | 303 rl.on("close", logRules); |
| OLD | NEW |