| OLD | NEW |
| 1 var readline = require("readline"); | 1 var readline = require("readline"); |
| 2 var punycode = require("punycode"); | 2 var punycode = require("punycode"); |
| 3 var tldjs = require("tldjs"); | 3 var tldjs = require("tldjs"); |
| 4 var filterClasses = require("./adblockplus.js"); | 4 var filterClasses = require("./adblockplus.js"); |
| 5 | 5 |
| 6 var typeMap = filterClasses.RegExpFilter.typeMap; | 6 var typeMap = filterClasses.RegExpFilter.typeMap; |
| 7 | 7 |
| 8 var requestFilters = []; | 8 var requestFilters = []; |
| 9 var requestExceptions = []; | 9 var requestExceptions = []; |
| 10 var elemhideFilters = []; | 10 var elemhideFilters = []; |
| 11 var elemhideExceptions = []; | 11 var elemhideExceptions = []; |
| 12 var elemhideSelectorExceptions = Object.create(null); | 12 var elemhideSelectorExceptions = Object.create(null); |
| 13 | 13 |
| 14 function recordException(filter) { | 14 function recordException(filter) |
| 15 if (filter.contentType & (typeMap.IMAGE | 15 { |
| 16 | typeMap.STYLESHEET | 16 if (filter.contentType & (typeMap.IMAGE |
| 17 | typeMap.SCRIPT | 17 | typeMap.STYLESHEET |
| 18 | typeMap.FONT | 18 | typeMap.SCRIPT |
| 19 | typeMap.MEDIA | 19 | typeMap.FONT |
| 20 | typeMap.POPUP | 20 | typeMap.MEDIA |
| 21 | typeMap.OBJECT | 21 | typeMap.POPUP |
| 22 | typeMap.OBJECT_SUBREQUEST | 22 | typeMap.OBJECT |
| 23 | typeMap.XMLHTTPREQUEST | 23 | typeMap.OBJECT_SUBREQUEST |
| 24 | typeMap.PING | 24 | typeMap.XMLHTTPREQUEST |
| 25 | typeMap.SUBDOCUMENT | 25 | typeMap.PING |
| 26 | typeMap.OTHER)) | 26 | typeMap.SUBDOCUMENT |
| 27 requestExceptions.push(filter); | 27 | typeMap.OTHER)) |
| 28 | 28 requestExceptions.push(filter); |
| 29 if (filter.contentType & typeMap.ELEMHIDE) | 29 |
| 30 elemhideExceptions.push(filter); | 30 if (filter.contentType & typeMap.ELEMHIDE) |
| 31 } | 31 elemhideExceptions.push(filter); |
| 32 | 32 } |
| 33 function parseDomains(domains, included, excluded) { | 33 |
| 34 for (var domain in domains) { | 34 function parseDomains(domains, included, excluded) |
| 35 if (domain != "") { | 35 { |
| 36 var enabled = domains[domain]; | 36 for (var domain in domains) |
| 37 domain = punycode.toASCII(domain.toLowerCase()); | 37 { |
| 38 | 38 if (domain != "") |
| 39 if (!enabled) | 39 { |
| 40 excluded.push(domain); | 40 var enabled = domains[domain]; |
| 41 else if (!domains[""]) | 41 domain = punycode.toASCII(domain.toLowerCase()); |
| 42 included.push(domain); | 42 |
| 43 } | 43 if (!enabled) |
| 44 } | 44 excluded.push(domain); |
| 45 } | 45 else if (!domains[""]) |
| 46 | 46 included.push(domain); |
| 47 function recordSelectorException(filter) { | 47 } |
| 48 var domains = elemhideSelectorExceptions[filter.selector]; | 48 } |
| 49 if (!domains) | 49 } |
| 50 domains = elemhideSelectorExceptions[filter.selector] = []; | 50 |
| 51 | 51 function recordSelectorException(filter) |
| 52 parseDomains(filter.domains, domains, []); | 52 { |
| 53 } | 53 var domains = elemhideSelectorExceptions[filter.selector]; |
| 54 | 54 if (!domains) |
| 55 function parseFilter(line) { | 55 domains = elemhideSelectorExceptions[filter.selector] = []; |
| 56 if (line.charAt(0) == "[") | 56 |
| 57 return; | 57 parseDomains(filter.domains, domains, []); |
| 58 | 58 } |
| 59 var filter = filterClasses.Filter.fromText(line); | 59 |
| 60 | 60 function parseFilter(line) |
| 61 if (filter.sitekeys) | 61 { |
| 62 return; | 62 if (line.charAt(0) == "[") |
| 63 if (filter instanceof filterClasses.RegExpFilter && !filter.regexpSource
) | 63 return; |
| 64 return; | 64 |
| 65 | 65 var filter = filterClasses.Filter.fromText(line); |
| 66 if (filter instanceof filterClasses.BlockingFilter) | 66 |
| 67 requestFilters.push(filter); | 67 if (filter.sitekeys) |
| 68 if (filter instanceof filterClasses.WhitelistFilter) | 68 return; |
| 69 recordException(filter); | 69 if (filter instanceof filterClasses.RegExpFilter && !filter.regexpSource) |
| 70 if (filter instanceof filterClasses.ElemHideFilter) | 70 return; |
| 71 elemhideFilters.push(filter); | 71 |
| 72 if (filter instanceof filterClasses.ElemHideException) | 72 if (filter instanceof filterClasses.BlockingFilter) |
| 73 recordSelectorException(filter); | 73 requestFilters.push(filter); |
| 74 } | 74 if (filter instanceof filterClasses.WhitelistFilter) |
| 75 | 75 recordException(filter); |
| 76 function escapeRegExp(s) { | 76 if (filter instanceof filterClasses.ElemHideFilter) |
| 77 return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); | 77 elemhideFilters.push(filter); |
| 78 } | 78 if (filter instanceof filterClasses.ElemHideException) |
| 79 | 79 recordSelectorException(filter); |
| 80 function matchDomain(domain) { | 80 } |
| 81 return "^https?://([^/:]*\\.)?" + escapeRegExp(domain) + "[/:]"; | 81 |
| 82 } | 82 function escapeRegExp(s) |
| 83 | 83 { |
| 84 function convertElemHideFilter(filter) { | 84 return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); |
| 85 var included = []; | 85 } |
| 86 var excluded = []; | 86 |
| 87 var rules = []; | 87 function matchDomain(domain) |
| 88 | 88 { |
| 89 parseDomains(filter.domains, included, excluded); | 89 return "^https?://([^/:]*\\.)?" + escapeRegExp(domain) + "[/:]"; |
| 90 | 90 } |
| 91 if (excluded.length == 0 && !(filter.selector in elemhideSelectorExcepti
ons)) { | 91 |
| 92 var action = { | 92 function convertElemHideFilter(filter) |
| 93 type: "css-display-none", | 93 { |
| 94 selector: filter.selector | 94 var included = []; |
| 95 }; | 95 var excluded = []; |
| 96 | 96 var rules = []; |
| 97 for (var i = 0; i < included.length; i++) | 97 |
| 98 rules.push({ | 98 parseDomains(filter.domains, included, excluded); |
| 99 trigger: {"url-filter": matchDomain(included[i])
}, | 99 |
| 100 action: action | 100 if (excluded.length == 0 && !(filter.selector in elemhideSelectorExceptions)) |
| 101 }); | 101 { |
| 102 | 102 var action = { |
| 103 if (included.length == 0) | 103 type: "css-display-none", |
| 104 rules.push({ | 104 selector: filter.selector |
| 105 trigger: {"url-filter": "^https?://"}, | 105 }; |
| 106 action: action | 106 |
| 107 }); | 107 for (var i = 0; i < included.length; i++) |
| 108 } | 108 rules.push({ |
| 109 | 109 trigger: {"url-filter": matchDomain(included[i])}, |
| 110 return rules; | 110 action: action |
| 111 } | 111 }); |
| 112 | 112 |
| 113 function toRegExp(text) { | 113 if (included.length == 0) |
| 114 var result = ""; | 114 rules.push({ |
| 115 var lastIndex = text.length - 1; | 115 trigger: {"url-filter": "^https?://"}, |
| 116 | 116 action: action |
| 117 for (var i = 0; i < text.length; i++) | 117 }); |
| 118 { | 118 } |
| 119 var c = text[i]; | 119 |
| 120 | 120 return rules; |
| 121 switch (c) | 121 } |
| 122 { | 122 |
| 123 case "*": | 123 function toRegExp(text) |
| 124 if (result.length > 0 && i < lastIndex && text[i
+ 1] != "*") | 124 { |
| 125 result += ".*"; | 125 var result = ""; |
| 126 break; | 126 var lastIndex = text.length - 1; |
| 127 case "^": | 127 |
| 128 if (i < lastIndex) | 128 for (var i = 0; i < text.length; i++) |
| 129 result += "."; | 129 { |
| 130 break; | 130 var c = text[i]; |
| 131 case "|": | 131 |
| 132 if (i == 0) | 132 switch (c) |
| 133 { | 133 { |
| 134 result += "^"; | 134 case "*": |
| 135 break; | 135 if (result.length > 0 && i < lastIndex && text[i + 1] != "*") |
| 136 } | 136 result += ".*"; |
| 137 if (i == lastIndex) | 137 break; |
| 138 { | 138 case "^": |
| 139 result += "$"; | 139 if (i < lastIndex) |
| 140 break; | 140 result += "."; |
| 141 } | 141 break; |
| 142 if (i == 1 && text[0] == "|") | 142 case "|": |
| 143 { | 143 if (i == 0) |
| 144 result += "https?://"; | 144 { |
| 145 break; | 145 result += "^"; |
| 146 } | 146 break; |
| 147 case ".": case "+": case "?": case "$": | 147 } |
| 148 case "{": case "}": case "(": case ")": | 148 if (i == lastIndex) |
| 149 case "[": case "]": case "\\": | 149 { |
| 150 result += "\\"; | 150 result += "$"; |
| 151 default: | 151 break; |
| 152 result += c; | 152 } |
| 153 } | 153 if (i == 1 && text[0] == "|") |
| 154 } | 154 { |
| 155 | 155 result += "https?://"; |
| 156 return result; | 156 break; |
| 157 } | 157 } |
| 158 | 158 case ".": case "+": case "?": case "$": |
| 159 function getRegExpSource(filter) { | 159 case "{": case "}": case "(": case ")": |
| 160 var source = toRegExp(filter.regexpSource.replace( | 160 case "[": case "]": case "\\": |
| 161 // Safari expects punycode, filter lists use unicode | 161 result += "\\"; |
| 162 /^(\|\||\|?https?:\/\/)([\w\-.*\u0080-\uFFFF]+)/i, | 162 default: |
| 163 function (match, prefix, domain) { | 163 result += c; |
| 164 return prefix + punycode.toASCII(domain); | 164 } |
| 165 } | 165 } |
| 166 )); | 166 |
| 167 | 167 return result; |
| 168 // Limit rules to to HTTP(S) URLs | 168 } |
| 169 if (!/^(\^|http)/i.test(source)) | 169 |
| 170 source = "^https?://.*" + source; | 170 function getRegExpSource(filter) |
| 171 | 171 { |
| 172 return source; | 172 var source = toRegExp(filter.regexpSource.replace( |
| 173 } | 173 // Safari expects punycode, filter lists use unicode |
| 174 | 174 /^(\|\||\|?https?:\/\/)([\w\-.*\u0080-\uFFFF]+)/i, |
| 175 function getResourceTypes(filter) { | 175 function (match, prefix, domain) |
| 176 var types = []; | 176 { |
| 177 | 177 return prefix + punycode.toASCII(domain); |
| 178 if (filter.contentType & typeMap.IMAGE) | 178 } |
| 179 types.push("image"); | 179 )); |
| 180 if (filter.contentType & typeMap.STYLESHEET) | 180 |
| 181 types.push("style-sheet"); | 181 // Limit rules to to HTTP(S) URLs |
| 182 if (filter.contentType & typeMap.SCRIPT) | 182 if (!/^(\^|http)/i.test(source)) |
| 183 types.push("script"); | 183 source = "^https?://.*" + source; |
| 184 if (filter.contentType & typeMap.FONT) | 184 |
| 185 types.push("font"); | 185 return source; |
| 186 if (filter.contentType & (typeMap.MEDIA | typeMap.OBJECT)) | 186 } |
| 187 types.push("media"); | 187 |
| 188 if (filter.contentType & typeMap.POPUP) | 188 function getResourceTypes(filter) |
| 189 types.push("popup"); | 189 { |
| 190 if (filter.contentType & (typeMap.XMLHTTPREQUEST | typeMap.OBJECT_SUBREQ
UEST | typeMap.PING | typeMap.OTHER)) | 190 var types = []; |
| 191 types.push("raw"); | 191 |
| 192 if (filter.contentType & typeMap.SUBDOCUMENT) | 192 if (filter.contentType & typeMap.IMAGE) |
| 193 types.push("document"); | 193 types.push("image"); |
| 194 | 194 if (filter.contentType & typeMap.STYLESHEET) |
| 195 return types; | 195 types.push("style-sheet"); |
| 196 } | 196 if (filter.contentType & typeMap.SCRIPT) |
| 197 | 197 types.push("script"); |
| 198 function addDomainPrefix(domains) { | 198 if (filter.contentType & typeMap.FONT) |
| 199 var result = []; | 199 types.push("font"); |
| 200 | 200 if (filter.contentType & (typeMap.MEDIA | typeMap.OBJECT)) |
| 201 for (var i = 0; i < domains.length; i++) { | 201 types.push("media"); |
| 202 var domain = domains[i]; | 202 if (filter.contentType & typeMap.POPUP) |
| 203 result.push(domain); | 203 types.push("popup"); |
| 204 | 204 if (filter.contentType & (typeMap.XMLHTTPREQUEST | typeMap.OBJECT_SUBREQUEST |
| 205 if (tldjs.getSubdomain(domain) == "") | 205 | typeMap.PING | typeMap.OTHER)) |
| 206 result.push("www." + domain); | 206 types.push("raw"); |
| 207 } | 207 if (filter.contentType & typeMap.SUBDOCUMENT) |
| 208 | 208 types.push("document"); |
| 209 return result; | 209 |
| 210 } | 210 return types; |
| 211 | 211 } |
| 212 function convertFilter(filter, action, withResourceTypes) { | 212 |
| 213 var trigger = {"url-filter": getRegExpSource(filter)}; | 213 function addDomainPrefix(domains) |
| 214 var included = []; | 214 { |
| 215 var excluded = []; | 215 var result = []; |
| 216 | 216 |
| 217 parseDomains(filter.domains, included, excluded); | 217 for (var i = 0; i < domains.length; i++) |
| 218 | 218 { |
| 219 if (withResourceTypes) | 219 var domain = domains[i]; |
| 220 trigger["resource-type"] = getResourceTypes(filter); | 220 result.push(domain); |
| 221 if (filter.thirdParty != null) | 221 |
| 222 trigger["load-type"] = [filter.thirdParty ? "third-party" : "fir
st-party"]; | 222 if (tldjs.getSubdomain(domain) == "") |
| 223 | 223 result.push("www." + domain); |
| 224 if (included.length > 0) | 224 } |
| 225 trigger["if-domain"] = addDomainPrefix(included); | 225 |
| 226 else if (excluded.length > 0) | 226 return result; |
| 227 trigger["unless-domain"] = addDomainPrefix(excluded); | 227 } |
| 228 | 228 |
| 229 return {trigger: trigger, action: {type: action}}; | 229 function convertFilter(filter, action, withResourceTypes) |
| 230 } | 230 { |
| 231 | 231 var trigger = {"url-filter": getRegExpSource(filter)}; |
| 232 function hasNonASCI(obj) { | 232 var included = []; |
| 233 if (typeof obj == "string") { | 233 var excluded = []; |
| 234 if (/[^\x00-\x7F]/.test(obj)) | 234 |
| 235 return true; | 235 parseDomains(filter.domains, included, excluded); |
| 236 } | 236 |
| 237 | 237 if (withResourceTypes) |
| 238 if (typeof obj == "object") { | 238 trigger["resource-type"] = getResourceTypes(filter); |
| 239 var i; | 239 if (filter.thirdParty != null) |
| 240 if (obj instanceof Array) | 240 trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"]; |
| 241 for (i = 0; i < obj.length; i++) | 241 |
| 242 if (hasNonASCI(obj[i])) | 242 if (included.length > 0) |
| 243 return true; | 243 trigger["if-domain"] = addDomainPrefix(included); |
| 244 | 244 else if (excluded.length > 0) |
| 245 var names = Object.getOwnPropertyNames(obj); | 245 trigger["unless-domain"] = addDomainPrefix(excluded); |
| 246 for (i = 0; i < names.length; i++) | 246 |
| 247 if (hasNonASCI(obj[names[i]])) | 247 return {trigger: trigger, action: {type: action}}; |
| 248 return true; | 248 } |
| 249 } | 249 |
| 250 | 250 function hasNonASCI(obj) |
| 251 return false; | 251 { |
| 252 } | 252 if (typeof obj == "string") |
| 253 | 253 { |
| 254 function logRules() { | 254 if (/[^\x00-\x7F]/.test(obj)) |
| 255 var rules = []; | 255 return true; |
| 256 var i; | 256 } |
| 257 | 257 |
| 258 function addRule(rule) { | 258 if (typeof obj == "object") |
| 259 if (!hasNonASCI(rule)) | 259 { |
| 260 rules.push(rule); | 260 var i; |
| 261 } | 261 if (obj instanceof Array) |
| 262 | 262 for (i = 0; i < obj.length; i++) |
| 263 // HACK: We ignore element hiding filter for now to get the list of | 263 if (hasNonASCI(obj[i])) |
| 264 // rules down below 50K. This limit is enforced by iOS and Safari. | 264 return true; |
| 265 // To be undone with https://issues.adblockplus.org/ticket/3585 | 265 |
| 266 | 266 var names = Object.getOwnPropertyNames(obj); |
| 267 //for (i = 0; i < elemhideFilters.length; i++) | 267 for (i = 0; i < names.length; i++) |
| 268 // convertElemHideFilter(elemhideFilters[i]).forEach(addRule); | 268 if (hasNonASCI(obj[names[i]])) |
| 269 //for (i = 0; i < elemhideExceptions.length; i++) | 269 return true; |
| 270 // addRule(convertFilter(elemhideExceptions[i], "ignore-previous-ru
les", false)); | 270 } |
| 271 | 271 |
| 272 for (i = 0; i < requestFilters.length; i++) | 272 return false; |
| 273 addRule(convertFilter(requestFilters[i], "block", true)); | 273 } |
| 274 for (i = 0; i < requestExceptions.length; i++) | 274 |
| 275 addRule(convertFilter(requestExceptions[i], "ignore-previous-rul
es", true)); | 275 function logRules() |
| 276 | 276 { |
| 277 console.log(JSON.stringify(rules, null, "\t")); | 277 var rules = []; |
| 278 var i; |
| 279 |
| 280 // HACK: We ignore element hiding filter for now to get the list of |
| 281 // rules down below 50K. This limit is enforced by iOS and Safari. |
| 282 // To be undone with https://issues.adblockplus.org/ticket/3585 |
| 283 function addRule(rule) |
| 284 { |
| 285 if (!hasNonASCI(rule)) |
| 286 rules.push(rule); |
| 287 } |
| 288 |
| 289 //for (i = 0; i < elemhideFilters.length; i++) |
| 290 // convertElemHideFilter(elemhideFilters[i]).forEach(addRule); |
| 291 //for (i = 0; i < elemhideExceptions.length; i++) |
| 292 // addRule(convertFilter(elemhideExceptions[i], "ignore-previous-rules", fals
e)); |
| 293 |
| 294 for (i = 0; i < requestFilters.length; i++) |
| 295 addRule(convertFilter(requestFilters[i], "block", true)); |
| 296 for (i = 0; i < requestExceptions.length; i++) |
| 297 addRule(convertFilter(requestExceptions[i], "ignore-previous-rules", true)); |
| 298 |
| 299 console.log(JSON.stringify(rules, null, "\t")); |
| 278 } | 300 } |
| 279 | 301 |
| 280 var rl = readline.createInterface({input: process.stdin, terminal: false}); | 302 var rl = readline.createInterface({input: process.stdin, terminal: false}); |
| 281 rl.on("line", parseFilter); | 303 rl.on("line", parseFilter); |
| 282 rl.on("close", logRules); | 304 rl.on("close", logRules); |
| OLD | NEW |