OLD | NEW |
1 var readline = require("readline"); | 1 "use strict"; |
2 var punycode = require("punycode"); | |
3 var tldjs = require("tldjs"); | |
4 var filterClasses = require("./adblockplus.js"); | |
5 | 2 |
6 var typeMap = filterClasses.RegExpFilter.typeMap; | 3 let readline = require("readline"); |
| 4 let punycode = require("punycode"); |
| 5 let tldjs = require("tldjs"); |
| 6 let filterClasses = require("./adblockplus.js"); |
7 | 7 |
8 var requestFilters = []; | 8 let typeMap = filterClasses.RegExpFilter.typeMap; |
9 var requestExceptions = []; | 9 |
10 var elemhideFilters = []; | 10 let requestFilters = []; |
11 var elemhideExceptions = []; | 11 let requestExceptions = []; |
12 var elemhideSelectorExceptions = Object.create(null); | 12 let elemhideFilters = []; |
| 13 let elemhideExceptions = []; |
| 14 let elemhideSelectorExceptions = new Map(); |
13 | 15 |
14 function recordException(filter) | 16 function recordException(filter) |
15 { | 17 { |
16 if (filter.contentType & (typeMap.IMAGE | 18 if (filter.contentType & (typeMap.IMAGE |
17 | typeMap.STYLESHEET | 19 | typeMap.STYLESHEET |
18 | typeMap.SCRIPT | 20 | typeMap.SCRIPT |
19 | typeMap.FONT | 21 | typeMap.FONT |
20 | typeMap.MEDIA | 22 | typeMap.MEDIA |
21 | typeMap.POPUP | 23 | typeMap.POPUP |
22 | typeMap.OBJECT | 24 | typeMap.OBJECT |
23 | typeMap.OBJECT_SUBREQUEST | 25 | typeMap.OBJECT_SUBREQUEST |
24 | typeMap.XMLHTTPREQUEST | 26 | typeMap.XMLHTTPREQUEST |
25 | typeMap.PING | 27 | typeMap.PING |
26 | typeMap.SUBDOCUMENT | 28 | typeMap.SUBDOCUMENT |
27 | typeMap.OTHER)) | 29 | typeMap.OTHER)) |
28 requestExceptions.push(filter); | 30 requestExceptions.push(filter); |
29 | 31 |
30 if (filter.contentType & typeMap.ELEMHIDE) | 32 if (filter.contentType & typeMap.ELEMHIDE) |
31 elemhideExceptions.push(filter); | 33 elemhideExceptions.push(filter); |
32 } | 34 } |
33 | 35 |
34 function parseDomains(domains, included, excluded) | 36 function parseDomains(domains, included, excluded) |
35 { | 37 { |
36 for (var domain in domains) | 38 for (let domain in domains) |
37 { | 39 { |
38 if (domain != "") | 40 if (domain != "") |
39 { | 41 { |
40 var enabled = domains[domain]; | 42 let enabled = domains[domain]; |
41 domain = punycode.toASCII(domain.toLowerCase()); | 43 domain = punycode.toASCII(domain.toLowerCase()); |
42 | 44 |
43 if (!enabled) | 45 if (!enabled) |
44 excluded.push(domain); | 46 excluded.push(domain); |
45 else if (!domains[""]) | 47 else if (!domains[""]) |
46 included.push(domain); | 48 included.push(domain); |
47 } | 49 } |
48 } | 50 } |
49 } | 51 } |
50 | 52 |
51 function recordSelectorException(filter) | 53 function recordSelectorException(filter) |
52 { | 54 { |
53 var domains = elemhideSelectorExceptions[filter.selector]; | 55 let domains = elemhideSelectorExceptions[filter.selector]; |
54 if (!domains) | 56 if (!domains) |
55 domains = elemhideSelectorExceptions[filter.selector] = []; | 57 domains = elemhideSelectorExceptions[filter.selector] = []; |
56 | 58 |
57 parseDomains(filter.domains, domains, []); | 59 parseDomains(filter.domains, domains, []); |
58 } | 60 } |
59 | 61 |
60 function parseFilter(line) | 62 function parseFilter(line) |
61 { | 63 { |
62 if (line.charAt(0) == "[") | 64 if (line.charAt(0) == "[") |
63 return; | 65 return; |
64 | 66 |
65 var filter = filterClasses.Filter.fromText(line); | 67 let filter = filterClasses.Filter.fromText(line); |
66 | 68 |
67 if (filter.sitekeys) | 69 if (filter.sitekeys) |
68 return; | 70 return; |
69 if (filter instanceof filterClasses.RegExpFilter && !filter.regexpSource) | 71 if (filter instanceof filterClasses.RegExpFilter && !filter.regexpSource) |
70 return; | 72 return; |
71 | 73 |
72 if (filter instanceof filterClasses.BlockingFilter) | 74 if (filter instanceof filterClasses.BlockingFilter) |
73 requestFilters.push(filter); | 75 requestFilters.push(filter); |
74 if (filter instanceof filterClasses.WhitelistFilter) | 76 if (filter instanceof filterClasses.WhitelistFilter) |
75 recordException(filter); | 77 recordException(filter); |
76 if (filter instanceof filterClasses.ElemHideFilter) | 78 if (filter instanceof filterClasses.ElemHideFilter) |
77 elemhideFilters.push(filter); | 79 elemhideFilters.push(filter); |
78 if (filter instanceof filterClasses.ElemHideException) | 80 if (filter instanceof filterClasses.ElemHideException) |
79 recordSelectorException(filter); | 81 recordSelectorException(filter); |
80 } | 82 } |
81 | 83 |
82 function escapeRegExp(s) | 84 function escapeRegExp(s) |
83 { | 85 { |
84 return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); | 86 return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); |
85 } | 87 } |
86 | 88 |
87 function matchDomain(domain) | 89 function matchDomain(domain) |
88 { | 90 { |
89 return "^https?://([^/:]*\\.)?" + escapeRegExp(domain) + "[/:]"; | 91 return "^https?://([^/:]*\\.)?" + escapeRegExp(domain) + "[/:]"; |
90 } | 92 } |
91 | 93 |
92 function convertElemHideFilter(filter) | 94 function convertElemHideFilter(filter) |
93 { | 95 { |
94 var included = []; | 96 let included = []; |
95 var excluded = []; | 97 let excluded = []; |
96 var rules = []; | 98 let rules = []; |
97 | 99 |
98 parseDomains(filter.domains, included, excluded); | 100 parseDomains(filter.domains, included, excluded); |
99 | 101 |
100 if (excluded.length == 0 && !(filter.selector in elemhideSelectorExceptions)) | 102 if (excluded.length == 0 && !(filter.selector in elemhideSelectorExceptions)) |
101 { | 103 { |
102 var action = { | 104 let action = { |
103 type: "css-display-none", | 105 type: "css-display-none", |
104 selector: filter.selector | 106 selector: filter.selector |
105 }; | 107 }; |
106 | 108 |
107 for (var i = 0; i < included.length; i++) | 109 for (let domain of included) |
108 rules.push({ | 110 rules.push({ |
109 trigger: {"url-filter": matchDomain(included[i])}, | 111 trigger: {"url-filter": matchDomain(domain)}, |
110 action: action | 112 action: action |
111 }); | 113 }); |
112 | 114 |
113 if (included.length == 0) | 115 if (included.length == 0) |
114 rules.push({ | 116 rules.push({ |
115 trigger: {"url-filter": "^https?://"}, | 117 trigger: {"url-filter": "^https?://"}, |
116 action: action | 118 action: action |
117 }); | 119 }); |
118 } | 120 } |
119 | 121 |
120 return rules; | 122 return rules; |
121 } | 123 } |
122 | 124 |
123 function toRegExp(text) | 125 function toRegExp(text) |
124 { | 126 { |
125 var result = ""; | 127 let result = ""; |
126 var lastIndex = text.length - 1; | 128 let lastIndex = text.length - 1; |
127 | 129 |
128 for (var i = 0; i < text.length; i++) | 130 for (let i = 0; i < text.length; i++) |
129 { | 131 { |
130 var c = text[i]; | 132 let c = text[i]; |
131 | 133 |
132 switch (c) | 134 switch (c) |
133 { | 135 { |
134 case "*": | 136 case "*": |
135 if (result.length > 0 && i < lastIndex && text[i + 1] != "*") | 137 if (result.length > 0 && i < lastIndex && text[i + 1] != "*") |
136 result += ".*"; | 138 result += ".*"; |
137 break; | 139 break; |
138 case "^": | 140 case "^": |
139 if (i < lastIndex) | 141 if (i < lastIndex) |
140 result += "."; | 142 result += "."; |
(...skipping 21 matching lines...) Expand all Loading... |
162 default: | 164 default: |
163 result += c; | 165 result += c; |
164 } | 166 } |
165 } | 167 } |
166 | 168 |
167 return result; | 169 return result; |
168 } | 170 } |
169 | 171 |
170 function getRegExpSource(filter) | 172 function getRegExpSource(filter) |
171 { | 173 { |
172 var source = toRegExp(filter.regexpSource.replace( | 174 let source = toRegExp(filter.regexpSource.replace( |
173 // Safari expects punycode, filter lists use unicode | 175 // Safari expects punycode, filter lists use unicode |
174 /^(\|\||\|?https?:\/\/)([\w\-.*\u0080-\uFFFF]+)/i, | 176 /^(\|\||\|?https?:\/\/)([\w\-.*\u0080-\uFFFF]+)/i, |
175 function (match, prefix, domain) | 177 function (match, prefix, domain) |
176 { | 178 { |
177 return prefix + punycode.toASCII(domain); | 179 return prefix + punycode.toASCII(domain); |
178 } | 180 } |
179 )); | 181 )); |
180 | 182 |
181 // Limit rules to to HTTP(S) URLs | 183 // Limit rules to to HTTP(S) URLs |
182 if (!/^(\^|http)/i.test(source)) | 184 if (!/^(\^|http)/i.test(source)) |
183 source = "^https?://.*" + source; | 185 source = "^https?://.*" + source; |
184 | 186 |
185 return source; | 187 return source; |
186 } | 188 } |
187 | 189 |
188 function getResourceTypes(filter) | 190 function getResourceTypes(filter) |
189 { | 191 { |
190 var types = []; | 192 let types = []; |
191 | 193 |
192 if (filter.contentType & typeMap.IMAGE) | 194 if (filter.contentType & typeMap.IMAGE) |
193 types.push("image"); | 195 types.push("image"); |
194 if (filter.contentType & typeMap.STYLESHEET) | 196 if (filter.contentType & typeMap.STYLESHEET) |
195 types.push("style-sheet"); | 197 types.push("style-sheet"); |
196 if (filter.contentType & typeMap.SCRIPT) | 198 if (filter.contentType & typeMap.SCRIPT) |
197 types.push("script"); | 199 types.push("script"); |
198 if (filter.contentType & typeMap.FONT) | 200 if (filter.contentType & typeMap.FONT) |
199 types.push("font"); | 201 types.push("font"); |
200 if (filter.contentType & (typeMap.MEDIA | typeMap.OBJECT)) | 202 if (filter.contentType & (typeMap.MEDIA | typeMap.OBJECT)) |
201 types.push("media"); | 203 types.push("media"); |
202 if (filter.contentType & typeMap.POPUP) | 204 if (filter.contentType & typeMap.POPUP) |
203 types.push("popup"); | 205 types.push("popup"); |
204 if (filter.contentType & (typeMap.XMLHTTPREQUEST | typeMap.OBJECT_SUBREQUEST | 206 if (filter.contentType & (typeMap.XMLHTTPREQUEST | typeMap.OBJECT_SUBREQUEST |
205 | typeMap.PING | typeMap.OTHER)) | 207 | typeMap.PING | typeMap.OTHER)) |
206 types.push("raw"); | 208 types.push("raw"); |
207 if (filter.contentType & typeMap.SUBDOCUMENT) | 209 if (filter.contentType & typeMap.SUBDOCUMENT) |
208 types.push("document"); | 210 types.push("document"); |
209 | 211 |
210 return types; | 212 return types; |
211 } | 213 } |
212 | 214 |
213 function addDomainPrefix(domains) | 215 function addDomainPrefix(domains) |
214 { | 216 { |
215 var result = []; | 217 let result = []; |
216 | 218 |
217 for (var i = 0; i < domains.length; i++) | 219 for (let domain of domains) |
218 { | 220 { |
219 var domain = domains[i]; | |
220 result.push(domain); | 221 result.push(domain); |
221 | 222 |
222 if (tldjs.getSubdomain(domain) == "") | 223 if (tldjs.getSubdomain(domain) == "") |
223 result.push("www." + domain); | 224 result.push("www." + domain); |
224 } | 225 } |
225 | 226 |
226 return result; | 227 return result; |
227 } | 228 } |
228 | 229 |
229 function convertFilter(filter, action, withResourceTypes) | 230 function convertFilter(filter, action, withResourceTypes) |
230 { | 231 { |
231 var trigger = {"url-filter": getRegExpSource(filter)}; | 232 let trigger = {"url-filter": getRegExpSource(filter)}; |
232 var included = []; | 233 let included = []; |
233 var excluded = []; | 234 let excluded = []; |
234 | 235 |
235 parseDomains(filter.domains, included, excluded); | 236 parseDomains(filter.domains, included, excluded); |
236 | 237 |
237 if (withResourceTypes) | 238 if (withResourceTypes) |
238 trigger["resource-type"] = getResourceTypes(filter); | 239 trigger["resource-type"] = getResourceTypes(filter); |
239 if (filter.thirdParty != null) | 240 if (filter.thirdParty != null) |
240 trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"]; | 241 trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"]; |
241 | 242 |
242 if (included.length > 0) | 243 if (included.length > 0) |
243 trigger["if-domain"] = addDomainPrefix(included); | 244 trigger["if-domain"] = addDomainPrefix(included); |
244 else if (excluded.length > 0) | 245 else if (excluded.length > 0) |
245 trigger["unless-domain"] = addDomainPrefix(excluded); | 246 trigger["unless-domain"] = addDomainPrefix(excluded); |
246 | 247 |
247 return {trigger: trigger, action: {type: action}}; | 248 return {trigger: trigger, action: {type: action}}; |
248 } | 249 } |
249 | 250 |
250 function hasNonASCI(obj) | 251 function hasNonASCI(obj) |
251 { | 252 { |
252 if (typeof obj == "string") | 253 if (typeof obj == "string") |
253 { | 254 { |
254 if (/[^\x00-\x7F]/.test(obj)) | 255 if (/[^\x00-\x7F]/.test(obj)) |
255 return true; | 256 return true; |
256 } | 257 } |
257 | 258 |
258 if (typeof obj == "object") | 259 if (typeof obj == "object") |
259 { | 260 { |
260 var i; | |
261 if (obj instanceof Array) | 261 if (obj instanceof Array) |
262 for (i = 0; i < obj.length; i++) | 262 for (let item of obj) |
263 if (hasNonASCI(obj[i])) | 263 if (hasNonASCI(item)) |
264 return true; | 264 return true; |
265 | 265 |
266 var names = Object.getOwnPropertyNames(obj); | 266 for (let name of Object.getOwnPropertyNames(obj)) |
267 for (i = 0; i < names.length; i++) | 267 if (hasNonASCI(obj[name])) |
268 if (hasNonASCI(obj[names[i]])) | |
269 return true; | 268 return true; |
270 } | 269 } |
271 | 270 |
272 return false; | 271 return false; |
273 } | 272 } |
274 | 273 |
275 function logRules() | 274 function logRules() |
276 { | 275 { |
277 var rules = []; | 276 let rules = []; |
278 var i; | |
279 | 277 |
280 function addRule(rule) | 278 function addRule(rule) |
281 { | 279 { |
282 if (!hasNonASCI(rule)) | 280 if (!hasNonASCI(rule)) |
283 rules.push(rule); | 281 rules.push(rule); |
284 } | 282 } |
285 | 283 |
286 // HACK: We ignore element hiding filter for now to get the list of | 284 // HACK: We ignore element hiding filter for now to get the list of |
287 // rules down below 50K. This limit is enforced by iOS and Safari. | 285 // rules down below 50K. This limit is enforced by iOS and Safari. |
288 // To be undone with https://issues.adblockplus.org/ticket/3585 | 286 // To be undone with https://issues.adblockplus.org/ticket/3585 |
289 | 287 |
290 //for (i = 0; i < elemhideFilters.length; i++) | 288 //for (let filter of elemhideFilters) |
291 // convertElemHideFilter(elemhideFilters[i]).forEach(addRule); | 289 // convertElemHideFilter(filter).forEach(addRule); |
292 //for (i = 0; i < elemhideExceptions.length; i++) | 290 //for (let filter of elemhideExceptions) |
293 // addRule(convertFilter(elemhideExceptions[i], "ignore-previous-rules", fals
e)); | 291 // addRule(convertFilter(filter, "ignore-previous-rules", false)); |
294 | 292 |
295 for (i = 0; i < requestFilters.length; i++) | 293 for (let filter of requestFilters) |
296 addRule(convertFilter(requestFilters[i], "block", true)); | 294 addRule(convertFilter(filter, "block", true)); |
297 for (i = 0; i < requestExceptions.length; i++) | 295 for (let filter of requestExceptions) |
298 addRule(convertFilter(requestExceptions[i], "ignore-previous-rules", true)); | 296 addRule(convertFilter(filter, "ignore-previous-rules", true)); |
299 | 297 |
300 console.log(JSON.stringify(rules, null, "\t")); | 298 console.log(JSON.stringify(rules, null, "\t")); |
301 } | 299 } |
302 | 300 |
303 var rl = readline.createInterface({input: process.stdin, terminal: false}); | 301 let rl = readline.createInterface({input: process.stdin, terminal: false}); |
304 rl.on("line", parseFilter); | 302 rl.on("line", parseFilter); |
305 rl.on("close", logRules); | 303 rl.on("close", logRules); |
OLD | NEW |