OLD | NEW |
1 var readline = require("readline"); | 1 var readline = require("readline"); |
2 var punycode = require("punycode"); | 2 var punycode = require("punycode"); |
3 var tldjs = require("tldjs"); | 3 var tldjs = require("tldjs"); |
4 var filterClasses = require("./adblockplus.js"); | 4 var filterClasses = require("./adblockplus.js"); |
5 | 5 |
6 var typeMap = filterClasses.RegExpFilter.typeMap; | 6 var typeMap = filterClasses.RegExpFilter.typeMap; |
7 | 7 |
8 var requestFilters = []; | 8 var requestFilters = []; |
9 var requestExceptions = []; | 9 var requestExceptions = []; |
10 var elemhideFilters = []; | 10 var elemhideFilters = []; |
11 var elemhideExceptions = []; | 11 var elemhideExceptions = []; |
12 var elemhideSelectorExceptions = Object.create(null); | 12 var elemhideSelectorExceptions = Object.create(null); |
13 | 13 |
14 function recordException(filter) { | 14 function recordException(filter) |
15 if (filter.contentType & (typeMap.IMAGE | 15 { |
16 | typeMap.STYLESHEET | 16 if (filter.contentType & (typeMap.IMAGE |
17 | typeMap.SCRIPT | 17 | typeMap.STYLESHEET |
18 | typeMap.FONT | 18 | typeMap.SCRIPT |
19 | typeMap.MEDIA | 19 | typeMap.FONT |
20 | typeMap.POPUP | 20 | typeMap.MEDIA |
21 | typeMap.OBJECT | 21 | typeMap.POPUP |
22 | typeMap.OBJECT_SUBREQUEST | 22 | typeMap.OBJECT |
23 | typeMap.XMLHTTPREQUEST | 23 | typeMap.OBJECT_SUBREQUEST |
24 | typeMap.PING | 24 | typeMap.XMLHTTPREQUEST |
25 | typeMap.SUBDOCUMENT | 25 | typeMap.PING |
26 | typeMap.OTHER)) | 26 | typeMap.SUBDOCUMENT |
27 requestExceptions.push(filter); | 27 | typeMap.OTHER)) |
28 | 28 requestExceptions.push(filter); |
29 if (filter.contentType & typeMap.ELEMHIDE) | 29 |
30 elemhideExceptions.push(filter); | 30 if (filter.contentType & typeMap.ELEMHIDE) |
31 } | 31 elemhideExceptions.push(filter); |
32 | 32 } |
33 function parseDomains(domains, included, excluded) { | 33 |
34 for (var domain in domains) { | 34 function parseDomains(domains, included, excluded) |
35 if (domain != "") { | 35 { |
36 var enabled = domains[domain]; | 36 for (var domain in domains) |
37 domain = punycode.toASCII(domain.toLowerCase()); | 37 { |
38 | 38 if (domain != "") |
39 if (!enabled) | 39 { |
40 excluded.push(domain); | 40 var enabled = domains[domain]; |
41 else if (!domains[""]) | 41 domain = punycode.toASCII(domain.toLowerCase()); |
42 included.push(domain); | 42 |
43 } | 43 if (!enabled) |
44 } | 44 excluded.push(domain); |
45 } | 45 else if (!domains[""]) |
46 | 46 included.push(domain); |
47 function recordSelectorException(filter) { | 47 } |
48 var domains = elemhideSelectorExceptions[filter.selector]; | 48 } |
49 if (!domains) | 49 } |
50 domains = elemhideSelectorExceptions[filter.selector] = []; | 50 |
51 | 51 function recordSelectorException(filter) |
52 parseDomains(filter.domains, domains, []); | 52 { |
53 } | 53 var domains = elemhideSelectorExceptions[filter.selector]; |
54 | 54 if (!domains) |
55 function parseFilter(line) { | 55 domains = elemhideSelectorExceptions[filter.selector] = []; |
56 if (line.charAt(0) == "[") | 56 |
57 return; | 57 parseDomains(filter.domains, domains, []); |
58 | 58 } |
59 var filter = filterClasses.Filter.fromText(line); | 59 |
60 | 60 function parseFilter(line) |
61 if (filter.sitekeys) | 61 { |
62 return; | 62 if (line.charAt(0) == "[") |
63 if (filter instanceof filterClasses.RegExpFilter && !filter.regexpSource
) | 63 return; |
64 return; | 64 |
65 | 65 var filter = filterClasses.Filter.fromText(line); |
66 if (filter instanceof filterClasses.BlockingFilter) | 66 |
67 requestFilters.push(filter); | 67 if (filter.sitekeys) |
68 if (filter instanceof filterClasses.WhitelistFilter) | 68 return; |
69 recordException(filter); | 69 if (filter instanceof filterClasses.RegExpFilter && !filter.regexpSource) |
70 if (filter instanceof filterClasses.ElemHideFilter) | 70 return; |
71 elemhideFilters.push(filter); | 71 |
72 if (filter instanceof filterClasses.ElemHideException) | 72 if (filter instanceof filterClasses.BlockingFilter) |
73 recordSelectorException(filter); | 73 requestFilters.push(filter); |
74 } | 74 if (filter instanceof filterClasses.WhitelistFilter) |
75 | 75 recordException(filter); |
76 function escapeRegExp(s) { | 76 if (filter instanceof filterClasses.ElemHideFilter) |
77 return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); | 77 elemhideFilters.push(filter); |
78 } | 78 if (filter instanceof filterClasses.ElemHideException) |
79 | 79 recordSelectorException(filter); |
80 function matchDomain(domain) { | 80 } |
81 return "^https?://([^/:]*\\.)?" + escapeRegExp(domain) + "[/:]"; | 81 |
82 } | 82 function escapeRegExp(s) |
83 | 83 { |
84 function convertElemHideFilter(filter) { | 84 return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); |
85 var included = []; | 85 } |
86 var excluded = []; | 86 |
87 var rules = []; | 87 function matchDomain(domain) |
88 | 88 { |
89 parseDomains(filter.domains, included, excluded); | 89 return "^https?://([^/:]*\\.)?" + escapeRegExp(domain) + "[/:]"; |
90 | 90 } |
91 if (excluded.length == 0 && !(filter.selector in elemhideSelectorExcepti
ons)) { | 91 |
92 var action = { | 92 function convertElemHideFilter(filter) |
93 type: "css-display-none", | 93 { |
94 selector: filter.selector | 94 var included = []; |
95 }; | 95 var excluded = []; |
96 | 96 var rules = []; |
97 for (var i = 0; i < included.length; i++) | 97 |
98 rules.push({ | 98 parseDomains(filter.domains, included, excluded); |
99 trigger: {"url-filter": matchDomain(included[i])
}, | 99 |
100 action: action | 100 if (excluded.length == 0 && !(filter.selector in elemhideSelectorExceptions)) |
101 }); | 101 { |
102 | 102 var action = { |
103 if (included.length == 0) | 103 type: "css-display-none", |
104 rules.push({ | 104 selector: filter.selector |
105 trigger: {"url-filter": "^https?://"}, | 105 }; |
106 action: action | 106 |
107 }); | 107 for (var i = 0; i < included.length; i++) |
108 } | 108 rules.push({ |
109 | 109 trigger: {"url-filter": matchDomain(included[i])}, |
110 return rules; | 110 action: action |
111 } | 111 }); |
112 | 112 |
113 function toRegExp(text) { | 113 if (included.length == 0) |
114 var result = ""; | 114 rules.push({ |
115 var lastIndex = text.length - 1; | 115 trigger: {"url-filter": "^https?://"}, |
116 | 116 action: action |
117 for (var i = 0; i < text.length; i++) | 117 }); |
118 { | 118 } |
119 var c = text[i]; | 119 |
120 | 120 return rules; |
121 switch (c) | 121 } |
122 { | 122 |
123 case "*": | 123 function toRegExp(text) |
124 if (result.length > 0 && i < lastIndex && text[i
+ 1] != "*") | 124 { |
125 result += ".*"; | 125 var result = ""; |
126 break; | 126 var lastIndex = text.length - 1; |
127 case "^": | 127 |
128 if (i < lastIndex) | 128 for (var i = 0; i < text.length; i++) |
129 result += "."; | 129 { |
130 break; | 130 var c = text[i]; |
131 case "|": | 131 |
132 if (i == 0) | 132 switch (c) |
133 { | 133 { |
134 result += "^"; | 134 case "*": |
135 break; | 135 if (result.length > 0 && i < lastIndex && text[i + 1] != "*") |
136 } | 136 result += ".*"; |
137 if (i == lastIndex) | 137 break; |
138 { | 138 case "^": |
139 result += "$"; | 139 if (i < lastIndex) |
140 break; | 140 result += "."; |
141 } | 141 break; |
142 if (i == 1 && text[0] == "|") | 142 case "|": |
143 { | 143 if (i == 0) |
144 result += "https?://"; | 144 { |
145 break; | 145 result += "^"; |
146 } | 146 break; |
147 case ".": case "+": case "?": case "$": | 147 } |
148 case "{": case "}": case "(": case ")": | 148 if (i == lastIndex) |
149 case "[": case "]": case "\\": | 149 { |
150 result += "\\"; | 150 result += "$"; |
151 default: | 151 break; |
152 result += c; | 152 } |
153 } | 153 if (i == 1 && text[0] == "|") |
154 } | 154 { |
155 | 155 result += "https?://"; |
156 return result; | 156 break; |
157 } | 157 } |
158 | 158 case ".": case "+": case "?": case "$": |
159 function getRegExpSource(filter) { | 159 case "{": case "}": case "(": case ")": |
160 var source = toRegExp(filter.regexpSource.replace( | 160 case "[": case "]": case "\\": |
161 // Safari expects punycode, filter lists use unicode | 161 result += "\\"; |
162 /^(\|\||\|?https?:\/\/)([\w\-.*\u0080-\uFFFF]+)/i, | 162 default: |
163 function (match, prefix, domain) { | 163 result += c; |
164 return prefix + punycode.toASCII(domain); | 164 } |
165 } | 165 } |
166 )); | 166 |
167 | 167 return result; |
168 // Limit rules to to HTTP(S) URLs | 168 } |
169 if (!/^(\^|http)/i.test(source)) | 169 |
170 source = "^https?://.*" + source; | 170 function getRegExpSource(filter) |
171 | 171 { |
172 return source; | 172 var source = toRegExp(filter.regexpSource.replace( |
173 } | 173 // Safari expects punycode, filter lists use unicode |
174 | 174 /^(\|\||\|?https?:\/\/)([\w\-.*\u0080-\uFFFF]+)/i, |
175 function getResourceTypes(filter) { | 175 function (match, prefix, domain) |
176 var types = []; | 176 { |
177 | 177 return prefix + punycode.toASCII(domain); |
178 if (filter.contentType & typeMap.IMAGE) | 178 } |
179 types.push("image"); | 179 )); |
180 if (filter.contentType & typeMap.STYLESHEET) | 180 |
181 types.push("style-sheet"); | 181 // Limit rules to to HTTP(S) URLs |
182 if (filter.contentType & typeMap.SCRIPT) | 182 if (!/^(\^|http)/i.test(source)) |
183 types.push("script"); | 183 source = "^https?://.*" + source; |
184 if (filter.contentType & typeMap.FONT) | 184 |
185 types.push("font"); | 185 return source; |
186 if (filter.contentType & (typeMap.MEDIA | typeMap.OBJECT)) | 186 } |
187 types.push("media"); | 187 |
188 if (filter.contentType & typeMap.POPUP) | 188 function getResourceTypes(filter) |
189 types.push("popup"); | 189 { |
190 if (filter.contentType & (typeMap.XMLHTTPREQUEST | typeMap.OBJECT_SUBREQ
UEST | typeMap.PING | typeMap.OTHER)) | 190 var types = []; |
191 types.push("raw"); | 191 |
192 if (filter.contentType & typeMap.SUBDOCUMENT) | 192 if (filter.contentType & typeMap.IMAGE) |
193 types.push("document"); | 193 types.push("image"); |
194 | 194 if (filter.contentType & typeMap.STYLESHEET) |
195 return types; | 195 types.push("style-sheet"); |
196 } | 196 if (filter.contentType & typeMap.SCRIPT) |
197 | 197 types.push("script"); |
198 function addDomainPrefix(domains) { | 198 if (filter.contentType & typeMap.FONT) |
199 var result = []; | 199 types.push("font"); |
200 | 200 if (filter.contentType & (typeMap.MEDIA | typeMap.OBJECT)) |
201 for (var i = 0; i < domains.length; i++) { | 201 types.push("media"); |
202 var domain = domains[i]; | 202 if (filter.contentType & typeMap.POPUP) |
203 result.push(domain); | 203 types.push("popup"); |
204 | 204 if (filter.contentType & (typeMap.XMLHTTPREQUEST | typeMap.OBJECT_SUBREQUEST |
205 if (tldjs.getSubdomain(domain) == "") | 205 | typeMap.PING | typeMap.OTHER)) |
206 result.push("www." + domain); | 206 types.push("raw"); |
207 } | 207 if (filter.contentType & typeMap.SUBDOCUMENT) |
208 | 208 types.push("document"); |
209 return result; | 209 |
210 } | 210 return types; |
211 | 211 } |
212 function convertFilter(filter, action, withResourceTypes) { | 212 |
213 var trigger = {"url-filter": getRegExpSource(filter)}; | 213 function addDomainPrefix(domains) |
214 var included = []; | 214 { |
215 var excluded = []; | 215 var result = []; |
216 | 216 |
217 parseDomains(filter.domains, included, excluded); | 217 for (var i = 0; i < domains.length; i++) |
218 | 218 { |
219 if (withResourceTypes) | 219 var domain = domains[i]; |
220 trigger["resource-type"] = getResourceTypes(filter); | 220 result.push(domain); |
221 if (filter.thirdParty != null) | 221 |
222 trigger["load-type"] = [filter.thirdParty ? "third-party" : "fir
st-party"]; | 222 if (tldjs.getSubdomain(domain) == "") |
223 | 223 result.push("www." + domain); |
224 if (included.length > 0) | 224 } |
225 trigger["if-domain"] = addDomainPrefix(included); | 225 |
226 else if (excluded.length > 0) | 226 return result; |
227 trigger["unless-domain"] = addDomainPrefix(excluded); | 227 } |
228 | 228 |
229 return {trigger: trigger, action: {type: action}}; | 229 function convertFilter(filter, action, withResourceTypes) |
230 } | 230 { |
231 | 231 var trigger = {"url-filter": getRegExpSource(filter)}; |
232 function hasNonASCI(obj) { | 232 var included = []; |
233 if (typeof obj == "string") { | 233 var excluded = []; |
234 if (/[^\x00-\x7F]/.test(obj)) | 234 |
235 return true; | 235 parseDomains(filter.domains, included, excluded); |
236 } | 236 |
237 | 237 if (withResourceTypes) |
238 if (typeof obj == "object") { | 238 trigger["resource-type"] = getResourceTypes(filter); |
239 var i; | 239 if (filter.thirdParty != null) |
240 if (obj instanceof Array) | 240 trigger["load-type"] = [filter.thirdParty ? "third-party" : "first-party"]; |
241 for (i = 0; i < obj.length; i++) | 241 |
242 if (hasNonASCI(obj[i])) | 242 if (included.length > 0) |
243 return true; | 243 trigger["if-domain"] = addDomainPrefix(included); |
244 | 244 else if (excluded.length > 0) |
245 var names = Object.getOwnPropertyNames(obj); | 245 trigger["unless-domain"] = addDomainPrefix(excluded); |
246 for (i = 0; i < names.length; i++) | 246 |
247 if (hasNonASCI(obj[names[i]])) | 247 return {trigger: trigger, action: {type: action}}; |
248 return true; | 248 } |
249 } | 249 |
250 | 250 function hasNonASCI(obj) |
251 return false; | 251 { |
252 } | 252 if (typeof obj == "string") |
253 | 253 { |
254 function logRules() { | 254 if (/[^\x00-\x7F]/.test(obj)) |
255 var rules = []; | 255 return true; |
256 var i; | 256 } |
257 | 257 |
258 function addRule(rule) { | 258 if (typeof obj == "object") |
259 if (!hasNonASCI(rule)) | 259 { |
260 rules.push(rule); | 260 var i; |
261 } | 261 if (obj instanceof Array) |
262 | 262 for (i = 0; i < obj.length; i++) |
263 // HACK: We ignore element hiding filter for now to get the list of | 263 if (hasNonASCI(obj[i])) |
264 // rules down below 50K. This limit is enforced by iOS and Safari. | 264 return true; |
265 // To be undone with https://issues.adblockplus.org/ticket/3585 | 265 |
266 | 266 var names = Object.getOwnPropertyNames(obj); |
267 //for (i = 0; i < elemhideFilters.length; i++) | 267 for (i = 0; i < names.length; i++) |
268 // convertElemHideFilter(elemhideFilters[i]).forEach(addRule); | 268 if (hasNonASCI(obj[names[i]])) |
269 //for (i = 0; i < elemhideExceptions.length; i++) | 269 return true; |
270 // addRule(convertFilter(elemhideExceptions[i], "ignore-previous-ru
les", false)); | 270 } |
271 | 271 |
272 for (i = 0; i < requestFilters.length; i++) | 272 return false; |
273 addRule(convertFilter(requestFilters[i], "block", true)); | 273 } |
274 for (i = 0; i < requestExceptions.length; i++) | 274 |
275 addRule(convertFilter(requestExceptions[i], "ignore-previous-rul
es", true)); | 275 function logRules() |
276 | 276 { |
277 console.log(JSON.stringify(rules, null, "\t")); | 277 var rules = []; |
| 278 var i; |
| 279 |
| 280 // HACK: We ignore element hiding filter for now to get the list of |
| 281 // rules down below 50K. This limit is enforced by iOS and Safari. |
| 282 // To be undone with https://issues.adblockplus.org/ticket/3585 |
| 283 function addRule(rule) |
| 284 { |
| 285 if (!hasNonASCI(rule)) |
| 286 rules.push(rule); |
| 287 } |
| 288 |
| 289 //for (i = 0; i < elemhideFilters.length; i++) |
| 290 // convertElemHideFilter(elemhideFilters[i]).forEach(addRule); |
| 291 //for (i = 0; i < elemhideExceptions.length; i++) |
| 292 // addRule(convertFilter(elemhideExceptions[i], "ignore-previous-rules", fals
e)); |
| 293 |
| 294 for (i = 0; i < requestFilters.length; i++) |
| 295 addRule(convertFilter(requestFilters[i], "block", true)); |
| 296 for (i = 0; i < requestExceptions.length; i++) |
| 297 addRule(convertFilter(requestExceptions[i], "ignore-previous-rules", true)); |
| 298 |
| 299 console.log(JSON.stringify(rules, null, "\t")); |
278 } | 300 } |
279 | 301 |
280 var rl = readline.createInterface({input: process.stdin, terminal: false}); | 302 var rl = readline.createInterface({input: process.stdin, terminal: false}); |
281 rl.on("line", parseFilter); | 303 rl.on("line", parseFilter); |
282 rl.on("close", logRules); | 304 rl.on("close", logRules); |
OLD | NEW |