Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: lib/requestBlocker.js

Issue 29705719: Issue 6402 - Split filter hit / request logging out into own API (Closed)
Left Patch Set: Fixed typo and exposed pageId for logged requests Created Feb. 22, 2018, 3:55 p.m.
Right Patch Set: Addressed Manish's feedback Created May 9, 2018, 5:53 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « lib/popupBlocker.js ('k') | lib/whitelisting.js » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 /* 1 /*
2 * This file is part of Adblock Plus <https://adblockplus.org/>, 2 * This file is part of Adblock Plus <https://adblockplus.org/>,
3 * Copyright (C) 2006-present eyeo GmbH 3 * Copyright (C) 2006-present eyeo GmbH
4 * 4 *
5 * Adblock Plus is free software: you can redistribute it and/or modify 5 * Adblock Plus is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 3 as 6 * it under the terms of the GNU General Public License version 3 as
7 * published by the Free Software Foundation. 7 * published by the Free Software Foundation.
8 * 8 *
9 * Adblock Plus is distributed in the hope that it will be useful, 9 * Adblock Plus is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16 */ 16 */
17 17
18 /** @module requestBlocker */ 18 /** @module requestBlocker */
19 19
20 "use strict"; 20 "use strict";
21 21
22 const {Filter, RegExpFilter, BlockingFilter} = require("filterClasses"); 22 const {Filter, RegExpFilter, BlockingFilter} =
23 const {Subscription} = require("subscriptionClasses"); 23 require("../adblockpluscore/lib/filterClasses");
24 const {defaultMatcher} = require("matcher"); 24 const {Subscription} = require("../adblockpluscore/lib/subscriptionClasses");
25 const {FilterNotifier} = require("filterNotifier"); 25 const {defaultMatcher} = require("../adblockpluscore/lib/matcher");
26 const {Prefs} = require("prefs"); 26 const {FilterNotifier} = require("../adblockpluscore/lib/filterNotifier");
27 const {checkWhitelisted, getKey} = require("whitelisting"); 27 const {Prefs} = require("./prefs");
28 const {stringifyURL, extractHostFromFrame, isThirdParty} = require("url"); 28 const {checkWhitelisted, getKey} = require("./whitelisting");
29 const {port} = require("messaging"); 29 const {extractHostFromFrame, isThirdParty} = require("./url");
30 const {logRequest} = require("devLogger"); 30 const {port} = require("./messaging");
31 const {logRequest: hitLoggerLogRequest} = require("./hitLogger");
32
33 const extensionProtocol = new URL(browser.extension.getURL("")).protocol;
31 34
32 // Chrome can't distinguish between OBJECT_SUBREQUEST and OBJECT requests. 35 // Chrome can't distinguish between OBJECT_SUBREQUEST and OBJECT requests.
33 if (!browser.webRequest.ResourceType || 36 if (!browser.webRequest.ResourceType ||
34 !("OBJECT_SUBREQUEST" in browser.webRequest.ResourceType)) 37 !("OBJECT_SUBREQUEST" in browser.webRequest.ResourceType))
35 { 38 {
36 RegExpFilter.typeMap.OBJECT_SUBREQUEST = RegExpFilter.typeMap.OBJECT; 39 RegExpFilter.typeMap.OBJECT_SUBREQUEST = RegExpFilter.typeMap.OBJECT;
37 } 40 }
38 41
39 // Map of content types reported by the browser to the respecitve content types 42 // Map of content types reported by the browser to the respecitve content types
40 // used by Adblock Plus. Other content types are simply mapped to OTHER. 43 // used by Adblock Plus. Other content types are simply mapped to OTHER.
(...skipping 15 matching lines...) Expand all
56 exports.filterTypes = new Set(function*() 59 exports.filterTypes = new Set(function*()
57 { 60 {
58 // Microsoft Edge does not have webRequest.ResourceType or the devtools panel. 61 // Microsoft Edge does not have webRequest.ResourceType or the devtools panel.
59 // Since filterTypes is only used by devtools, we can just bail out here. 62 // Since filterTypes is only used by devtools, we can just bail out here.
60 if (!(browser.webRequest.ResourceType)) 63 if (!(browser.webRequest.ResourceType))
61 return; 64 return;
62 65
63 for (let type in browser.webRequest.ResourceType) 66 for (let type in browser.webRequest.ResourceType)
64 yield resourceTypes.get(browser.webRequest.ResourceType[type]) || "OTHER"; 67 yield resourceTypes.get(browser.webRequest.ResourceType[type]) || "OTHER";
65 68
66 // WEBSOCKET and WEBRTC get addressed through workarounds, even if the 69 // WEBRTC gets addressed through a workaround, even if the webRequest API is
67 // webRequest API is lacking support to block these kind of requests. 70 // lacking support to block this kind of a request.
68 yield "WEBSOCKET";
69 yield "WEBRTC"; 71 yield "WEBRTC";
70 72
71 // POPUP and ELEMHIDE filters aren't blocked on the request level but by other 73 // POPUP, CSP and ELEMHIDE filters aren't mapped to resource types.
72 // means. They don't have a corresponding value in webRequest.ResourceType.
73 yield "POPUP"; 74 yield "POPUP";
74 yield "ELEMHIDE"; 75 yield "ELEMHIDE";
76 yield "CSP";
75 }()); 77 }());
76 78
77 function onBeforeRequestAsync(page, url, type, docDomain, 79 function getDocumentInfo(page, frame, originUrl)
78 thirdParty, sitekey, 80 {
79 specificOnly, filter) 81 return [
82 extractHostFromFrame(frame, originUrl),
83 getKey(page, frame, originUrl),
84 !!checkWhitelisted(page, frame, originUrl,
85 RegExpFilter.typeMap.GENERICBLOCK)
86 ];
87 }
88
89 function matchRequest(url, type, docDomain, sitekey, specificOnly)
90 {
91 let thirdParty = isThirdParty(url, docDomain);
92 let filter = defaultMatcher.matchesAny(url.href, RegExpFilter.typeMap[type],
93 docDomain, thirdParty,
94 sitekey, specificOnly);
95 return [filter, thirdParty];
96 }
97
98 function getRelatedTabIds(details)
99 {
100 // This is the common case, the request is associated with a single tab.
101 // If tabId is -1, its not (e.g. the request was sent by
102 // a Service/Shared Worker) and we have to identify the related tabs.
103 if (details.tabId != -1)
104 return Promise.resolve([details.tabId]);
105
106 let url; // Firefox provides "originUrl" indicating the
107 if (details.originUrl) // URL of the tab that caused this request.
108 url = details.originUrl; // In case of Service/Shared Worker, this is the
109 // URL of the tab that caused the worker to spawn.
110
111 else if (details.initiator) // Chromium >=63 provides "intiator" which
112 url = details.initiator + "/*"; // is equivalent to "originUrl" on Firefox
113 // except that its not a full URL but just
114 // an origin (proto + host).
115 else
116 return Promise.resolve([]);
117
118 return browser.tabs.query({url}).then(tabs => tabs.map(tab => tab.id));
119 }
120
121 function logRequest(tabIds, request, filter)
80 { 122 {
81 if (filter) 123 if (filter)
82 FilterNotifier.emit("filter.hitCount", filter, 0, 0, page); 124 FilterNotifier.emit("filter.hitCount", filter, 0, 0, tabIds);
83 125
84 logRequest( 126 hitLoggerLogRequest(tabIds, request, filter);
85 page, url, type, docDomain, 127 }
86 thirdParty, sitekey, 128
87 specificOnly, filter 129 browser.webRequest.onBeforeRequest.addListener(details =>
130 {
131 // Never block top-level documents.
132 if (details.type == "main_frame")
133 return;
134
135 // Filter out requests from non web protocols. Ideally, we'd explicitly
136 // specify the protocols we are interested in (i.e. http://, https://,
137 // ws:// and wss://) with the url patterns, given below, when adding this
138 // listener. But unfortunately, Chrome <=57 doesn't support the WebSocket
139 // protocol and is causing an error if it is given.
140 let url = new URL(details.url);
141 if (url.protocol != "http:" && url.protocol != "https:" &&
142 url.protocol != "ws:" && url.protocol != "wss:")
143 return;
144
145 // Firefox provides us with the full origin URL, while Chromium (>=63)
146 // provides only the protocol + host of the (top-level) document which
147 // the request originates from through the "initiator" property.
148 let originUrl = details.originUrl ? new URL(details.originUrl) :
149 details.initiator ? new URL(details.initiator) : null;
150
151 // Ignore requests sent by extensions or by Firefox itself:
152 // * Firefox intercepts requests sent by any extensions, indicated with
153 // an "originURL" starting with "moz-extension:".
154 // * Chromium intercepts requests sent by this extension only, indicated
155 // on Chromium >=63 with an "initiator" starting with "chrome-extension:".
156 // * On Firefox, requests that don't relate to any document or extension are
157 // indicated with an "originUrl" starting with "chrome:".
158 if (originUrl && (originUrl.protocol == extensionProtocol ||
159 originUrl.protocol == "chrome:"))
160 return;
161
162 let page = new ext.Page({id: details.tabId});
163 let frame = ext.getFrame(
164 details.tabId,
165 // We are looking for the frame that contains the element which
166 // has triggered this request. For most requests (e.g. images) we
167 // can just use the request's frame ID, but for subdocument requests
168 // (e.g. iframes) we must instead use the request's parent frame ID.
169 details.type == "sub_frame" ? details.parentFrameId : details.frameId
88 ); 170 );
89 } 171
90 172 // On Chromium >= 63, if both the frame is unknown and we haven't get
91 ext.webRequest.onBeforeRequest.addListener((url, type, page, frame) => 173 // an "initator", this implies a request sent by the browser itself
92 { 174 // (on older versions of Chromium, due to the lack of "initator",
93 let docDomain = null; 175 // this can also indicate a request sent by a Shared/Service Worker).
94 let sitekey = null; 176 if (!frame && !originUrl)
95 let specificOnly = false; 177 return;
96 let thirdParty = false; 178
97 let urlString = stringifyURL(url); 179 if (checkWhitelisted(page, frame, originUrl))
98 180 return;
99 181
100 if (frame && page) 182 let type = resourceTypes.get(details.type) || "OTHER";
183 let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame,
184 originUrl);
185 let [filter, thirdParty] = matchRequest(url, type, docDomain,
186 sitekey, specificOnly);
187
188 getRelatedTabIds(details).then(tabIds =>
101 { 189 {
102 if (checkWhitelisted(page, frame)) 190 logRequest(
103 return true; 191 tabIds,
104 192 {url: details.url, type, docDomain, thirdParty, sitekey, specificOnly},
105 docDomain = extractHostFromFrame(frame); 193 filter
106 sitekey = getKey(page, frame); 194 );
107 thirdParty = isThirdParty(url, docDomain); 195 });
108 specificOnly = !!checkWhitelisted(page, frame, 196
109 RegExpFilter.typeMap.GENERICBLOCK); 197 if (filter instanceof BlockingFilter)
110 } 198 return {cancel: true};
111 199 }, {urls: ["<all_urls>"]}, ["blocking"]);
112 let mappedType = resourceTypes.get(type) || "OTHER";
113
114 let filter = defaultMatcher.matchesAny(
115 urlString, RegExpFilter.typeMap[mappedType],
116 docDomain, thirdParty, sitekey, specificOnly
117 );
118
119 setTimeout(onBeforeRequestAsync, 0, page, urlString,
120 mappedType, docDomain,
121 thirdParty, sitekey,
122 specificOnly, filter);
123
124 return !(filter instanceof BlockingFilter);
125 });
126 200
127 port.on("filters.collapse", (message, sender) => 201 port.on("filters.collapse", (message, sender) =>
128 { 202 {
129 if (checkWhitelisted(sender.page, sender.frame)) 203 let {page, frame} = sender;
204
205 if (checkWhitelisted(page, frame))
130 return false; 206 return false;
131 207
132 let typeMask = RegExpFilter.typeMap[message.mediatype];
133 let documentHost = extractHostFromFrame(sender.frame);
134 let sitekey = getKey(sender.page, sender.frame);
135 let blocked = false; 208 let blocked = false;
136 209 let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame);
137 let specificOnly = checkWhitelisted(
138 sender.page, sender.frame,
139 RegExpFilter.typeMap.GENERICBLOCK
140 );
141 210
142 for (let url of message.urls) 211 for (let url of message.urls)
143 { 212 {
144 let urlObj = new URL(url, message.baseURL); 213 let [filter] = matchRequest(new URL(url, message.baseURL),
145 let filter = defaultMatcher.matchesAny( 214 message.mediatype, docDomain,
146 stringifyURL(urlObj), 215 sitekey, specificOnly);
147 typeMask, documentHost,
148 isThirdParty(urlObj, documentHost),
149 sitekey, specificOnly
150 );
151 216
152 if (filter instanceof BlockingFilter) 217 if (filter instanceof BlockingFilter)
153 { 218 {
154 if (filter.collapse != null) 219 if (filter.collapse != null)
155 return filter.collapse; 220 return filter.collapse;
156 blocked = true; 221 blocked = true;
157 } 222 }
158 } 223 }
159 224
160 return blocked && Prefs.hidePlaceholders; 225 return blocked && Prefs.hidePlaceholders;
161 }); 226 });
162 227
228 port.on("request.blockedByRTCWrapper", (msg, sender) =>
229 {
230 let {page, frame} = sender;
231
232 if (checkWhitelisted(page, frame))
233 return false;
234
235 let {url} = msg;
236 let [docDomain, sitekey, specificOnly] = getDocumentInfo(page, frame);
237 let [filter, thirdParty] = matchRequest(new URL(url), "WEBRTC", docDomain,
238 sitekey, specificOnly);
239 logRequest(
240 [sender.page.id],
241 {url, type: "WEBRTC", docDomain, thirdParty, sitekey, specificOnly},
242 filter
243 );
244
245 return filter instanceof BlockingFilter;
246 });
247
163 let ignoreFilterNotifications = false; 248 let ignoreFilterNotifications = false;
249 let handlerBehaviorChangedQuota =
250 browser.webRequest.MAX_HANDLER_BEHAVIOR_CHANGED_CALLS_PER_10_MINUTES;
251
252 function propagateHandlerBehaviorChange()
253 {
254 // Make sure to not call handlerBehaviorChanged() more often than allowed
255 // by browser.webRequest.MAX_HANDLER_BEHAVIOR_CHANGED_CALLS_PER_10_MINUTES.
256 // Otherwise Chrome notifies the user that this extension is causing issues.
257 if (handlerBehaviorChangedQuota > 0)
258 {
259 browser.webNavigation.onBeforeNavigate.removeListener(
260 propagateHandlerBehaviorChange
261 );
262 browser.webRequest.handlerBehaviorChanged();
263 handlerBehaviorChangedQuota--;
264 setTimeout(() => { handlerBehaviorChangedQuota++; }, 600000);
265 }
266 }
164 267
165 function onFilterChange(arg, isDisabledAction) 268 function onFilterChange(arg, isDisabledAction)
166 { 269 {
167 // Avoid triggering filters.behaviorChanged multiple times 270 // Avoid triggering filters.behaviorChanged multiple times
168 // when multiple filter hanges happen at the same time. 271 // when multiple filter hanges happen at the same time.
169 if (ignoreFilterNotifications) 272 if (ignoreFilterNotifications)
170 return; 273 return;
171 274
172 // Ignore disabled subscriptions and filters, unless they just got 275 // Ignore disabled subscriptions and filters, unless they just got
173 // disabled, otherwise they have no effect on the handler behavior. 276 // disabled, otherwise they have no effect on the handler behavior.
174 if (arg && arg.disabled && !isDisabledAction) 277 if (arg && arg.disabled && !isDisabledAction)
175 return; 278 return;
176 279
177 // Ignore empty subscriptions. This includes subscriptions 280 // Ignore empty subscriptions. This includes subscriptions
178 // that have just been added, but not downloaded yet. 281 // that have just been added, but not downloaded yet.
179 if (arg instanceof Subscription && arg.filters.length == 0) 282 if (arg instanceof Subscription && arg.filters.length == 0)
180 return; 283 return;
181 284
182 // Ignore all types of filters but request filters, 285 // Ignore all types of filters but request filters,
183 // only these have an effect on the handler behavior. 286 // only these have an effect on the handler behavior.
184 if (arg instanceof Filter && !(arg instanceof RegExpFilter)) 287 if (arg instanceof Filter && !(arg instanceof RegExpFilter))
185 return; 288 return;
186 289
187 ignoreFilterNotifications = true; 290 ignoreFilterNotifications = true;
188 setTimeout(() => 291 setTimeout(() =>
189 { 292 {
293 // Defer handlerBehaviorChanged() until navigation occurs.
294 // There wouldn't be any visible effect when calling it earlier,
295 // but it's an expensive operation and that way we avoid to call
296 // it multiple times, if multiple filters are added/removed.
297 if (!browser.webNavigation.onBeforeNavigate
298 .hasListener(propagateHandlerBehaviorChange))
299 browser.webNavigation.onBeforeNavigate
300 .addListener(propagateHandlerBehaviorChange);
301
190 ignoreFilterNotifications = false; 302 ignoreFilterNotifications = false;
191 ext.webRequest.handlerBehaviorChanged();
192 FilterNotifier.emit("filter.behaviorChanged"); 303 FilterNotifier.emit("filter.behaviorChanged");
193 }); 304 });
194 } 305 }
195 306
196 FilterNotifier.on("subscription.added", onFilterChange); 307 FilterNotifier.on("subscription.added", onFilterChange);
197 FilterNotifier.on("subscription.removed", onFilterChange); 308 FilterNotifier.on("subscription.removed", onFilterChange);
198 FilterNotifier.on("subscription.updated", onFilterChange); 309 FilterNotifier.on("subscription.updated", onFilterChange);
199 FilterNotifier.on("subscription.disabled", arg => onFilterChange(arg, true)); 310 FilterNotifier.on("subscription.disabled", arg => onFilterChange(arg, true));
200 FilterNotifier.on("filter.added", onFilterChange); 311 FilterNotifier.on("filter.added", onFilterChange);
201 FilterNotifier.on("filter.removed", onFilterChange); 312 FilterNotifier.on("filter.removed", onFilterChange);
202 FilterNotifier.on("filter.disabled", arg => onFilterChange(arg, true)); 313 FilterNotifier.on("filter.disabled", arg => onFilterChange(arg, true));
203 FilterNotifier.on("load", onFilterChange); 314 FilterNotifier.on("load", onFilterChange);
204
205 port.on("request.blockedByWrapper", (msg, sender) =>
206 {
207 // Chrome 58 onwards directly supports WebSocket blocking, so we can ignore
208 // messages from the wrapper here (see https://crbug.com/129353). Hopefully
209 // WebRTC will be supported soon too (see https://crbug.com/707683).
210 // Edge supports neither webRequest.ResourceType nor WebSocket blocking yet:
211 // https://developer.microsoft.com/en-us/microsoft-edge/platform/issues/102973 76/
212 if (browser.webRequest.ResourceType &&
213 (msg.requestType.toUpperCase() in browser.webRequest.ResourceType))
214 {
215 return false;
216 }
217
218 return ext.webRequest.onBeforeRequest._dispatch(
219 new URL(msg.url),
220 msg.requestType,
221 sender.page,
222 sender.frame
223 ).includes(false);
224 });
LEFTRIGHT

Powered by Google App Engine
This is Rietveld