Move URL from WebCore to WTF
[WebKit-https.git] / Source / WebCore / loader / ResourceLoadObserver.cpp
1 /*
2  * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadObserver.h"
28
29 #include "DeprecatedGlobalSettings.h"
30 #include "Document.h"
31 #include "Frame.h"
32 #include "FrameLoader.h"
33 #include "HTMLFrameOwnerElement.h"
34 #include "Logging.h"
35 #include "Page.h"
36 #include "ResourceLoadStatistics.h"
37 #include "ResourceRequest.h"
38 #include "ResourceResponse.h"
39 #include "RuntimeEnabledFeatures.h"
40 #include "ScriptExecutionContext.h"
41 #include "SecurityOrigin.h"
42 #include "Settings.h"
43 #include <wtf/URL.h>
44
45 namespace WebCore {
46
47 template<typename T> static inline String primaryDomain(const T& value)
48 {
49     return ResourceLoadStatistics::primaryDomain(value);
50 }
51
52 static const Seconds minimumNotificationInterval { 5_s };
53
54 ResourceLoadObserver& ResourceLoadObserver::shared()
55 {
56     static NeverDestroyed<ResourceLoadObserver> resourceLoadObserver;
57     return resourceLoadObserver;
58 }
59
60 void ResourceLoadObserver::setNotificationCallback(WTF::Function<void (Vector<ResourceLoadStatistics>&&)>&& notificationCallback)
61 {
62     ASSERT(!m_notificationCallback);
63     m_notificationCallback = WTFMove(notificationCallback);
64 }
65
66 void ResourceLoadObserver::setRequestStorageAccessUnderOpenerCallback(WTF::Function<void(const String& domainInNeedOfStorageAccess, uint64_t openerPageID, const String& openerDomain)>&& callback)
67 {
68     ASSERT(!m_requestStorageAccessUnderOpenerCallback);
69     m_requestStorageAccessUnderOpenerCallback = WTFMove(callback);
70 }
71
72 ResourceLoadObserver::ResourceLoadObserver()
73     : m_notificationTimer(*this, &ResourceLoadObserver::notifyObserver)
74 {
75 }
76
77 static inline bool is3xxRedirect(const ResourceResponse& response)
78 {
79     return response.httpStatusCode() >= 300 && response.httpStatusCode() <= 399;
80 }
81
82 bool ResourceLoadObserver::shouldLog(bool usesEphemeralSession) const
83 {
84     return DeprecatedGlobalSettings::resourceLoadStatisticsEnabled() && !usesEphemeralSession && m_notificationCallback;
85 }
86
87 void ResourceLoadObserver::logSubresourceLoading(const Frame* frame, const ResourceRequest& newRequest, const ResourceResponse& redirectResponse)
88 {
89     ASSERT(frame->page());
90
91     if (!frame)
92         return;
93
94     auto* page = frame->page();
95     if (!page || !shouldLog(page->usesEphemeralSession()))
96         return;
97
98     bool isRedirect = is3xxRedirect(redirectResponse);
99     const URL& sourceURL = redirectResponse.url();
100     const URL& targetURL = newRequest.url();
101     const URL& mainFrameURL = frame ? frame->mainFrame().document()->url() : URL();
102     
103     auto targetHost = targetURL.host();
104     auto mainFrameHost = mainFrameURL.host();
105
106     if (targetHost.isEmpty() || mainFrameHost.isEmpty() || targetHost == mainFrameHost || (isRedirect && targetHost == sourceURL.host()))
107         return;
108
109     auto targetPrimaryDomain = primaryDomain(targetURL);
110     auto mainFramePrimaryDomain = primaryDomain(mainFrameURL);
111     auto sourcePrimaryDomain = primaryDomain(sourceURL);
112
113     if (targetPrimaryDomain == mainFramePrimaryDomain || (isRedirect && targetPrimaryDomain == sourcePrimaryDomain))
114         return;
115
116     bool shouldCallNotificationCallback = false;
117     {
118         auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
119         targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
120         if (targetStatistics.subresourceUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
121             shouldCallNotificationCallback = true;
122     }
123
124     if (isRedirect) {
125         auto& redirectingOriginStatistics = ensureResourceStatisticsForPrimaryDomain(sourcePrimaryDomain);
126         bool isNewRedirectToEntry = redirectingOriginStatistics.subresourceUniqueRedirectsTo.add(targetPrimaryDomain).isNewEntry;
127         auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
128         bool isNewRedirectFromEntry = targetStatistics.subresourceUniqueRedirectsFrom.add(sourcePrimaryDomain).isNewEntry;
129
130         if (isNewRedirectToEntry || isNewRedirectFromEntry)
131             shouldCallNotificationCallback = true;
132     }
133
134     if (shouldCallNotificationCallback)
135         scheduleNotificationIfNeeded();
136 }
137
138 void ResourceLoadObserver::logWebSocketLoading(const URL& targetURL, const URL& mainFrameURL, bool usesEphemeralSession)
139 {
140     if (!shouldLog(usesEphemeralSession))
141         return;
142
143     auto targetHost = targetURL.host();
144     auto mainFrameHost = mainFrameURL.host();
145     
146     if (targetHost.isEmpty() || mainFrameHost.isEmpty() || targetHost == mainFrameHost)
147         return;
148     
149     auto targetPrimaryDomain = primaryDomain(targetURL);
150     auto mainFramePrimaryDomain = primaryDomain(mainFrameURL);
151
152     if (targetPrimaryDomain == mainFramePrimaryDomain)
153         return;
154
155     auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
156     targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
157     if (targetStatistics.subresourceUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
158         scheduleNotificationIfNeeded();
159 }
160
161 void ResourceLoadObserver::logUserInteractionWithReducedTimeResolution(const Document& document)
162 {
163     if (!shouldLog(document.sessionID().isEphemeral()))
164         return;
165
166     auto& url = document.url();
167     if (url.protocolIsAbout() || url.isEmpty())
168         return;
169
170     auto domain = primaryDomain(url);
171     auto newTime = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
172     auto lastReportedUserInteraction = m_lastReportedUserInteractionMap.get(domain);
173     if (newTime == lastReportedUserInteraction)
174         return;
175
176     m_lastReportedUserInteractionMap.set(domain, newTime);
177
178     auto& statistics = ensureResourceStatisticsForPrimaryDomain(domain);
179     statistics.hadUserInteraction = true;
180     statistics.lastSeen = newTime;
181     statistics.mostRecentUserInteractionTime = newTime;
182
183 #if ENABLE(RESOURCE_LOAD_STATISTICS)
184     if (auto* opener = document.frame()->loader().opener()) {
185         if (auto* openerDocument = opener->document()) {
186             if (auto* openerFrame = openerDocument->frame()) {
187                 if (auto openerPageID = openerFrame->loader().client().pageID()) {
188                     requestStorageAccessUnderOpener(domain, openerPageID.value(), *openerDocument);
189                 }
190             }
191         }
192     }
193 #endif
194
195     m_notificationTimer.stop();
196     notifyObserver();
197
198 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
199     if (shouldLogUserInteraction()) {
200         auto counter = ++m_loggingCounter;
201 #define LOCAL_LOG(str, ...) \
202         RELEASE_LOG(ResourceLoadStatistics, "ResourceLoadObserver::logUserInteraction: counter = %" PRIu64 ": " str, counter, ##__VA_ARGS__)
203
204         auto escapeForJSON = [](String s) {
205             s.replace('\\', "\\\\").replace('"', "\\\"");
206             return s;
207         };
208         auto escapedURL = escapeForJSON(url.string());
209         auto escapedDomain = escapeForJSON(domain);
210
211         LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
212         LOCAL_LOG(R"(  "domain" : "%{public}s",)", escapedDomain.utf8().data());
213         LOCAL_LOG(R"(  "until" : %f })", newTime.secondsSinceEpoch().seconds());
214
215 #undef LOCAL_LOG
216     }
217 #endif
218 }
219
220 #if ENABLE(RESOURCE_LOAD_STATISTICS)
221 void ResourceLoadObserver::requestStorageAccessUnderOpener(const String& domainInNeedOfStorageAccess, uint64_t openerPageID, Document& openerDocument)
222 {
223     auto openerUrl = openerDocument.url();
224     auto openerPrimaryDomain = primaryDomain(openerUrl);
225     if (domainInNeedOfStorageAccess != openerPrimaryDomain
226         && !openerDocument.hasRequestedPageSpecificStorageAccessWithUserInteraction(domainInNeedOfStorageAccess)
227         && !equalIgnoringASCIICase(openerUrl.string(), WTF::blankURL())) {
228         m_requestStorageAccessUnderOpenerCallback(domainInNeedOfStorageAccess, openerPageID, openerPrimaryDomain);
229         // Remember user interaction-based requests since they don't need to be repeated.
230         openerDocument.setHasRequestedPageSpecificStorageAccessWithUserInteraction(domainInNeedOfStorageAccess);
231     }
232 }
233 #endif
234
235 void ResourceLoadObserver::logFontLoad(const Document& document, const String& familyName, bool loadStatus)
236 {
237     if (!shouldLog(document.sessionID().isEphemeral()))
238         return;
239     auto registrableDomain = primaryDomain(document.url());
240     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
241     bool shouldCallNotificationCallback = false;
242     if (!loadStatus) {
243         if (statistics.fontsFailedToLoad.add(familyName).isNewEntry)
244             shouldCallNotificationCallback = true;
245     } else {
246         if (statistics.fontsSuccessfullyLoaded.add(familyName).isNewEntry)
247             shouldCallNotificationCallback = true;
248     }
249     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
250     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
251         shouldCallNotificationCallback = true;
252     if (shouldCallNotificationCallback)
253         scheduleNotificationIfNeeded();
254 }
255     
256 void ResourceLoadObserver::logCanvasRead(const Document& document)
257 {
258     if (!shouldLog(document.sessionID().isEphemeral()))
259         return;
260     auto registrableDomain = primaryDomain(document.url());
261     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
262     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
263     statistics.canvasActivityRecord.wasDataRead = true;
264     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
265         scheduleNotificationIfNeeded();
266 }
267
268 void ResourceLoadObserver::logCanvasWriteOrMeasure(const Document& document, const String& textWritten)
269 {
270     if (!shouldLog(document.sessionID().isEphemeral()))
271         return;
272     auto registrableDomain = primaryDomain(document.url());
273     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
274     bool shouldCallNotificationCallback = false;
275     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
276     if (statistics.canvasActivityRecord.recordWrittenOrMeasuredText(textWritten))
277         shouldCallNotificationCallback = true;
278     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
279         shouldCallNotificationCallback = true;
280     if (shouldCallNotificationCallback)
281         scheduleNotificationIfNeeded();
282 }
283     
284 void ResourceLoadObserver::logNavigatorAPIAccessed(const Document& document, const ResourceLoadStatistics::NavigatorAPI functionName)
285 {
286     if (!shouldLog(document.sessionID().isEphemeral()))
287         return;
288     auto registrableDomain = primaryDomain(document.url());
289     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
290     bool shouldCallNotificationCallback = false;
291     if (!statistics.navigatorFunctionsAccessed.contains(functionName)) {
292         statistics.navigatorFunctionsAccessed.add(functionName);
293         shouldCallNotificationCallback = true;
294     }
295     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
296     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
297         shouldCallNotificationCallback = true;
298     if (shouldCallNotificationCallback)
299         scheduleNotificationIfNeeded();
300 }
301     
302 void ResourceLoadObserver::logScreenAPIAccessed(const Document& document, const ResourceLoadStatistics::ScreenAPI functionName)
303 {
304     if (!shouldLog(document.sessionID().isEphemeral()))
305         return;
306     auto registrableDomain = primaryDomain(document.url());
307     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
308     bool shouldCallNotificationCallback = false;
309     if (!statistics.screenFunctionsAccessed.contains(functionName)) {
310         statistics.screenFunctionsAccessed.add(functionName);
311         shouldCallNotificationCallback = true;
312     }
313     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
314     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
315         shouldCallNotificationCallback = true;
316     if (shouldCallNotificationCallback)
317         scheduleNotificationIfNeeded();
318 }
319     
320 ResourceLoadStatistics& ResourceLoadObserver::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
321 {
322     auto addResult = m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
323         return ResourceLoadStatistics(primaryDomain);
324     });
325     return addResult.iterator->value;
326 }
327
328 void ResourceLoadObserver::scheduleNotificationIfNeeded()
329 {
330     ASSERT(m_notificationCallback);
331     if (m_resourceStatisticsMap.isEmpty()) {
332         m_notificationTimer.stop();
333         return;
334     }
335
336     if (!m_notificationTimer.isActive())
337         m_notificationTimer.startOneShot(minimumNotificationInterval);
338 }
339
340 void ResourceLoadObserver::notifyObserver()
341 {
342     ASSERT(m_notificationCallback);
343     m_notificationTimer.stop();
344     m_notificationCallback(takeStatistics());
345 }
346
347 String ResourceLoadObserver::statisticsForOrigin(const String& origin)
348 {
349     auto iter = m_resourceStatisticsMap.find(origin);
350     if (iter == m_resourceStatisticsMap.end())
351         return emptyString();
352
353     return "Statistics for " + origin + ":\n" + iter->value.toString();
354 }
355
356 Vector<ResourceLoadStatistics> ResourceLoadObserver::takeStatistics()
357 {
358     Vector<ResourceLoadStatistics> statistics;
359     statistics.reserveInitialCapacity(m_resourceStatisticsMap.size());
360     for (auto& statistic : m_resourceStatisticsMap.values())
361         statistics.uncheckedAppend(WTFMove(statistic));
362
363     m_resourceStatisticsMap.clear();
364
365     return statistics;
366 }
367
368 void ResourceLoadObserver::clearState()
369 {
370     m_notificationTimer.stop();
371     m_resourceStatisticsMap.clear();
372     m_lastReportedUserInteractionMap.clear();
373 }
374
375 URL ResourceLoadObserver::nonNullOwnerURL(const Document& document) const
376 {
377     auto url = document.url();
378     auto* frame = document.frame();
379     auto host = document.url().host();
380
381     while ((host.isNull() || host.isEmpty()) && frame && !frame->isMainFrame()) {
382         auto* ownerElement = frame->ownerElement();
383
384         ASSERT(ownerElement != nullptr);
385         
386         auto& doc = ownerElement->document();
387         frame = doc.frame();
388         url = doc.url();
389         host = url.host();
390     }
391
392     return url;
393 }
394
395 } // namespace WebCore