Change from HAVE(CFNETWORK_STORAGE_PARTITIONING) to ENABLE(RESOURCE_LOAD_STATISTICS)
[WebKit-https.git] / Source / WebCore / loader / ResourceLoadObserver.cpp
1 /*
2  * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadObserver.h"
28
29 #include "DeprecatedGlobalSettings.h"
30 #include "Document.h"
31 #include "Frame.h"
32 #include "FrameLoader.h"
33 #include "HTMLFrameOwnerElement.h"
34 #include "Logging.h"
35 #include "Page.h"
36 #include "ResourceLoadStatistics.h"
37 #include "ResourceRequest.h"
38 #include "ResourceResponse.h"
39 #include "RuntimeEnabledFeatures.h"
40 #include "ScriptExecutionContext.h"
41 #include "SecurityOrigin.h"
42 #include "Settings.h"
43 #include "URL.h"
44
45 namespace WebCore {
46
47 template<typename T> static inline String primaryDomain(const T& value)
48 {
49     return ResourceLoadStatistics::primaryDomain(value);
50 }
51
52 static const Seconds minimumNotificationInterval { 5_s };
53
54 ResourceLoadObserver& ResourceLoadObserver::shared()
55 {
56     static NeverDestroyed<ResourceLoadObserver> resourceLoadObserver;
57     return resourceLoadObserver;
58 }
59
60 void ResourceLoadObserver::setNotificationCallback(WTF::Function<void (Vector<ResourceLoadStatistics>&&)>&& notificationCallback)
61 {
62     ASSERT(!m_notificationCallback);
63     m_notificationCallback = WTFMove(notificationCallback);
64 }
65
66 void ResourceLoadObserver::setRequestStorageAccessUnderOpenerCallback(WTF::Function<void(const String& domainInNeedOfStorageAccess, uint64_t openerPageID, const String& openerDomain, bool isTriggeredByUserGesture)>&& callback)
67 {
68     ASSERT(!m_requestStorageAccessUnderOpenerCallback);
69     m_requestStorageAccessUnderOpenerCallback = WTFMove(callback);
70 }
71
72 ResourceLoadObserver::ResourceLoadObserver()
73     : m_notificationTimer(*this, &ResourceLoadObserver::notifyObserver)
74 {
75 }
76
77 static inline bool is3xxRedirect(const ResourceResponse& response)
78 {
79     return response.httpStatusCode() >= 300 && response.httpStatusCode() <= 399;
80 }
81
82 bool ResourceLoadObserver::shouldLog(bool usesEphemeralSession) const
83 {
84     return DeprecatedGlobalSettings::resourceLoadStatisticsEnabled() && !usesEphemeralSession && m_notificationCallback;
85 }
86
87 void ResourceLoadObserver::logSubresourceLoading(const Frame* frame, const ResourceRequest& newRequest, const ResourceResponse& redirectResponse)
88 {
89     ASSERT(frame->page());
90
91     auto* page = frame->page();
92     if (!shouldLog(page->usesEphemeralSession()))
93         return;
94
95     bool isRedirect = is3xxRedirect(redirectResponse);
96     const URL& sourceURL = redirectResponse.url();
97     const URL& targetURL = newRequest.url();
98     const URL& mainFrameURL = frame ? frame->mainFrame().document()->url() : URL();
99     
100     auto targetHost = targetURL.host();
101     auto mainFrameHost = mainFrameURL.host();
102
103     if (targetHost.isEmpty() || mainFrameHost.isEmpty() || targetHost == mainFrameHost || (isRedirect && targetHost == sourceURL.host()))
104         return;
105
106     auto targetPrimaryDomain = primaryDomain(targetURL);
107     auto mainFramePrimaryDomain = primaryDomain(mainFrameURL);
108     auto sourcePrimaryDomain = primaryDomain(sourceURL);
109
110     if (targetPrimaryDomain == mainFramePrimaryDomain || (isRedirect && targetPrimaryDomain == sourcePrimaryDomain))
111         return;
112
113     bool shouldCallNotificationCallback = false;
114     {
115         auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
116         targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
117         if (targetStatistics.subresourceUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
118             shouldCallNotificationCallback = true;
119     }
120
121     if (isRedirect) {
122         auto& redirectingOriginStatistics = ensureResourceStatisticsForPrimaryDomain(sourcePrimaryDomain);
123         bool isNewRedirectToEntry = redirectingOriginStatistics.subresourceUniqueRedirectsTo.add(targetPrimaryDomain).isNewEntry;
124         auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
125         bool isNewRedirectFromEntry = targetStatistics.subresourceUniqueRedirectsFrom.add(sourcePrimaryDomain).isNewEntry;
126
127         if (isNewRedirectToEntry || isNewRedirectFromEntry)
128             shouldCallNotificationCallback = true;
129     }
130
131     if (shouldCallNotificationCallback)
132         scheduleNotificationIfNeeded();
133 }
134
135 void ResourceLoadObserver::logWebSocketLoading(const URL& targetURL, const URL& mainFrameURL, bool usesEphemeralSession)
136 {
137     if (!shouldLog(usesEphemeralSession))
138         return;
139
140     auto targetHost = targetURL.host();
141     auto mainFrameHost = mainFrameURL.host();
142     
143     if (targetHost.isEmpty() || mainFrameHost.isEmpty() || targetHost == mainFrameHost)
144         return;
145     
146     auto targetPrimaryDomain = primaryDomain(targetURL);
147     auto mainFramePrimaryDomain = primaryDomain(mainFrameURL);
148
149     if (targetPrimaryDomain == mainFramePrimaryDomain)
150         return;
151
152     auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
153     targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
154     if (targetStatistics.subresourceUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
155         scheduleNotificationIfNeeded();
156 }
157
158 void ResourceLoadObserver::logUserInteractionWithReducedTimeResolution(const Document& document)
159 {
160     ASSERT(document.page());
161
162     if (!shouldLog(document.page()->usesEphemeralSession()))
163         return;
164
165     auto& url = document.url();
166     if (url.isBlankURL() || url.isEmpty())
167         return;
168
169     auto domain = primaryDomain(url);
170     auto newTime = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
171     auto lastReportedUserInteraction = m_lastReportedUserInteractionMap.get(domain);
172     if (newTime == lastReportedUserInteraction)
173         return;
174
175     m_lastReportedUserInteractionMap.set(domain, newTime);
176
177     auto& statistics = ensureResourceStatisticsForPrimaryDomain(domain);
178     statistics.hadUserInteraction = true;
179     statistics.lastSeen = newTime;
180     statistics.mostRecentUserInteractionTime = newTime;
181
182 #if ENABLE(RESOURCE_LOAD_STATISTICS)
183     if (auto* opener = document.frame()->loader().opener()) {
184         if (auto* openerDocument = opener->document()) {
185             if (auto* openerFrame = openerDocument->frame()) {
186                 if (auto openerPageID = openerFrame->loader().client().pageID()) {
187                     requestStorageAccessUnderOpener(domain, openerPageID.value(), *openerDocument, true);
188                 }
189             }
190         }
191     }
192 #endif
193
194     m_notificationTimer.stop();
195     notifyObserver();
196
197 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
198     if (shouldLogUserInteraction()) {
199         auto counter = ++m_loggingCounter;
200 #define LOCAL_LOG(str, ...) \
201         RELEASE_LOG(ResourceLoadStatistics, "ResourceLoadObserver::logUserInteraction: counter = %" PRIu64 ": " str, counter, ##__VA_ARGS__)
202
203         auto escapeForJSON = [](String s) {
204             s.replace('\\', "\\\\").replace('"', "\\\"");
205             return s;
206         };
207         auto escapedURL = escapeForJSON(url.string());
208         auto escapedDomain = escapeForJSON(domain);
209
210         LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
211         LOCAL_LOG(R"(  "domain" : "%{public}s",)", escapedDomain.utf8().data());
212         LOCAL_LOG(R"(  "until" : %f })", newTime.secondsSinceEpoch().seconds());
213
214 #undef LOCAL_LOG
215     }
216 #endif
217 }
218
219 void ResourceLoadObserver::logWindowCreation(const URL& popupUrl, uint64_t openerPageID, Document& openerDocument)
220 {
221 #if ENABLE(RESOURCE_LOAD_STATISTICS)
222     requestStorageAccessUnderOpener(primaryDomain(popupUrl), openerPageID, openerDocument, false);
223 #else
224     UNUSED_PARAM(popupUrl);
225     UNUSED_PARAM(openerPageID);
226     UNUSED_PARAM(openerDocument);
227 #endif
228 }
229
230 #if ENABLE(RESOURCE_LOAD_STATISTICS)
231 void ResourceLoadObserver::requestStorageAccessUnderOpener(const String& domainInNeedOfStorageAccess, uint64_t openerPageID, Document& openerDocument, bool isTriggeredByUserGesture)
232 {
233     auto openerUrl = openerDocument.url();
234     auto openerPrimaryDomain = primaryDomain(openerUrl);
235     if (domainInNeedOfStorageAccess != openerPrimaryDomain
236         && !openerDocument.hasRequestedPageSpecificStorageAccessWithUserInteraction(domainInNeedOfStorageAccess)
237         && !equalIgnoringASCIICase(openerUrl.string(), blankURL())) {
238         m_requestStorageAccessUnderOpenerCallback(domainInNeedOfStorageAccess, openerPageID, openerPrimaryDomain, isTriggeredByUserGesture);
239         // Remember user interaction-based requests since they don't need to be repeated.
240         if (isTriggeredByUserGesture)
241             openerDocument.setHasRequestedPageSpecificStorageAccessWithUserInteraction(domainInNeedOfStorageAccess);
242     }
243 }
244 #endif
245
246 void ResourceLoadObserver::logFontLoad(const Document& document, const String& familyName, bool loadStatus)
247 {
248     if (!shouldLog(document.sessionID().isEphemeral()))
249         return;
250     auto registrableDomain = primaryDomain(document.url());
251     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
252     bool shouldCallNotificationCallback = false;
253     if (!loadStatus) {
254         if (statistics.fontsFailedToLoad.add(familyName).isNewEntry)
255             shouldCallNotificationCallback = true;
256     } else {
257         if (statistics.fontsSuccessfullyLoaded.add(familyName).isNewEntry)
258             shouldCallNotificationCallback = true;
259     }
260     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
261     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
262         shouldCallNotificationCallback = true;
263     if (shouldCallNotificationCallback)
264         scheduleNotificationIfNeeded();
265 }
266     
267 void ResourceLoadObserver::logCanvasRead(const Document& document)
268 {
269     if (!shouldLog(document.sessionID().isEphemeral()))
270         return;
271     auto registrableDomain = primaryDomain(document.url());
272     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
273     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
274     statistics.canvasActivityRecord.wasDataRead = true;
275     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
276         scheduleNotificationIfNeeded();
277 }
278
279 void ResourceLoadObserver::logCanvasWriteOrMeasure(const Document& document, const String& textWritten)
280 {
281     if (!shouldLog(document.sessionID().isEphemeral()))
282         return;
283     auto registrableDomain = primaryDomain(document.url());
284     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
285     bool shouldCallNotificationCallback = false;
286     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
287     if (statistics.canvasActivityRecord.recordWrittenOrMeasuredText(textWritten))
288         shouldCallNotificationCallback = true;
289     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
290         shouldCallNotificationCallback = true;
291     if (shouldCallNotificationCallback)
292         scheduleNotificationIfNeeded();
293 }
294     
295 void ResourceLoadObserver::logNavigatorAPIAccessed(const Document& document, const ResourceLoadStatistics::NavigatorAPI functionName)
296 {
297     if (!shouldLog(document.sessionID().isEphemeral()))
298         return;
299     auto registrableDomain = primaryDomain(document.url());
300     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
301     bool shouldCallNotificationCallback = false;
302     if (!statistics.navigatorFunctionsAccessed.contains(functionName)) {
303         statistics.navigatorFunctionsAccessed.add(functionName);
304         shouldCallNotificationCallback = true;
305     }
306     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
307     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
308         shouldCallNotificationCallback = true;
309     if (shouldCallNotificationCallback)
310         scheduleNotificationIfNeeded();
311 }
312     
313 void ResourceLoadObserver::logScreenAPIAccessed(const Document& document, const ResourceLoadStatistics::ScreenAPI functionName)
314 {
315     if (!shouldLog(document.sessionID().isEphemeral()))
316         return;
317     auto registrableDomain = primaryDomain(document.url());
318     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
319     bool shouldCallNotificationCallback = false;
320     if (!statistics.screenFunctionsAccessed.contains(functionName)) {
321         statistics.screenFunctionsAccessed.add(functionName);
322         shouldCallNotificationCallback = true;
323     }
324     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
325     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
326         shouldCallNotificationCallback = true;
327     if (shouldCallNotificationCallback)
328         scheduleNotificationIfNeeded();
329 }
330     
331 ResourceLoadStatistics& ResourceLoadObserver::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
332 {
333     auto addResult = m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
334         return ResourceLoadStatistics(primaryDomain);
335     });
336     return addResult.iterator->value;
337 }
338
339 void ResourceLoadObserver::scheduleNotificationIfNeeded()
340 {
341     ASSERT(m_notificationCallback);
342     if (m_resourceStatisticsMap.isEmpty()) {
343         m_notificationTimer.stop();
344         return;
345     }
346
347     if (!m_notificationTimer.isActive())
348         m_notificationTimer.startOneShot(minimumNotificationInterval);
349 }
350
351 void ResourceLoadObserver::notifyObserver()
352 {
353     ASSERT(m_notificationCallback);
354     m_notificationTimer.stop();
355     m_notificationCallback(takeStatistics());
356 }
357
358 String ResourceLoadObserver::statisticsForOrigin(const String& origin)
359 {
360     auto iter = m_resourceStatisticsMap.find(origin);
361     if (iter == m_resourceStatisticsMap.end())
362         return emptyString();
363
364     return "Statistics for " + origin + ":\n" + iter->value.toString();
365 }
366
367 Vector<ResourceLoadStatistics> ResourceLoadObserver::takeStatistics()
368 {
369     Vector<ResourceLoadStatistics> statistics;
370     statistics.reserveInitialCapacity(m_resourceStatisticsMap.size());
371     for (auto& statistic : m_resourceStatisticsMap.values())
372         statistics.uncheckedAppend(WTFMove(statistic));
373
374     m_resourceStatisticsMap.clear();
375
376     return statistics;
377 }
378
379 void ResourceLoadObserver::clearState()
380 {
381     m_notificationTimer.stop();
382     m_resourceStatisticsMap.clear();
383     m_lastReportedUserInteractionMap.clear();
384 }
385
386 URL ResourceLoadObserver::nonNullOwnerURL(const Document& document) const
387 {
388     auto url = document.url();
389     auto* frame = document.frame();
390     auto host = document.url().host();
391
392     while ((host.isNull() || host.isEmpty()) && frame && !frame->isMainFrame()) {
393         auto* ownerElement = frame->ownerElement();
394
395         ASSERT(ownerElement != nullptr);
396         
397         auto& doc = ownerElement->document();
398         frame = doc.frame();
399         url = doc.url();
400         host = url.host();
401     }
402
403     return url;
404 }
405
406 } // namespace WebCore