Unreviewed, rolling out r240446.
[WebKit-https.git] / Source / WebCore / loader / ResourceLoadObserver.cpp
1 /*
2  * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadObserver.h"
28
29 #include "DeprecatedGlobalSettings.h"
30 #include "Document.h"
31 #include "Frame.h"
32 #include "FrameLoader.h"
33 #include "HTMLFrameOwnerElement.h"
34 #include "Logging.h"
35 #include "Page.h"
36 #include "ResourceLoadStatistics.h"
37 #include "ResourceRequest.h"
38 #include "ResourceResponse.h"
39 #include "RuntimeEnabledFeatures.h"
40 #include "ScriptExecutionContext.h"
41 #include "SecurityOrigin.h"
42 #include "Settings.h"
43 #include <wtf/URL.h>
44
45 namespace WebCore {
46
47 template<typename T> static inline String primaryDomain(const T& value)
48 {
49     return ResourceLoadStatistics::primaryDomain(value);
50 }
51
52 static const Seconds minimumNotificationInterval { 5_s };
53
54 ResourceLoadObserver& ResourceLoadObserver::shared()
55 {
56     static NeverDestroyed<ResourceLoadObserver> resourceLoadObserver;
57     return resourceLoadObserver;
58 }
59
60 void ResourceLoadObserver::setNotificationCallback(WTF::Function<void (Vector<ResourceLoadStatistics>&&)>&& notificationCallback)
61 {
62     ASSERT(!m_notificationCallback);
63     m_notificationCallback = WTFMove(notificationCallback);
64 }
65
66 void ResourceLoadObserver::setRequestStorageAccessUnderOpenerCallback(WTF::Function<void(const String& domainInNeedOfStorageAccess, uint64_t openerPageID, const String& openerDomain)>&& callback)
67 {
68     ASSERT(!m_requestStorageAccessUnderOpenerCallback);
69     m_requestStorageAccessUnderOpenerCallback = WTFMove(callback);
70 }
71
72 void ResourceLoadObserver::setLogUserInteractionNotificationCallback(Function<void(PAL::SessionID, const String&)>&& callback)
73 {
74     ASSERT(!m_logUserInteractionNotificationCallback);
75     m_logUserInteractionNotificationCallback = WTFMove(callback);
76 }
77
78 ResourceLoadObserver::ResourceLoadObserver()
79     : m_notificationTimer(*this, &ResourceLoadObserver::notifyObserver)
80 {
81 }
82
83 static inline bool is3xxRedirect(const ResourceResponse& response)
84 {
85     return response.httpStatusCode() >= 300 && response.httpStatusCode() <= 399;
86 }
87
88 bool ResourceLoadObserver::shouldLog(bool usesEphemeralSession) const
89 {
90     return DeprecatedGlobalSettings::resourceLoadStatisticsEnabled() && !usesEphemeralSession && m_notificationCallback;
91 }
92
93 void ResourceLoadObserver::logSubresourceLoading(const Frame* frame, const ResourceRequest& newRequest, const ResourceResponse& redirectResponse)
94 {
95     ASSERT(frame->page());
96
97     if (!frame)
98         return;
99
100     auto* page = frame->page();
101     if (!page || !shouldLog(page->usesEphemeralSession()))
102         return;
103
104     bool isRedirect = is3xxRedirect(redirectResponse);
105     const URL& sourceURL = redirectResponse.url();
106     const URL& targetURL = newRequest.url();
107     const URL& mainFrameURL = frame ? frame->mainFrame().document()->url() : URL();
108     
109     auto targetHost = targetURL.host();
110     auto mainFrameHost = mainFrameURL.host();
111
112     if (targetHost.isEmpty() || mainFrameHost.isEmpty() || targetHost == mainFrameHost || (isRedirect && targetHost == sourceURL.host()))
113         return;
114
115     auto targetPrimaryDomain = primaryDomain(targetURL);
116     auto mainFramePrimaryDomain = primaryDomain(mainFrameURL);
117     auto sourcePrimaryDomain = primaryDomain(sourceURL);
118
119     if (targetPrimaryDomain == mainFramePrimaryDomain || (isRedirect && targetPrimaryDomain == sourcePrimaryDomain))
120         return;
121
122     bool shouldCallNotificationCallback = false;
123     {
124         auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
125         targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
126         if (targetStatistics.subresourceUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
127             shouldCallNotificationCallback = true;
128     }
129
130     if (isRedirect) {
131         auto& redirectingOriginStatistics = ensureResourceStatisticsForPrimaryDomain(sourcePrimaryDomain);
132         bool isNewRedirectToEntry = redirectingOriginStatistics.subresourceUniqueRedirectsTo.add(targetPrimaryDomain).isNewEntry;
133         auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
134         bool isNewRedirectFromEntry = targetStatistics.subresourceUniqueRedirectsFrom.add(sourcePrimaryDomain).isNewEntry;
135
136         if (isNewRedirectToEntry || isNewRedirectFromEntry)
137             shouldCallNotificationCallback = true;
138     }
139
140     if (shouldCallNotificationCallback)
141         scheduleNotificationIfNeeded();
142 }
143
144 void ResourceLoadObserver::logWebSocketLoading(const URL& targetURL, const URL& mainFrameURL, bool usesEphemeralSession)
145 {
146     if (!shouldLog(usesEphemeralSession))
147         return;
148
149     auto targetHost = targetURL.host();
150     auto mainFrameHost = mainFrameURL.host();
151     
152     if (targetHost.isEmpty() || mainFrameHost.isEmpty() || targetHost == mainFrameHost)
153         return;
154     
155     auto targetPrimaryDomain = primaryDomain(targetURL);
156     auto mainFramePrimaryDomain = primaryDomain(mainFrameURL);
157
158     if (targetPrimaryDomain == mainFramePrimaryDomain)
159         return;
160
161     auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
162     targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
163     if (targetStatistics.subresourceUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
164         scheduleNotificationIfNeeded();
165 }
166
167 void ResourceLoadObserver::logUserInteractionWithReducedTimeResolution(const Document& document)
168 {
169     if (!shouldLog(document.sessionID().isEphemeral()))
170         return;
171
172     auto& url = document.url();
173     if (url.protocolIsAbout() || url.isEmpty())
174         return;
175
176     auto domain = primaryDomain(url);
177     auto newTime = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
178     auto lastReportedUserInteraction = m_lastReportedUserInteractionMap.get(domain);
179     if (newTime == lastReportedUserInteraction)
180         return;
181
182     m_lastReportedUserInteractionMap.set(domain, newTime);
183
184     auto& statistics = ensureResourceStatisticsForPrimaryDomain(domain);
185     statistics.hadUserInteraction = true;
186     statistics.lastSeen = newTime;
187     statistics.mostRecentUserInteractionTime = newTime;
188
189 #if ENABLE(RESOURCE_LOAD_STATISTICS)
190     if (auto* opener = document.frame()->loader().opener()) {
191         if (auto* openerDocument = opener->document()) {
192             if (auto* openerFrame = openerDocument->frame()) {
193                 if (auto openerPageID = openerFrame->loader().client().pageID()) {
194                     requestStorageAccessUnderOpener(domain, openerPageID.value(), *openerDocument);
195                 }
196             }
197         }
198     }
199
200     // FIXME(193297): Uncomment this line when ResourceLoadStatistics are no longer gathered in the UI Process.
201     // m_logUserInteractionNotificationCallback(document.sessionID(), domain);
202 #endif
203
204     m_notificationTimer.stop();
205     notifyObserver();
206
207 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
208     if (shouldLogUserInteraction()) {
209         auto counter = ++m_loggingCounter;
210 #define LOCAL_LOG(str, ...) \
211         RELEASE_LOG(ResourceLoadStatistics, "ResourceLoadObserver::logUserInteraction: counter = %" PRIu64 ": " str, counter, ##__VA_ARGS__)
212
213         auto escapeForJSON = [](String s) {
214             s.replace('\\', "\\\\").replace('"', "\\\"");
215             return s;
216         };
217         auto escapedURL = escapeForJSON(url.string());
218         auto escapedDomain = escapeForJSON(domain);
219
220         LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
221         LOCAL_LOG(R"(  "domain" : "%{public}s",)", escapedDomain.utf8().data());
222         LOCAL_LOG(R"(  "until" : %f })", newTime.secondsSinceEpoch().seconds());
223
224 #undef LOCAL_LOG
225     }
226 #endif
227 }
228
229 #if ENABLE(RESOURCE_LOAD_STATISTICS)
230 void ResourceLoadObserver::requestStorageAccessUnderOpener(const String& domainInNeedOfStorageAccess, uint64_t openerPageID, Document& openerDocument)
231 {
232     auto openerUrl = openerDocument.url();
233     auto openerPrimaryDomain = primaryDomain(openerUrl);
234     if (domainInNeedOfStorageAccess != openerPrimaryDomain
235         && !openerDocument.hasRequestedPageSpecificStorageAccessWithUserInteraction(domainInNeedOfStorageAccess)
236         && !equalIgnoringASCIICase(openerUrl.string(), WTF::blankURL())) {
237         m_requestStorageAccessUnderOpenerCallback(domainInNeedOfStorageAccess, openerPageID, openerPrimaryDomain);
238         // Remember user interaction-based requests since they don't need to be repeated.
239         openerDocument.setHasRequestedPageSpecificStorageAccessWithUserInteraction(domainInNeedOfStorageAccess);
240     }
241 }
242 #endif
243
244 void ResourceLoadObserver::logFontLoad(const Document& document, const String& familyName, bool loadStatus)
245 {
246 #if ENABLE(WEB_API_STATISTICS)
247     if (!shouldLog(document.sessionID().isEphemeral()))
248         return;
249     auto registrableDomain = primaryDomain(document.url());
250     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
251     bool shouldCallNotificationCallback = false;
252     if (!loadStatus) {
253         if (statistics.fontsFailedToLoad.add(familyName).isNewEntry)
254             shouldCallNotificationCallback = true;
255     } else {
256         if (statistics.fontsSuccessfullyLoaded.add(familyName).isNewEntry)
257             shouldCallNotificationCallback = true;
258     }
259     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
260     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
261         shouldCallNotificationCallback = true;
262     if (shouldCallNotificationCallback)
263         scheduleNotificationIfNeeded();
264 #else
265     UNUSED_PARAM(document);
266     UNUSED_PARAM(familyName);
267     UNUSED_PARAM(loadStatus);
268 #endif
269 }
270     
271 void ResourceLoadObserver::logCanvasRead(const Document& document)
272 {
273 #if ENABLE(WEB_API_STATISTICS)
274     if (!shouldLog(document.sessionID().isEphemeral()))
275         return;
276     auto registrableDomain = primaryDomain(document.url());
277     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
278     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
279     statistics.canvasActivityRecord.wasDataRead = true;
280     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
281         scheduleNotificationIfNeeded();
282 #else
283     UNUSED_PARAM(document);
284 #endif
285 }
286
287 void ResourceLoadObserver::logCanvasWriteOrMeasure(const Document& document, const String& textWritten)
288 {
289 #if ENABLE(WEB_API_STATISTICS)
290     if (!shouldLog(document.sessionID().isEphemeral()))
291         return;
292     auto registrableDomain = primaryDomain(document.url());
293     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
294     bool shouldCallNotificationCallback = false;
295     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
296     if (statistics.canvasActivityRecord.recordWrittenOrMeasuredText(textWritten))
297         shouldCallNotificationCallback = true;
298     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
299         shouldCallNotificationCallback = true;
300     if (shouldCallNotificationCallback)
301         scheduleNotificationIfNeeded();
302 #else
303     UNUSED_PARAM(document);
304     UNUSED_PARAM(textWritten);
305 #endif
306 }
307     
308 void ResourceLoadObserver::logNavigatorAPIAccessed(const Document& document, const ResourceLoadStatistics::NavigatorAPI functionName)
309 {
310 #if ENABLE(WEB_API_STATISTICS)
311     if (!shouldLog(document.sessionID().isEphemeral()))
312         return;
313     auto registrableDomain = primaryDomain(document.url());
314     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
315     bool shouldCallNotificationCallback = false;
316     if (!statistics.navigatorFunctionsAccessed.contains(functionName)) {
317         statistics.navigatorFunctionsAccessed.add(functionName);
318         shouldCallNotificationCallback = true;
319     }
320     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
321     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
322         shouldCallNotificationCallback = true;
323     if (shouldCallNotificationCallback)
324         scheduleNotificationIfNeeded();
325 #else
326     UNUSED_PARAM(document);
327     UNUSED_PARAM(functionName);
328 #endif
329 }
330     
331 void ResourceLoadObserver::logScreenAPIAccessed(const Document& document, const ResourceLoadStatistics::ScreenAPI functionName)
332 {
333 #if ENABLE(WEB_API_STATISTICS)
334     if (!shouldLog(document.sessionID().isEphemeral()))
335         return;
336     auto registrableDomain = primaryDomain(document.url());
337     auto& statistics = ensureResourceStatisticsForPrimaryDomain(registrableDomain);
338     bool shouldCallNotificationCallback = false;
339     if (!statistics.screenFunctionsAccessed.contains(functionName)) {
340         statistics.screenFunctionsAccessed.add(functionName);
341         shouldCallNotificationCallback = true;
342     }
343     auto mainFrameRegistrableDomain = primaryDomain(document.topDocument().url());
344     if (statistics.topFrameRegistrableDomainsWhichAccessedWebAPIs.add(mainFrameRegistrableDomain).isNewEntry)
345         shouldCallNotificationCallback = true;
346     if (shouldCallNotificationCallback)
347         scheduleNotificationIfNeeded();
348 #else
349     UNUSED_PARAM(document);
350     UNUSED_PARAM(functionName);
351 #endif
352 }
353     
354 ResourceLoadStatistics& ResourceLoadObserver::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
355 {
356     auto addResult = m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
357         return ResourceLoadStatistics(primaryDomain);
358     });
359     return addResult.iterator->value;
360 }
361
362 void ResourceLoadObserver::scheduleNotificationIfNeeded()
363 {
364     ASSERT(m_notificationCallback);
365     if (m_resourceStatisticsMap.isEmpty()) {
366         m_notificationTimer.stop();
367         return;
368     }
369
370     if (!m_notificationTimer.isActive())
371         m_notificationTimer.startOneShot(minimumNotificationInterval);
372 }
373
374 void ResourceLoadObserver::notifyObserver()
375 {
376     ASSERT(m_notificationCallback);
377     m_notificationTimer.stop();
378     m_notificationCallback(takeStatistics());
379 }
380
381 String ResourceLoadObserver::statisticsForOrigin(const String& origin)
382 {
383     auto iter = m_resourceStatisticsMap.find(origin);
384     if (iter == m_resourceStatisticsMap.end())
385         return emptyString();
386
387     return "Statistics for " + origin + ":\n" + iter->value.toString();
388 }
389
390 Vector<ResourceLoadStatistics> ResourceLoadObserver::takeStatistics()
391 {
392     Vector<ResourceLoadStatistics> statistics;
393     statistics.reserveInitialCapacity(m_resourceStatisticsMap.size());
394     for (auto& statistic : m_resourceStatisticsMap.values())
395         statistics.uncheckedAppend(WTFMove(statistic));
396
397     m_resourceStatisticsMap.clear();
398
399     return statistics;
400 }
401
402 void ResourceLoadObserver::clearState()
403 {
404     m_notificationTimer.stop();
405     m_resourceStatisticsMap.clear();
406     m_lastReportedUserInteractionMap.clear();
407 }
408
409 URL ResourceLoadObserver::nonNullOwnerURL(const Document& document) const
410 {
411     auto url = document.url();
412     auto* frame = document.frame();
413     auto host = document.url().host();
414
415     while ((host.isNull() || host.isEmpty()) && frame && !frame->isMainFrame()) {
416         auto* ownerElement = frame->ownerElement();
417
418         ASSERT(ownerElement != nullptr);
419         
420         auto& doc = ownerElement->document();
421         frame = doc.frame();
422         url = doc.url();
423         host = url.host();
424     }
425
426     return url;
427 }
428
429 } // namespace WebCore