Use a Variant for FormDataElement
[WebKit-https.git] / Source / WebKit / NetworkProcess / NetworkResourceLoader.cpp
1 /*
2  * Copyright (C) 2012-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkResourceLoader.h"
28
29 #include "DataReference.h"
30 #include "FormDataReference.h"
31 #include "Logging.h"
32 #include "NetworkBlobRegistry.h"
33 #include "NetworkCache.h"
34 #include "NetworkConnectionToWebProcess.h"
35 #include "NetworkLoad.h"
36 #include "NetworkLoadChecker.h"
37 #include "NetworkProcess.h"
38 #include "NetworkProcessConnectionMessages.h"
39 #include "SessionTracker.h"
40 #include "SharedBufferDataReference.h"
41 #include "WebCoreArgumentCoders.h"
42 #include "WebErrors.h"
43 #include "WebPageMessages.h"
44 #include "WebResourceLoaderMessages.h"
45 #include "WebsiteDataStoreParameters.h"
46 #include <JavaScriptCore/ConsoleTypes.h>
47 #include <WebCore/BlobDataFileReference.h>
48 #include <WebCore/CertificateInfo.h>
49 #include <WebCore/ContentSecurityPolicy.h>
50 #include <WebCore/DiagnosticLoggingKeys.h>
51 #include <WebCore/HTTPHeaderNames.h>
52 #include <WebCore/HTTPParsers.h>
53 #include <WebCore/NetworkLoadMetrics.h>
54 #include <WebCore/ProtectionSpace.h>
55 #include <WebCore/SameSiteInfo.h>
56 #include <WebCore/SecurityOrigin.h>
57 #include <WebCore/SharedBuffer.h>
58 #include <WebCore/SynchronousLoaderClient.h>
59 #include <wtf/RunLoop.h>
60
61 #if HAVE(CFNETWORK_STORAGE_PARTITIONING) && !RELEASE_LOG_DISABLED
62 #include <WebCore/NetworkStorageSession.h>
63 #endif
64
65 #if USE(QUICK_LOOK)
66 #include <WebCore/PreviewLoader.h>
67 #endif
68
69 #define RELEASE_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__)
70 #define RELEASE_LOG_ERROR_IF_ALLOWED(fmt, ...) RELEASE_LOG_ERROR_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__)
71
72 namespace WebKit {
73 using namespace WebCore;
74
75 struct NetworkResourceLoader::SynchronousLoadData {
76     SynchronousLoadData(Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply&& reply)
77         : delayedReply(WTFMove(reply))
78     {
79         ASSERT(delayedReply);
80     }
81     ResourceRequest currentRequest;
82     Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply delayedReply;
83     ResourceResponse response;
84     ResourceError error;
85 };
86
87 static void sendReplyToSynchronousRequest(NetworkResourceLoader::SynchronousLoadData& data, const SharedBuffer* buffer)
88 {
89     ASSERT(data.delayedReply);
90     ASSERT(!data.response.isNull() || !data.error.isNull());
91
92     Vector<char> responseBuffer;
93     if (buffer && buffer->size())
94         responseBuffer.append(buffer->data(), buffer->size());
95
96     data.delayedReply(data.error, data.response, responseBuffer);
97     data.delayedReply = nullptr;
98 }
99
100 NetworkResourceLoader::NetworkResourceLoader(NetworkResourceLoadParameters&& parameters, NetworkConnectionToWebProcess& connection, Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply&& synchronousReply)
101     : m_parameters { WTFMove(parameters) }
102     , m_connection { connection }
103     , m_defersLoading { parameters.defersLoading }
104     , m_isAllowedToAskUserForCredentials { m_parameters.clientCredentialPolicy == ClientCredentialPolicy::MayAskClientForCredentials }
105     , m_bufferingTimer { *this, &NetworkResourceLoader::bufferingTimerFired }
106     , m_cache { sessionID().isEphemeral() ? nullptr : NetworkProcess::singleton().cache() }
107 {
108     ASSERT(RunLoop::isMain());
109     // FIXME: This is necessary because of the existence of EmptyFrameLoaderClient in WebCore.
110     //        Once bug 116233 is resolved, this ASSERT can just be "m_webPageID && m_webFrameID"
111     ASSERT((m_parameters.webPageID && m_parameters.webFrameID) || m_parameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials);
112
113     if (originalRequest().httpBody()) {
114         for (const auto& element : originalRequest().httpBody()->elements()) {
115             if (auto* blobData = WTF::get_if<FormDataElement::EncodedBlobData>(element.data))
116                 m_fileReferences.appendVector(NetworkBlobRegistry::singleton().filesInBlob(connection, blobData->url));
117         }
118     }
119
120     if (synchronousReply || parameters.shouldRestrictHTTPResponseAccess) {
121         m_networkLoadChecker = std::make_unique<NetworkLoadChecker>(FetchOptions { m_parameters.options }, m_parameters.sessionID, m_parameters.webPageID, m_parameters.webFrameID, HTTPHeaderMap { m_parameters.originalRequestHeaders }, URL { m_parameters.request.url() }, m_parameters.sourceOrigin.copyRef(), m_parameters.preflightPolicy, originalRequest().httpReferrer(), shouldCaptureExtraNetworkLoadMetrics());
122         if (m_parameters.cspResponseHeaders)
123             m_networkLoadChecker->setCSPResponseHeaders(ContentSecurityPolicyResponseHeaders { m_parameters.cspResponseHeaders.value() });
124 #if ENABLE(CONTENT_EXTENSIONS)
125         m_networkLoadChecker->setContentExtensionController(URL { m_parameters.mainDocumentURL }, m_parameters.userContentControllerIdentifier);
126 #endif
127     }
128     if (synchronousReply)
129         m_synchronousLoadData = std::make_unique<SynchronousLoadData>(WTFMove(synchronousReply));
130 }
131
132 NetworkResourceLoader::~NetworkResourceLoader()
133 {
134     ASSERT(RunLoop::isMain());
135     ASSERT(!m_networkLoad);
136     ASSERT(!isSynchronous() || !m_synchronousLoadData->delayedReply);
137 }
138
139 bool NetworkResourceLoader::canUseCache(const ResourceRequest& request) const
140 {
141     if (!m_cache)
142         return false;
143     ASSERT(!sessionID().isEphemeral());
144
145     if (!request.url().protocolIsInHTTPFamily())
146         return false;
147     if (originalRequest().cachePolicy() == WebCore::ResourceRequestCachePolicy::DoNotUseAnyCache)
148         return false;
149
150     return true;
151 }
152
153 bool NetworkResourceLoader::canUseCachedRedirect(const ResourceRequest& request) const
154 {
155     if (!canUseCache(request))
156         return false;
157     // Limit cached redirects to avoid cycles and other trouble.
158     // Networking layer follows over 30 redirects but caching that many seems unnecessary.
159     static const unsigned maximumCachedRedirectCount { 5 };
160     if (m_redirectCount > maximumCachedRedirectCount)
161         return false;
162
163     return true;
164 }
165
166 bool NetworkResourceLoader::isSynchronous() const
167 {
168     return !!m_synchronousLoadData;
169 }
170
171 void NetworkResourceLoader::start()
172 {
173     ASSERT(RunLoop::isMain());
174
175     m_networkActivityTracker = m_connection->startTrackingResourceLoad(m_parameters.webPageID, m_parameters.identifier, isMainResource(), sessionID());
176
177     if (m_defersLoading) {
178         RELEASE_LOG_IF_ALLOWED("start: Loading is deferred (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
179         return;
180     }
181
182     ASSERT(!m_wasStarted);
183     m_wasStarted = true;
184
185     if (m_networkLoadChecker) {
186         m_networkLoadChecker->check(ResourceRequest { originalRequest() }, this, [this] (auto&& result) {
187             if (!result.has_value()) {
188                 if (!result.error().isCancellation())
189                     this->didFailLoading(result.error());
190                 return;
191             }
192             if (this->canUseCache(this->originalRequest())) {
193                 RELEASE_LOG_IF_ALLOWED("start: Checking cache for resource (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, this->isMainResource(), this->isSynchronous());
194                 this->retrieveCacheEntry(this->originalRequest());
195                 return;
196             }
197
198             this->startNetworkLoad(WTFMove(result.value()), FirstLoad::Yes);
199         });
200         return;
201     }
202     // FIXME: Remove that code path once m_networkLoadChecker is used for all network loads.
203     if (canUseCache(originalRequest())) {
204         RELEASE_LOG_IF_ALLOWED("start: Checking cache for resource (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
205         retrieveCacheEntry(originalRequest());
206         return;
207     }
208
209     startNetworkLoad(ResourceRequest { originalRequest() }, FirstLoad::Yes);
210 }
211
212 void NetworkResourceLoader::retrieveCacheEntry(const ResourceRequest& request)
213 {
214     ASSERT(canUseCache(request));
215
216     RefPtr<NetworkResourceLoader> loader(this);
217     m_cache->retrieve(request, { m_parameters.webPageID, m_parameters.webFrameID }, sessionID(), [this, loader = WTFMove(loader), request = ResourceRequest { request }](auto entry, auto info) mutable {
218         if (loader->hasOneRef()) {
219             // The loader has been aborted and is only held alive by this lambda.
220             return;
221         }
222
223         loader->logSlowCacheRetrieveIfNeeded(info);
224
225         if (!entry) {
226             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource not in cache (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
227             loader->startNetworkLoad(WTFMove(request), FirstLoad::Yes);
228             return;
229         }
230         if (entry->redirectRequest()) {
231             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Handling redirect (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
232             loader->dispatchWillSendRequestForCacheEntry(WTFMove(request), WTFMove(entry));
233             return;
234         }
235         if (loader->m_parameters.needsCertificateInfo && !entry->response().certificateInfo()) {
236             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource does not have required certificate (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
237             loader->startNetworkLoad(WTFMove(request), FirstLoad::Yes);
238             return;
239         }
240         if (entry->needsValidation() || request.cachePolicy() == WebCore::ResourceRequestCachePolicy::RefreshAnyCacheData) {
241             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Validating cache entry (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
242             loader->validateCacheEntry(WTFMove(entry));
243             return;
244         }
245         RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Retrieved resource from cache (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
246         loader->didRetrieveCacheEntry(WTFMove(entry));
247     });
248 }
249
250 void NetworkResourceLoader::startNetworkLoad(ResourceRequest&& request, FirstLoad load)
251 {
252     if (load == FirstLoad::Yes) {
253         RELEASE_LOG_IF_ALLOWED("startNetworkLoad: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
254
255         consumeSandboxExtensions();
256
257         if (isSynchronous() || m_parameters.maximumBufferingTime > 0_s)
258             m_bufferedData = SharedBuffer::create();
259
260         if (canUseCache(request))
261             m_bufferedDataForCache = SharedBuffer::create();
262     }
263
264     NetworkLoadParameters parameters = m_parameters;
265     parameters.defersLoading = m_defersLoading;
266     parameters.networkActivityTracker = m_networkActivityTracker;
267     if (m_networkLoadChecker)
268         parameters.storedCredentialsPolicy = m_networkLoadChecker->storedCredentialsPolicy();
269
270     if (request.url().protocolIsBlob())
271         parameters.blobFileReferences = NetworkBlobRegistry::singleton().filesInBlob(m_connection, originalRequest().url());
272
273     auto* networkSession = SessionTracker::networkSession(parameters.sessionID);
274     if (!networkSession && parameters.sessionID.isEphemeral()) {
275         NetworkProcess::singleton().addWebsiteDataStore(WebsiteDataStoreParameters::privateSessionParameters(parameters.sessionID));
276         networkSession = SessionTracker::networkSession(parameters.sessionID);
277     }
278     if (!networkSession) {
279         WTFLogAlways("Attempted to create a NetworkLoad with a session (id=%" PRIu64 ") that does not exist.", parameters.sessionID.sessionID());
280         RELEASE_LOG_ERROR_IF_ALLOWED("startNetworkLoad: Attempted to create a NetworkLoad with a session that does not exist (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", sessionID=%" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, parameters.sessionID.sessionID());
281         NetworkProcess::singleton().logDiagnosticMessage(m_parameters.webPageID, WebCore::DiagnosticLoggingKeys::internalErrorKey(), WebCore::DiagnosticLoggingKeys::invalidSessionIDKey(), WebCore::ShouldSample::No);
282         didFailLoading(internalError(request.url()));
283         return;
284     }
285
286     parameters.request = WTFMove(request);
287     m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters), *networkSession);
288
289     RELEASE_LOG_IF_ALLOWED("startNetworkLoad: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", description = %{public}s)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, m_networkLoad->description().utf8().data());
290
291     if (m_defersLoading) {
292         RELEASE_LOG_IF_ALLOWED("startNetworkLoad: Created, but deferred (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")",
293             m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
294     }
295 }
296
297 void NetworkResourceLoader::setDefersLoading(bool defers)
298 {
299     if (m_defersLoading == defers)
300         return;
301     m_defersLoading = defers;
302
303     if (defers)
304         RELEASE_LOG_IF_ALLOWED("setDefersLoading: Deferring resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
305     else
306         RELEASE_LOG_IF_ALLOWED("setDefersLoading: Resuming deferred resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
307
308     if (m_networkLoad) {
309         m_networkLoad->setDefersLoading(defers);
310         return;
311     }
312
313     if (!m_defersLoading && !m_wasStarted)
314         start();
315     else
316         RELEASE_LOG_IF_ALLOWED("setDefersLoading: defers = %d, but nothing to do (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_defersLoading, m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
317 }
318
319 void NetworkResourceLoader::cleanup(LoadResult result)
320 {
321     ASSERT(RunLoop::isMain());
322
323     m_connection->stopTrackingResourceLoad(m_parameters.identifier,
324         result == LoadResult::Success ? NetworkActivityTracker::CompletionCode::Success :
325         result == LoadResult::Failure ? NetworkActivityTracker::CompletionCode::Failure :
326         NetworkActivityTracker::CompletionCode::None);
327
328     m_bufferingTimer.stop();
329
330     invalidateSandboxExtensions();
331
332     m_networkLoad = nullptr;
333
334     // This will cause NetworkResourceLoader to be destroyed and therefore we do it last.
335     m_connection->didCleanupResourceLoader(*this);
336 }
337
338 void NetworkResourceLoader::convertToDownload(DownloadID downloadID, const ResourceRequest& request, const ResourceResponse& response)
339 {
340     // This can happen if the resource came from the disk cache.
341     if (!m_networkLoad) {
342         NetworkProcess::singleton().downloadManager().startDownload(m_connection.ptr(), m_parameters.sessionID, downloadID, request);
343         abort();
344         return;
345     }
346
347     NetworkProcess::singleton().downloadManager().convertNetworkLoadToDownload(downloadID, std::exchange(m_networkLoad, nullptr), WTFMove(m_fileReferences), request, response);
348 }
349
350 void NetworkResourceLoader::abort()
351 {
352     ASSERT(RunLoop::isMain());
353
354     RELEASE_LOG_IF_ALLOWED("abort: Canceling resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")",
355         m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
356
357     if (m_networkLoad) {
358         if (canUseCache(m_networkLoad->currentRequest())) {
359             // We might already have used data from this incomplete load. Ensure older versions don't remain in the cache after cancel.
360             if (!m_response.isNull())
361                 m_cache->remove(m_networkLoad->currentRequest());
362         }
363         m_networkLoad->cancel();
364     }
365
366     cleanup(LoadResult::Cancel);
367 }
368
369 bool NetworkResourceLoader::shouldInterruptLoadForXFrameOptions(const String& xFrameOptions, const URL& url)
370 {
371     if (isMainFrameLoad())
372         return false;
373
374     switch (parseXFrameOptionsHeader(xFrameOptions)) {
375     case XFrameOptionsNone:
376     case XFrameOptionsAllowAll:
377         return false;
378     case XFrameOptionsDeny:
379         return true;
380     case XFrameOptionsSameOrigin: {
381         auto origin = SecurityOrigin::create(url);
382         auto topFrameOrigin = m_parameters.frameAncestorOrigins.last();
383         if (!origin->isSameSchemeHostPort(*topFrameOrigin))
384             return true;
385         for (auto& ancestorOrigin : m_parameters.frameAncestorOrigins) {
386             if (!origin->isSameSchemeHostPort(*ancestorOrigin))
387                 return true;
388         }
389         return false;
390     }
391     case XFrameOptionsConflict: {
392         String errorMessage = "Multiple 'X-Frame-Options' headers with conflicting values ('" + xFrameOptions + "') encountered when loading '" + url.stringCenterEllipsizedToLength() + "'. Falling back to 'DENY'.";
393         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::JS, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
394         return true;
395     }
396     case XFrameOptionsInvalid: {
397         String errorMessage = "Invalid 'X-Frame-Options' header encountered when loading '" + url.stringCenterEllipsizedToLength() + "': '" + xFrameOptions + "' is not a recognized directive. The header will be ignored.";
398         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::JS, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
399         return false;
400     }
401     }
402     ASSERT_NOT_REACHED();
403     return false;
404 }
405
406 bool NetworkResourceLoader::shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(const ResourceResponse& response)
407 {
408     ASSERT(isMainResource());
409
410 #if USE(QUICK_LOOK)
411     if (PreviewLoader::shouldCreateForMIMEType(response.mimeType()))
412         return false;
413 #endif
414
415     auto url = response.url();
416     ContentSecurityPolicy contentSecurityPolicy { URL { url }, this };
417     contentSecurityPolicy.didReceiveHeaders(ContentSecurityPolicyResponseHeaders { response }, originalRequest().httpReferrer());
418     if (!contentSecurityPolicy.allowFrameAncestors(m_parameters.frameAncestorOrigins, url))
419         return true;
420     String xFrameOptions = m_response.httpHeaderField(HTTPHeaderName::XFrameOptions);
421     if (!xFrameOptions.isNull() && shouldInterruptLoadForXFrameOptions(xFrameOptions, response.url())) {
422         String errorMessage = "Refused to display '" + response.url().stringCenterEllipsizedToLength() + "' in a frame because it set 'X-Frame-Options' to '" + xFrameOptions + "'.";
423         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::Security, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
424         return true;
425     }
426     return false;
427 }
428
429 auto NetworkResourceLoader::didReceiveResponse(ResourceResponse&& receivedResponse) -> ShouldContinueDidReceiveResponse
430 {
431     RELEASE_LOG_IF_ALLOWED("didReceiveResponse: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", httpStatusCode = %d, length = %" PRId64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, receivedResponse.httpStatusCode(), receivedResponse.expectedContentLength());
432
433     m_response = WTFMove(receivedResponse);
434
435     if (shouldCaptureExtraNetworkLoadMetrics() && m_networkLoadChecker) {
436         auto information = m_networkLoadChecker->takeNetworkLoadInformation();
437         information.response = m_response;
438         m_connection->addNetworkLoadInformation(identifier(), WTFMove(information));
439     }
440
441     // For multipart/x-mixed-replace didReceiveResponseAsync gets called multiple times and buffering would require special handling.
442     if (!isSynchronous() && m_response.isMultipart())
443         m_bufferedData = nullptr;
444
445     if (m_response.isMultipart())
446         m_bufferedDataForCache = nullptr;
447
448     if (m_cacheEntryForValidation) {
449         bool validationSucceeded = m_response.httpStatusCode() == 304; // 304 Not Modified
450         if (validationSucceeded) {
451             m_cacheEntryForValidation = m_cache->update(originalRequest(), { m_parameters.webPageID, m_parameters.webFrameID }, *m_cacheEntryForValidation, m_response);
452             // If the request was conditional then this revalidation was not triggered by the network cache and we pass the 304 response to WebCore.
453             if (originalRequest().isConditional())
454                 m_cacheEntryForValidation = nullptr;
455         } else
456             m_cacheEntryForValidation = nullptr;
457     }
458     if (m_cacheEntryForValidation)
459         return ShouldContinueDidReceiveResponse::Yes;
460
461     if (isMainResource() && shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(m_response)) {
462         send(Messages::WebResourceLoader::StopLoadingAfterXFrameOptionsOrContentSecurityPolicyDenied { });
463         return ShouldContinueDidReceiveResponse::No;
464     }
465
466     if (m_networkLoadChecker) {
467         auto error = m_networkLoadChecker->validateResponse(m_response);
468         if (!error.isNull()) {
469             RunLoop::main().dispatch([protectedThis = makeRef(*this), error = WTFMove(error)] {
470                 if (protectedThis->m_networkLoad)
471                     protectedThis->didFailLoading(error);
472             });
473             return ShouldContinueDidReceiveResponse::No;
474         }
475     }
476
477     auto response = sanitizeResponseIfPossible(ResourceResponse { m_response }, ResourceResponse::SanitizationType::CrossOriginSafe);
478     if (isSynchronous()) {
479         m_synchronousLoadData->response = WTFMove(response);
480         return ShouldContinueDidReceiveResponse::Yes;
481     }
482
483     // We wait to receive message NetworkResourceLoader::ContinueDidReceiveResponse before continuing a load for
484     // a main resource because the embedding client must decide whether to allow the load.
485     bool willWaitForContinueDidReceiveResponse = isMainResource();
486     send(Messages::WebResourceLoader::DidReceiveResponse { response, willWaitForContinueDidReceiveResponse });
487     return willWaitForContinueDidReceiveResponse ? ShouldContinueDidReceiveResponse::No : ShouldContinueDidReceiveResponse::Yes;
488 }
489
490 void NetworkResourceLoader::didReceiveBuffer(Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength)
491 {
492     if (!m_numBytesReceived) {
493         RELEASE_LOG_IF_ALLOWED("didReceiveBuffer: Started receiving data (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
494     }
495     m_numBytesReceived += buffer->size();
496
497     ASSERT(!m_cacheEntryForValidation);
498
499     if (m_bufferedDataForCache) {
500         // Prevent memory growth in case of streaming data.
501         const size_t maximumCacheBufferSize = 10 * 1024 * 1024;
502         if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize)
503             m_bufferedDataForCache->append(buffer.get());
504         else
505             m_bufferedDataForCache = nullptr;
506     }
507     // FIXME: At least on OS X Yosemite we always get -1 from the resource handle.
508     unsigned encodedDataLength = reportedEncodedDataLength >= 0 ? reportedEncodedDataLength : buffer->size();
509
510     m_bytesReceived += buffer->size();
511     if (m_bufferedData) {
512         m_bufferedData->append(buffer.get());
513         m_bufferedDataEncodedDataLength += encodedDataLength;
514         startBufferingTimerIfNeeded();
515         return;
516     }
517     sendBuffer(buffer, encodedDataLength);
518 }
519
520 void NetworkResourceLoader::didFinishLoading(const NetworkLoadMetrics& networkLoadMetrics)
521 {
522     RELEASE_LOG_IF_ALLOWED("didFinishLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", length = %zd)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, m_numBytesReceived);
523
524     if (shouldCaptureExtraNetworkLoadMetrics())
525         m_connection->addNetworkLoadInformationMetrics(identifier(), networkLoadMetrics);
526
527     if (m_cacheEntryForValidation) {
528         // 304 Not Modified
529         ASSERT(m_response.httpStatusCode() == 304);
530         LOG(NetworkCache, "(NetworkProcess) revalidated");
531         didRetrieveCacheEntry(WTFMove(m_cacheEntryForValidation));
532         return;
533     }
534
535 #if HAVE(CFNETWORK_STORAGE_PARTITIONING) && !RELEASE_LOG_DISABLED
536     if (shouldLogCookieInformation())
537         logCookieInformation();
538 #endif
539
540     if (isSynchronous())
541         sendReplyToSynchronousRequest(*m_synchronousLoadData, m_bufferedData.get());
542     else {
543         if (m_bufferedData && !m_bufferedData->isEmpty()) {
544             // FIXME: Pass a real value or remove the encoded data size feature.
545             sendBuffer(*m_bufferedData, -1);
546         }
547         send(Messages::WebResourceLoader::DidFinishResourceLoad(networkLoadMetrics));
548     }
549
550     tryStoreAsCacheEntry();
551
552     cleanup(LoadResult::Success);
553 }
554
555 void NetworkResourceLoader::didFailLoading(const ResourceError& error)
556 {
557     RELEASE_LOG_IF_ALLOWED("didFailLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isTimeout = %d, isCancellation = %d, isAccessControl = %d, errCode = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, error.isTimeout(), error.isCancellation(), error.isAccessControl(), error.errorCode());
558
559     if (shouldCaptureExtraNetworkLoadMetrics())
560         m_connection->removeNetworkLoadInformation(identifier());
561
562     ASSERT(!error.isNull());
563
564     m_cacheEntryForValidation = nullptr;
565
566     if (isSynchronous()) {
567         m_synchronousLoadData->error = error;
568         sendReplyToSynchronousRequest(*m_synchronousLoadData, nullptr);
569     } else if (auto* connection = messageSenderConnection())
570         connection->send(Messages::WebResourceLoader::DidFailResourceLoad(error), messageSenderDestinationID());
571
572     cleanup(LoadResult::Failure);
573 }
574
575 void NetworkResourceLoader::didBlockAuthenticationChallenge()
576 {
577     send(Messages::WebResourceLoader::DidBlockAuthenticationChallenge());
578 }
579
580 void NetworkResourceLoader::willSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse)
581 {
582     ++m_redirectCount;
583
584     if (redirectResponse.source() == ResourceResponse::Source::Network && canUseCachedRedirect(request))
585         m_cache->storeRedirect(request, redirectResponse, redirectRequest);
586
587     if (m_networkLoadChecker) {
588         m_networkLoadChecker->storeRedirectionIfNeeded(request, redirectResponse);
589         m_networkLoadChecker->checkRedirection(WTFMove(request), WTFMove(redirectRequest), WTFMove(redirectResponse), this, [protectedThis = makeRef(*this), this, storedCredentialsPolicy = m_networkLoadChecker->storedCredentialsPolicy()](auto&& result) mutable {
590             if (!result.has_value()) {
591                 if (result.error().isCancellation())
592                     return;
593
594                 this->didFailLoading(result.error());
595                 return;
596             }
597
598             if (m_parameters.options.redirect == FetchOptions::Redirect::Manual) {
599                 this->didFinishWithRedirectResponse(WTFMove(result->redirectResponse));
600                 return;
601             }
602
603             if (this->isSynchronous()) {
604                 if (storedCredentialsPolicy != m_networkLoadChecker->storedCredentialsPolicy()) {
605                     // We need to restart the load to update the session according the new credential policy.
606                     this->restartNetworkLoad(WTFMove(result->redirectRequest));
607                     return;
608                 }
609
610                 // We do not support prompting for credentials for synchronous loads. If we ever change this policy then
611                 // we need to take care to prompt if and only if request and redirectRequest are not mixed content.
612                 this->continueWillSendRequest(WTFMove(result->redirectRequest), false);
613                 return;
614             }
615
616             m_shouldRestartLoad = storedCredentialsPolicy != m_networkLoadChecker->storedCredentialsPolicy();
617             this->continueWillSendRedirectedRequest(WTFMove(result->request), WTFMove(result->redirectRequest), WTFMove(result->redirectResponse));
618         });
619         return;
620     }
621     continueWillSendRedirectedRequest(WTFMove(request), WTFMove(redirectRequest), WTFMove(redirectResponse));
622 }
623
624 void NetworkResourceLoader::continueWillSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse)
625 {
626     ASSERT(!isSynchronous());
627
628     send(Messages::WebResourceLoader::WillSendRequest(redirectRequest, sanitizeResponseIfPossible(WTFMove(redirectResponse), ResourceResponse::SanitizationType::Redirection)));
629 }
630
631 void NetworkResourceLoader::didFinishWithRedirectResponse(ResourceResponse&& redirectResponse)
632 {
633     redirectResponse.setType(ResourceResponse::Type::Opaqueredirect);
634     didReceiveResponse(WTFMove(redirectResponse));
635
636     WebCore::NetworkLoadMetrics networkLoadMetrics;
637     networkLoadMetrics.markComplete();
638     networkLoadMetrics.responseBodyBytesReceived = 0;
639     networkLoadMetrics.responseBodyDecodedSize = 0;
640     send(Messages::WebResourceLoader::DidFinishResourceLoad { networkLoadMetrics });
641
642     cleanup(LoadResult::Success);
643 }
644
645 ResourceResponse NetworkResourceLoader::sanitizeResponseIfPossible(ResourceResponse&& response, ResourceResponse::SanitizationType type)
646 {
647     if (m_parameters.shouldRestrictHTTPResponseAccess)
648         response.sanitizeHTTPHeaderFields(type);
649
650     return WTFMove(response);
651 }
652
653 void NetworkResourceLoader::restartNetworkLoad(WebCore::ResourceRequest&& newRequest)
654 {
655     if (m_networkLoad)
656         m_networkLoad->cancel();
657     if (m_networkLoadChecker)
658         m_networkLoadChecker->prepareRedirectedRequest(newRequest);
659
660     startNetworkLoad(WTFMove(newRequest), FirstLoad::No);
661 }
662
663 void NetworkResourceLoader::continueWillSendRequest(ResourceRequest&& newRequest, bool isAllowedToAskUserForCredentials)
664 {
665     if (m_shouldRestartLoad) {
666         m_shouldRestartLoad = false;
667         restartNetworkLoad(WTFMove(newRequest));
668         return;
669     }
670
671     if (m_networkLoadChecker) {
672         // FIXME: We should be doing this check when receiving the redirection and not allow about protocol as per fetch spec.
673         if (!newRequest.url().protocolIsInHTTPFamily() && !newRequest.url().isBlankURL() && m_redirectCount) {
674             didFailLoading(ResourceError { String { }, 0, newRequest.url(), "Redirection to URL with a scheme that is not HTTP(S)"_s, ResourceError::Type::AccessControl });
675             return;
676         }
677     }
678
679     RELEASE_LOG_IF_ALLOWED("continueWillSendRequest: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
680
681     if (m_networkLoadChecker)
682         m_networkLoadChecker->prepareRedirectedRequest(newRequest);
683
684     m_isAllowedToAskUserForCredentials = isAllowedToAskUserForCredentials;
685
686     // If there is a match in the network cache, we need to reuse the original cache policy and partition.
687     newRequest.setCachePolicy(originalRequest().cachePolicy());
688     newRequest.setCachePartition(originalRequest().cachePartition());
689
690     if (m_isWaitingContinueWillSendRequestForCachedRedirect) {
691         m_isWaitingContinueWillSendRequestForCachedRedirect = false;
692
693         LOG(NetworkCache, "(NetworkProcess) Retrieving cached redirect");
694
695         if (canUseCachedRedirect(newRequest))
696             retrieveCacheEntry(newRequest);
697         else
698             startNetworkLoad(WTFMove(newRequest), FirstLoad::Yes);
699
700         return;
701     }
702
703     if (m_networkLoad)
704         m_networkLoad->continueWillSendRequest(WTFMove(newRequest));
705 }
706
707 void NetworkResourceLoader::continueDidReceiveResponse()
708 {
709     if (m_cacheEntryWaitingForContinueDidReceiveResponse) {
710         continueProcessingCachedEntryAfterDidReceiveResponse(WTFMove(m_cacheEntryWaitingForContinueDidReceiveResponse));
711         return;
712     }
713
714     // FIXME: Remove this check once BlobResourceHandle implements didReceiveResponseAsync correctly.
715     // Currently, it does not wait for response, so the load is likely to finish before continueDidReceiveResponse.
716     if (m_networkLoad)
717         m_networkLoad->continueDidReceiveResponse();
718 }
719
720 void NetworkResourceLoader::didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent)
721 {
722     if (!isSynchronous())
723         send(Messages::WebResourceLoader::DidSendData(bytesSent, totalBytesToBeSent));
724 }
725
726 void NetworkResourceLoader::startBufferingTimerIfNeeded()
727 {
728     if (isSynchronous())
729         return;
730     if (m_bufferingTimer.isActive())
731         return;
732     m_bufferingTimer.startOneShot(m_parameters.maximumBufferingTime);
733 }
734
735 void NetworkResourceLoader::bufferingTimerFired()
736 {
737     ASSERT(m_bufferedData);
738     ASSERT(m_networkLoad);
739
740     if (m_bufferedData->isEmpty())
741         return;
742
743     send(Messages::WebResourceLoader::DidReceiveData({ *m_bufferedData }, m_bufferedDataEncodedDataLength));
744
745     m_bufferedData = SharedBuffer::create();
746     m_bufferedDataEncodedDataLength = 0;
747 }
748
749 void NetworkResourceLoader::sendBuffer(SharedBuffer& buffer, size_t encodedDataLength)
750 {
751     ASSERT(!isSynchronous());
752
753     send(Messages::WebResourceLoader::DidReceiveData({ buffer }, encodedDataLength));
754 }
755
756 void NetworkResourceLoader::tryStoreAsCacheEntry()
757 {
758     if (!canUseCache(m_networkLoad->currentRequest()))
759         return;
760     if (!m_bufferedDataForCache)
761         return;
762
763     m_cache->store(m_networkLoad->currentRequest(), m_response, WTFMove(m_bufferedDataForCache), [loader = makeRef(*this)](auto& mappedBody) mutable {
764 #if ENABLE(SHAREABLE_RESOURCE)
765         if (mappedBody.shareableResourceHandle.isNull())
766             return;
767         LOG(NetworkCache, "(NetworkProcess) sending DidCacheResource");
768         loader->send(Messages::NetworkProcessConnection::DidCacheResource(loader->originalRequest(), mappedBody.shareableResourceHandle, loader->sessionID()));
769 #endif
770     });
771 }
772
773 void NetworkResourceLoader::didRetrieveCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
774 {
775     auto response = entry->response();
776
777     if (isMainResource() && shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(response)) {
778         send(Messages::WebResourceLoader::StopLoadingAfterXFrameOptionsOrContentSecurityPolicyDenied { });
779         return;
780     }
781     if (m_networkLoadChecker) {
782         auto error = m_networkLoadChecker->validateResponse(response);
783         if (!error.isNull()) {
784             didFailLoading(error);
785             return;
786         }
787     }
788
789     response = sanitizeResponseIfPossible(WTFMove(response), ResourceResponse::SanitizationType::CrossOriginSafe);
790     if (isSynchronous()) {
791         m_synchronousLoadData->response = WTFMove(response);
792         sendReplyToSynchronousRequest(*m_synchronousLoadData, entry->buffer());
793         cleanup(LoadResult::Success);
794         return;
795     }
796
797     bool needsContinueDidReceiveResponseMessage = isMainResource();
798     send(Messages::WebResourceLoader::DidReceiveResponse { response, needsContinueDidReceiveResponseMessage });
799
800     if (needsContinueDidReceiveResponseMessage)
801         m_cacheEntryWaitingForContinueDidReceiveResponse = WTFMove(entry);
802     else
803         continueProcessingCachedEntryAfterDidReceiveResponse(WTFMove(entry));
804 }
805
806 void NetworkResourceLoader::continueProcessingCachedEntryAfterDidReceiveResponse(std::unique_ptr<NetworkCache::Entry> entry)
807 {
808     if (entry->sourceStorageRecord().bodyHash && !m_parameters.derivedCachedDataTypesToRetrieve.isEmpty()) {
809         auto bodyHash = *entry->sourceStorageRecord().bodyHash;
810         auto* entryPtr = entry.release();
811         auto retrieveCount = m_parameters.derivedCachedDataTypesToRetrieve.size();
812
813         for (auto& type : m_parameters.derivedCachedDataTypesToRetrieve) {
814             NetworkCache::DataKey key { originalRequest().cachePartition(), type, bodyHash };
815             m_cache->retrieveData(key, [loader = makeRef(*this), entryPtr, type, retrieveCount] (const uint8_t* data, size_t size) mutable {
816                 loader->m_retrievedDerivedDataCount++;
817                 bool retrievedAll = loader->m_retrievedDerivedDataCount == retrieveCount;
818                 std::unique_ptr<NetworkCache::Entry> entry(retrievedAll ? entryPtr : nullptr);
819                 if (loader->hasOneRef())
820                     return;
821                 if (data) {
822                     IPC::DataReference dataReference(data, size);
823                     loader->send(Messages::WebResourceLoader::DidRetrieveDerivedData(type, dataReference));
824                 }
825                 if (retrievedAll) {
826                     loader->sendResultForCacheEntry(WTFMove(entry));
827                     loader->cleanup(LoadResult::Success);
828                 }
829             });
830         }
831         return;
832     }
833
834     sendResultForCacheEntry(WTFMove(entry));
835
836     cleanup(LoadResult::Success);
837 }
838
839 void NetworkResourceLoader::sendResultForCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
840 {
841 #if ENABLE(SHAREABLE_RESOURCE)
842     if (!entry->shareableResourceHandle().isNull()) {
843         send(Messages::WebResourceLoader::DidReceiveResource(entry->shareableResourceHandle()));
844         return;
845     }
846 #endif
847
848 #if HAVE(CFNETWORK_STORAGE_PARTITIONING) && !RELEASE_LOG_DISABLED
849     if (shouldLogCookieInformation())
850         logCookieInformation();
851 #endif
852
853     WebCore::NetworkLoadMetrics networkLoadMetrics;
854     networkLoadMetrics.markComplete();
855     networkLoadMetrics.requestHeaderBytesSent = 0;
856     networkLoadMetrics.requestBodyBytesSent = 0;
857     networkLoadMetrics.responseHeaderBytesReceived = 0;
858     networkLoadMetrics.responseBodyBytesReceived = 0;
859     networkLoadMetrics.responseBodyDecodedSize = 0;
860
861     sendBuffer(*entry->buffer(), entry->buffer()->size());
862     send(Messages::WebResourceLoader::DidFinishResourceLoad(networkLoadMetrics));
863 }
864
865 void NetworkResourceLoader::validateCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
866 {
867     ASSERT(!m_networkLoad);
868
869     // If the request is already conditional then the revalidation was not triggered by the disk cache
870     // and we should not overwrite the existing conditional headers.
871     ResourceRequest revalidationRequest = originalRequest();
872     if (!revalidationRequest.isConditional()) {
873         String eTag = entry->response().httpHeaderField(HTTPHeaderName::ETag);
874         String lastModified = entry->response().httpHeaderField(HTTPHeaderName::LastModified);
875         if (!eTag.isEmpty())
876             revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfNoneMatch, eTag);
877         if (!lastModified.isEmpty())
878             revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfModifiedSince, lastModified);
879     }
880
881     m_cacheEntryForValidation = WTFMove(entry);
882
883     startNetworkLoad(WTFMove(revalidationRequest), FirstLoad::Yes);
884 }
885
886 void NetworkResourceLoader::dispatchWillSendRequestForCacheEntry(ResourceRequest&& request, std::unique_ptr<NetworkCache::Entry>&& entry)
887 {
888     ASSERT(entry->redirectRequest());
889     ASSERT(!m_isWaitingContinueWillSendRequestForCachedRedirect);
890
891     LOG(NetworkCache, "(NetworkProcess) Executing cached redirect");
892
893     m_isWaitingContinueWillSendRequestForCachedRedirect = true;
894     willSendRedirectedRequest(WTFMove(request), ResourceRequest { *entry->redirectRequest() }, ResourceResponse { entry->response() });
895 }
896
897 IPC::Connection* NetworkResourceLoader::messageSenderConnection()
898 {
899     return &connectionToWebProcess().connection();
900 }
901
902 void NetworkResourceLoader::consumeSandboxExtensions()
903 {
904     ASSERT(!m_didConsumeSandboxExtensions);
905
906     for (auto& extension : m_parameters.requestBodySandboxExtensions)
907         extension->consume();
908
909     if (auto& extension = m_parameters.resourceSandboxExtension)
910         extension->consume();
911
912     for (auto& fileReference : m_fileReferences)
913         fileReference->prepareForFileAccess();
914
915     m_didConsumeSandboxExtensions = true;
916 }
917
918 void NetworkResourceLoader::invalidateSandboxExtensions()
919 {
920     if (m_didConsumeSandboxExtensions) {
921         for (auto& extension : m_parameters.requestBodySandboxExtensions)
922             extension->revoke();
923         if (auto& extension = m_parameters.resourceSandboxExtension)
924             extension->revoke();
925         for (auto& fileReference : m_fileReferences)
926             fileReference->revokeFileAccess();
927
928         m_didConsumeSandboxExtensions = false;
929     }
930
931     m_fileReferences.clear();
932 }
933
934 bool NetworkResourceLoader::isAlwaysOnLoggingAllowed() const
935 {
936     if (NetworkProcess::singleton().sessionIsControlledByAutomation(sessionID()))
937         return true;
938
939     return sessionID().isAlwaysOnLoggingAllowed();
940 }
941
942 bool NetworkResourceLoader::shouldCaptureExtraNetworkLoadMetrics() const
943 {
944     return m_connection->captureExtraNetworkLoadMetricsEnabled();
945 }
946
947 #if HAVE(CFNETWORK_STORAGE_PARTITIONING) && !RELEASE_LOG_DISABLED
948 bool NetworkResourceLoader::shouldLogCookieInformation()
949 {
950     return NetworkProcess::singleton().shouldLogCookieInformation();
951 }
952
953 static String escapeForJSON(String s)
954 {
955     return s.replace('\\', "\\\\").replace('"', "\\\"");
956 }
957
958 static String escapeIDForJSON(const std::optional<uint64_t>& value)
959 {
960     return value ? String::number(value.value()) : String("None");
961 };
962
963 void NetworkResourceLoader::logCookieInformation() const
964 {
965     ASSERT(shouldLogCookieInformation());
966
967     auto networkStorageSession = WebCore::NetworkStorageSession::storageSession(sessionID());
968     ASSERT(networkStorageSession);
969
970     logCookieInformation("NetworkResourceLoader", reinterpret_cast<const void*>(this), *networkStorageSession, originalRequest().firstPartyForCookies(), SameSiteInfo::create(originalRequest()), originalRequest().url(), originalRequest().httpReferrer(), frameID(), pageID(), identifier());
971 }
972
973 static void logBlockedCookieInformation(const String& label, const void* loggedObject, const WebCore::NetworkStorageSession& networkStorageSession, const WebCore::URL& firstParty, const SameSiteInfo& sameSiteInfo, const WebCore::URL& url, const String& referrer, std::optional<uint64_t> frameID, std::optional<uint64_t> pageID, std::optional<uint64_t> identifier)
974 {
975     ASSERT(NetworkResourceLoader::shouldLogCookieInformation());
976
977     auto escapedURL = escapeForJSON(url.string());
978     auto escapedFirstParty = escapeForJSON(firstParty.string());
979     auto escapedFrameID = escapeIDForJSON(frameID);
980     auto escapedPageID = escapeIDForJSON(pageID);
981     auto escapedIdentifier = escapeIDForJSON(identifier);
982     auto escapedReferrer = escapeForJSON(referrer);
983
984 #define LOCAL_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(networkStorageSession.sessionID().isAlwaysOnLoggingAllowed(), Network, "%p - %s::" fmt, loggedObject, label.utf8().data(), ##__VA_ARGS__)
985 #define LOCAL_LOG(str, ...) \
986     LOCAL_LOG_IF_ALLOWED("logCookieInformation: BLOCKED cookie access for pageID = %s, frameID = %s, resourceID = %s, firstParty = %s: " str, escapedPageID.utf8().data(), escapedFrameID.utf8().data(), escapedIdentifier.utf8().data(), escapedFirstParty.utf8().data(), ##__VA_ARGS__)
987
988     LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
989     LOCAL_LOG(R"(  "partition": "%{public}s",)", "BLOCKED");
990     LOCAL_LOG(R"(  "hasStorageAccess": %{public}s,)", "false");
991     LOCAL_LOG(R"(  "referer": "%{public}s",)", escapedReferrer.utf8().data());
992     LOCAL_LOG(R"(  "isSameSite": "%{public}s",)", sameSiteInfo.isSameSite ? "true" : "false");
993     LOCAL_LOG(R"(  "isTopSite": "%{public}s",)", sameSiteInfo.isTopSite ? "true" : "false");
994     LOCAL_LOG(R"(  "cookies": [])");
995     LOCAL_LOG(R"(  })");
996 #undef LOCAL_LOG
997 #undef LOCAL_LOG_IF_ALLOWED
998 }
999
1000 static void logCookieInformationInternal(const String& label, const void* loggedObject, const WebCore::NetworkStorageSession& networkStorageSession, const WebCore::URL& firstParty, const WebCore::SameSiteInfo& sameSiteInfo, const WebCore::URL& url, const String& referrer, std::optional<uint64_t> frameID, std::optional<uint64_t> pageID, std::optional<uint64_t> identifier)
1001 {
1002     ASSERT(NetworkResourceLoader::shouldLogCookieInformation());
1003
1004     Vector<WebCore::Cookie> cookies;
1005     if (!networkStorageSession.getRawCookies(firstParty, sameSiteInfo, url, frameID, pageID, cookies))
1006         return;
1007
1008     auto escapedURL = escapeForJSON(url.string());
1009     auto escapedPartition = escapeForJSON(emptyString());
1010     auto escapedReferrer = escapeForJSON(referrer);
1011     auto escapedFrameID = escapeIDForJSON(frameID);
1012     auto escapedPageID = escapeIDForJSON(pageID);
1013     auto escapedIdentifier = escapeIDForJSON(identifier);
1014     bool hasStorageAccess = (frameID && pageID) ? networkStorageSession.hasStorageAccess(url.string(), firstParty.string(), frameID.value(), pageID.value()) : false;
1015
1016 #define LOCAL_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(networkStorageSession.sessionID().isAlwaysOnLoggingAllowed(), Network, "%p - %s::" fmt, loggedObject, label.utf8().data(), ##__VA_ARGS__)
1017 #define LOCAL_LOG(str, ...) \
1018     LOCAL_LOG_IF_ALLOWED("logCookieInformation: pageID = %s, frameID = %s, resourceID = %s: " str, escapedPageID.utf8().data(), escapedFrameID.utf8().data(), escapedIdentifier.utf8().data(), ##__VA_ARGS__)
1019
1020     LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
1021     LOCAL_LOG(R"(  "partition": "%{public}s",)", escapedPartition.utf8().data());
1022     LOCAL_LOG(R"(  "hasStorageAccess": %{public}s,)", hasStorageAccess ? "true" : "false");
1023     LOCAL_LOG(R"(  "referer": "%{public}s",)", escapedReferrer.utf8().data());
1024     LOCAL_LOG(R"(  "isSameSite": "%{public}s",)", sameSiteInfo.isSameSite ? "true" : "false");
1025     LOCAL_LOG(R"(  "isTopSite": "%{public}s",)", sameSiteInfo.isTopSite ? "true" : "false");
1026     LOCAL_LOG(R"(  "cookies": [)");
1027
1028     auto size = cookies.size();
1029     decltype(size) count = 0;
1030     for (const auto& cookie : cookies) {
1031         const char* trailingComma = ",";
1032         if (++count == size)
1033             trailingComma = "";
1034
1035         auto escapedName = escapeForJSON(cookie.name);
1036         auto escapedValue = escapeForJSON(cookie.value);
1037         auto escapedDomain = escapeForJSON(cookie.domain);
1038         auto escapedPath = escapeForJSON(cookie.path);
1039         auto escapedComment = escapeForJSON(cookie.comment);
1040         auto escapedCommentURL = escapeForJSON(cookie.commentURL.string());
1041         // FIXME: Log Same-Site policy for each cookie. See <https://bugs.webkit.org/show_bug.cgi?id=184894>.
1042
1043         LOCAL_LOG(R"(  { "name": "%{public}s",)", escapedName.utf8().data());
1044         LOCAL_LOG(R"(    "value": "%{public}s",)", escapedValue.utf8().data());
1045         LOCAL_LOG(R"(    "domain": "%{public}s",)", escapedDomain.utf8().data());
1046         LOCAL_LOG(R"(    "path": "%{public}s",)", escapedPath.utf8().data());
1047         LOCAL_LOG(R"(    "created": %f,)", cookie.created);
1048         LOCAL_LOG(R"(    "expires": %f,)", cookie.expires);
1049         LOCAL_LOG(R"(    "httpOnly": %{public}s,)", cookie.httpOnly ? "true" : "false");
1050         LOCAL_LOG(R"(    "secure": %{public}s,)", cookie.secure ? "true" : "false");
1051         LOCAL_LOG(R"(    "session": %{public}s,)", cookie.session ? "true" : "false");
1052         LOCAL_LOG(R"(    "comment": "%{public}s",)", escapedComment.utf8().data());
1053         LOCAL_LOG(R"(    "commentURL": "%{public}s")", escapedCommentURL.utf8().data());
1054         LOCAL_LOG(R"(  }%{public}s)", trailingComma);
1055     }
1056     LOCAL_LOG(R"(]})");
1057 #undef LOCAL_LOG
1058 #undef LOCAL_LOG_IF_ALLOWED
1059 }
1060
1061 void NetworkResourceLoader::logCookieInformation(const String& label, const void* loggedObject, const NetworkStorageSession& networkStorageSession, const URL& firstParty, const SameSiteInfo& sameSiteInfo, const URL& url, const String& referrer, std::optional<uint64_t> frameID, std::optional<uint64_t> pageID, std::optional<uint64_t> identifier)
1062 {
1063     ASSERT(shouldLogCookieInformation());
1064
1065     if (networkStorageSession.shouldBlockCookies(firstParty, url, frameID, pageID))
1066         logBlockedCookieInformation(label, loggedObject, networkStorageSession, firstParty, sameSiteInfo, url, referrer, frameID, pageID, identifier);
1067     else
1068         logCookieInformationInternal(label, loggedObject, networkStorageSession, firstParty, sameSiteInfo, url, referrer, frameID, pageID, identifier);
1069 }
1070 #endif
1071
1072 void NetworkResourceLoader::addConsoleMessage(MessageSource messageSource, MessageLevel messageLevel, const String& message, unsigned long)
1073 {
1074     send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  messageSource, messageLevel, message, identifier() }, m_parameters.webPageID);
1075 }
1076
1077 void NetworkResourceLoader::sendCSPViolationReport(URL&& reportURL, Ref<FormData>&& report)
1078 {
1079     send(Messages::WebPage::SendCSPViolationReport { m_parameters.webFrameID, WTFMove(reportURL), IPC::FormDataReference { WTFMove(report) } }, m_parameters.webPageID);
1080 }
1081
1082 void NetworkResourceLoader::enqueueSecurityPolicyViolationEvent(WebCore::SecurityPolicyViolationEvent::Init&& eventInit)
1083 {
1084     send(Messages::WebPage::EnqueueSecurityPolicyViolationEvent { m_parameters.webFrameID, WTFMove(eventInit) }, m_parameters.webPageID);
1085 }
1086
1087 void NetworkResourceLoader::logSlowCacheRetrieveIfNeeded(const NetworkCache::Cache::RetrieveInfo& info)
1088 {
1089 #if RELEASE_LOG_DISABLED
1090     UNUSED_PARAM(info);
1091 #else
1092     if (!isAlwaysOnLoggingAllowed())
1093         return;
1094     auto duration = info.completionTime - info.startTime;
1095     if (duration < 1_s)
1096         return;
1097     RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Took %.0fms, priority %d", duration.milliseconds(), info.priority);
1098     if (info.wasSpeculativeLoad)
1099         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Was speculative load");
1100     if (!info.storageTimings.startTime)
1101         return;
1102     RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Storage retrieve time %.0fms", (info.storageTimings.completionTime - info.storageTimings.startTime).milliseconds());
1103     if (info.storageTimings.dispatchTime) {
1104         auto time = (info.storageTimings.dispatchTime - info.storageTimings.startTime).milliseconds();
1105         auto count = info.storageTimings.dispatchCountAtDispatch - info.storageTimings.dispatchCountAtStart;
1106         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Dispatch delay %.0fms, dispatched %lu resources first", time, count);
1107     }
1108     if (info.storageTimings.recordIOStartTime)
1109         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Record I/O time %.0fms", (info.storageTimings.recordIOEndTime - info.storageTimings.recordIOStartTime).milliseconds());
1110     if (info.storageTimings.blobIOStartTime)
1111         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Blob I/O time %.0fms", (info.storageTimings.blobIOEndTime - info.storageTimings.blobIOStartTime).milliseconds());
1112     if (info.storageTimings.synchronizationInProgressAtDispatch)
1113         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Synchronization was in progress");
1114     if (info.storageTimings.shrinkInProgressAtDispatch)
1115         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Shrink was in progress");
1116     if (info.storageTimings.wasCanceled)
1117         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Retrieve was canceled");
1118 #endif
1119 }
1120
1121 } // namespace WebKit
1122
1123 #undef RELEASE_LOG_IF_ALLOWED
1124 #undef RELEASE_LOG_ERROR_IF_ALLOWED