Move URL from WebCore to WTF
[WebKit-https.git] / Source / WebKit / NetworkProcess / NetworkResourceLoader.cpp
1 /*
2  * Copyright (C) 2012-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkResourceLoader.h"
28
29 #include "DataReference.h"
30 #include "FormDataReference.h"
31 #include "Logging.h"
32 #include "NetworkBlobRegistry.h"
33 #include "NetworkCache.h"
34 #include "NetworkConnectionToWebProcess.h"
35 #include "NetworkLoad.h"
36 #include "NetworkLoadChecker.h"
37 #include "NetworkProcess.h"
38 #include "NetworkProcessConnectionMessages.h"
39 #include "SessionTracker.h"
40 #include "SharedBufferDataReference.h"
41 #include "WebCoreArgumentCoders.h"
42 #include "WebErrors.h"
43 #include "WebPageMessages.h"
44 #include "WebResourceLoaderMessages.h"
45 #include "WebsiteDataStoreParameters.h"
46 #include <JavaScriptCore/ConsoleTypes.h>
47 #include <WebCore/BlobDataFileReference.h>
48 #include <WebCore/CertificateInfo.h>
49 #include <WebCore/ContentSecurityPolicy.h>
50 #include <WebCore/DiagnosticLoggingKeys.h>
51 #include <WebCore/HTTPHeaderNames.h>
52 #include <WebCore/HTTPParsers.h>
53 #include <WebCore/NetworkLoadMetrics.h>
54 #include <WebCore/NetworkStorageSession.h>
55 #include <WebCore/ProtectionSpace.h>
56 #include <WebCore/SameSiteInfo.h>
57 #include <WebCore/SecurityOrigin.h>
58 #include <WebCore/SharedBuffer.h>
59 #include <WebCore/SynchronousLoaderClient.h>
60 #include <wtf/RunLoop.h>
61
62 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
63 #include <WebCore/NetworkStorageSession.h>
64 #endif
65
66 #if USE(QUICK_LOOK)
67 #include <WebCore/PreviewLoader.h>
68 #endif
69
70 #define RELEASE_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__)
71 #define RELEASE_LOG_ERROR_IF_ALLOWED(fmt, ...) RELEASE_LOG_ERROR_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__)
72
73 namespace WebKit {
74 using namespace WebCore;
75
76 struct NetworkResourceLoader::SynchronousLoadData {
77     SynchronousLoadData(Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply&& reply)
78         : delayedReply(WTFMove(reply))
79     {
80         ASSERT(delayedReply);
81     }
82     ResourceRequest currentRequest;
83     Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply delayedReply;
84     ResourceResponse response;
85     ResourceError error;
86 };
87
88 static void sendReplyToSynchronousRequest(NetworkResourceLoader::SynchronousLoadData& data, const SharedBuffer* buffer)
89 {
90     ASSERT(data.delayedReply);
91     ASSERT(!data.response.isNull() || !data.error.isNull());
92
93     Vector<char> responseBuffer;
94     if (buffer && buffer->size())
95         responseBuffer.append(buffer->data(), buffer->size());
96
97     data.delayedReply(data.error, data.response, responseBuffer);
98     data.delayedReply = nullptr;
99 }
100
101 NetworkResourceLoader::NetworkResourceLoader(NetworkResourceLoadParameters&& parameters, NetworkConnectionToWebProcess& connection, Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply&& synchronousReply)
102     : m_parameters { WTFMove(parameters) }
103     , m_connection { connection }
104     , m_defersLoading { parameters.defersLoading }
105     , m_isAllowedToAskUserForCredentials { m_parameters.clientCredentialPolicy == ClientCredentialPolicy::MayAskClientForCredentials }
106     , m_bufferingTimer { *this, &NetworkResourceLoader::bufferingTimerFired }
107     , m_cache { sessionID().isEphemeral() ? nullptr : NetworkProcess::singleton().cache() }
108 {
109     ASSERT(RunLoop::isMain());
110     // FIXME: This is necessary because of the existence of EmptyFrameLoaderClient in WebCore.
111     //        Once bug 116233 is resolved, this ASSERT can just be "m_webPageID && m_webFrameID"
112     ASSERT((m_parameters.webPageID && m_parameters.webFrameID) || m_parameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials);
113
114     if (originalRequest().httpBody()) {
115         for (const auto& element : originalRequest().httpBody()->elements()) {
116             if (auto* blobData = WTF::get_if<FormDataElement::EncodedBlobData>(element.data))
117                 m_fileReferences.appendVector(NetworkBlobRegistry::singleton().filesInBlob(connection, blobData->url));
118         }
119     }
120
121     if (synchronousReply || parameters.shouldRestrictHTTPResponseAccess) {
122         m_networkLoadChecker = std::make_unique<NetworkLoadChecker>(FetchOptions { m_parameters.options }, m_parameters.sessionID, m_parameters.webPageID, m_parameters.webFrameID, HTTPHeaderMap { m_parameters.originalRequestHeaders }, URL { m_parameters.request.url() }, m_parameters.sourceOrigin.copyRef(), m_parameters.preflightPolicy, originalRequest().httpReferrer(), shouldCaptureExtraNetworkLoadMetrics());
123         if (m_parameters.cspResponseHeaders)
124             m_networkLoadChecker->setCSPResponseHeaders(ContentSecurityPolicyResponseHeaders { m_parameters.cspResponseHeaders.value() });
125 #if ENABLE(CONTENT_EXTENSIONS)
126         m_networkLoadChecker->setContentExtensionController(URL { m_parameters.mainDocumentURL }, m_parameters.userContentControllerIdentifier);
127 #endif
128     }
129     if (synchronousReply)
130         m_synchronousLoadData = std::make_unique<SynchronousLoadData>(WTFMove(synchronousReply));
131 }
132
133 NetworkResourceLoader::~NetworkResourceLoader()
134 {
135     ASSERT(RunLoop::isMain());
136     ASSERT(!m_networkLoad);
137     ASSERT(!isSynchronous() || !m_synchronousLoadData->delayedReply);
138     if (m_responseCompletionHandler)
139         m_responseCompletionHandler(PolicyAction::Ignore);
140 }
141
142 bool NetworkResourceLoader::canUseCache(const ResourceRequest& request) const
143 {
144     if (!m_cache)
145         return false;
146     ASSERT(!sessionID().isEphemeral());
147
148     if (!request.url().protocolIsInHTTPFamily())
149         return false;
150     if (originalRequest().cachePolicy() == WebCore::ResourceRequestCachePolicy::DoNotUseAnyCache)
151         return false;
152
153     return true;
154 }
155
156 bool NetworkResourceLoader::canUseCachedRedirect(const ResourceRequest& request) const
157 {
158     if (!canUseCache(request) || m_cacheEntryForMaxAgeCapValidation)
159         return false;
160     // Limit cached redirects to avoid cycles and other trouble.
161     // Networking layer follows over 30 redirects but caching that many seems unnecessary.
162     static const unsigned maximumCachedRedirectCount { 5 };
163     if (m_redirectCount > maximumCachedRedirectCount)
164         return false;
165
166     return true;
167 }
168
169 bool NetworkResourceLoader::isSynchronous() const
170 {
171     return !!m_synchronousLoadData;
172 }
173
174 void NetworkResourceLoader::start()
175 {
176     ASSERT(RunLoop::isMain());
177
178     m_networkActivityTracker = m_connection->startTrackingResourceLoad(m_parameters.webPageID, m_parameters.identifier, isMainResource(), sessionID());
179
180     if (m_defersLoading) {
181         RELEASE_LOG_IF_ALLOWED("start: Loading is deferred (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
182         return;
183     }
184
185     ASSERT(!m_wasStarted);
186     m_wasStarted = true;
187
188     if (m_networkLoadChecker) {
189         m_networkLoadChecker->check(ResourceRequest { originalRequest() }, this, [this] (auto&& result) {
190             if (!result.has_value()) {
191                 if (!result.error().isCancellation())
192                     this->didFailLoading(result.error());
193                 return;
194             }
195
196             auto currentRequest = result.value();
197             if (this->canUseCache(currentRequest)) {
198                 RELEASE_LOG_IF_ALLOWED("start: Checking cache for resource (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, this->isMainResource(), this->isSynchronous());
199                 this->retrieveCacheEntry(currentRequest);
200                 return;
201             }
202
203             this->startNetworkLoad(WTFMove(result.value()), FirstLoad::Yes);
204         });
205         return;
206     }
207     // FIXME: Remove that code path once m_networkLoadChecker is used for all network loads.
208     if (canUseCache(originalRequest())) {
209         RELEASE_LOG_IF_ALLOWED("start: Checking cache for resource (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
210         retrieveCacheEntry(originalRequest());
211         return;
212     }
213
214     startNetworkLoad(ResourceRequest { originalRequest() }, FirstLoad::Yes);
215 }
216
217 void NetworkResourceLoader::retrieveCacheEntry(const ResourceRequest& request)
218 {
219     ASSERT(canUseCache(request));
220
221     RefPtr<NetworkResourceLoader> loader(this);
222     m_cache->retrieve(request, { m_parameters.webPageID, m_parameters.webFrameID }, [this, loader = WTFMove(loader), request = ResourceRequest { request }](auto entry, auto info) mutable {
223         if (loader->hasOneRef()) {
224             // The loader has been aborted and is only held alive by this lambda.
225             return;
226         }
227
228         loader->logSlowCacheRetrieveIfNeeded(info);
229
230         if (!entry) {
231             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource not in cache (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
232             loader->startNetworkLoad(WTFMove(request), FirstLoad::Yes);
233             return;
234         }
235 #if ENABLE(RESOURCE_LOAD_STATISTICS)
236         if (entry->hasReachedPrevalentResourceAgeCap()) {
237             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource has reached prevalent resource age cap (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
238             m_cacheEntryForMaxAgeCapValidation = WTFMove(entry);
239             ResourceRequest revalidationRequest = originalRequest();
240             loader->startNetworkLoad(WTFMove(revalidationRequest), FirstLoad::Yes);
241             return;
242         }
243 #endif
244         if (entry->redirectRequest()) {
245             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Handling redirect (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
246             loader->dispatchWillSendRequestForCacheEntry(WTFMove(request), WTFMove(entry));
247             return;
248         }
249         if (loader->m_parameters.needsCertificateInfo && !entry->response().certificateInfo()) {
250             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource does not have required certificate (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
251             loader->startNetworkLoad(WTFMove(request), FirstLoad::Yes);
252             return;
253         }
254         if (entry->needsValidation() || request.cachePolicy() == WebCore::ResourceRequestCachePolicy::RefreshAnyCacheData) {
255             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Validating cache entry (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
256             loader->validateCacheEntry(WTFMove(entry));
257             return;
258         }
259         RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Retrieved resource from cache (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
260         loader->didRetrieveCacheEntry(WTFMove(entry));
261     });
262 }
263
264 void NetworkResourceLoader::startNetworkLoad(ResourceRequest&& request, FirstLoad load)
265 {
266     if (load == FirstLoad::Yes) {
267         RELEASE_LOG_IF_ALLOWED("startNetworkLoad: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
268
269         consumeSandboxExtensions();
270
271         if (isSynchronous() || m_parameters.maximumBufferingTime > 0_s)
272             m_bufferedData = SharedBuffer::create();
273
274         if (canUseCache(request))
275             m_bufferedDataForCache = SharedBuffer::create();
276     }
277
278     NetworkLoadParameters parameters = m_parameters;
279     parameters.defersLoading = m_defersLoading;
280     parameters.networkActivityTracker = m_networkActivityTracker;
281     if (m_networkLoadChecker)
282         parameters.storedCredentialsPolicy = m_networkLoadChecker->storedCredentialsPolicy();
283
284     if (request.url().protocolIsBlob())
285         parameters.blobFileReferences = NetworkBlobRegistry::singleton().filesInBlob(m_connection, originalRequest().url());
286
287     auto* networkSession = SessionTracker::networkSession(parameters.sessionID);
288     if (!networkSession && parameters.sessionID.isEphemeral()) {
289         NetworkProcess::singleton().addWebsiteDataStore(WebsiteDataStoreParameters::privateSessionParameters(parameters.sessionID));
290         networkSession = SessionTracker::networkSession(parameters.sessionID);
291     }
292     if (!networkSession) {
293         WTFLogAlways("Attempted to create a NetworkLoad with a session (id=%" PRIu64 ") that does not exist.", parameters.sessionID.sessionID());
294         RELEASE_LOG_ERROR_IF_ALLOWED("startNetworkLoad: Attempted to create a NetworkLoad with a session that does not exist (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", sessionID=%" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, parameters.sessionID.sessionID());
295         NetworkProcess::singleton().logDiagnosticMessage(m_parameters.webPageID, WebCore::DiagnosticLoggingKeys::internalErrorKey(), WebCore::DiagnosticLoggingKeys::invalidSessionIDKey(), WebCore::ShouldSample::No);
296         didFailLoading(internalError(request.url()));
297         return;
298     }
299
300     parameters.request = WTFMove(request);
301     m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters), *networkSession);
302
303     RELEASE_LOG_IF_ALLOWED("startNetworkLoad: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", description = %{public}s)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, m_networkLoad->description().utf8().data());
304
305     if (m_defersLoading) {
306         RELEASE_LOG_IF_ALLOWED("startNetworkLoad: Created, but deferred (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")",
307             m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
308     }
309 }
310
311 void NetworkResourceLoader::setDefersLoading(bool defers)
312 {
313     if (m_defersLoading == defers)
314         return;
315     m_defersLoading = defers;
316
317     if (defers)
318         RELEASE_LOG_IF_ALLOWED("setDefersLoading: Deferring resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
319     else
320         RELEASE_LOG_IF_ALLOWED("setDefersLoading: Resuming deferred resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
321
322     if (m_networkLoad) {
323         m_networkLoad->setDefersLoading(defers);
324         return;
325     }
326
327     if (!m_defersLoading && !m_wasStarted)
328         start();
329     else
330         RELEASE_LOG_IF_ALLOWED("setDefersLoading: defers = %d, but nothing to do (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_defersLoading, m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
331 }
332
333 void NetworkResourceLoader::cleanup(LoadResult result)
334 {
335     ASSERT(RunLoop::isMain());
336
337     m_connection->stopTrackingResourceLoad(m_parameters.identifier,
338         result == LoadResult::Success ? NetworkActivityTracker::CompletionCode::Success :
339         result == LoadResult::Failure ? NetworkActivityTracker::CompletionCode::Failure :
340         NetworkActivityTracker::CompletionCode::None);
341
342     m_bufferingTimer.stop();
343
344     invalidateSandboxExtensions();
345
346     m_networkLoad = nullptr;
347
348     // This will cause NetworkResourceLoader to be destroyed and therefore we do it last.
349     m_connection->didCleanupResourceLoader(*this);
350 }
351
352 void NetworkResourceLoader::convertToDownload(DownloadID downloadID, const ResourceRequest& request, const ResourceResponse& response)
353 {
354     // This can happen if the resource came from the disk cache.
355     if (!m_networkLoad) {
356         NetworkProcess::singleton().downloadManager().startDownload(m_connection.ptr(), m_parameters.sessionID, downloadID, request);
357         abort();
358         return;
359     }
360
361     ASSERT(m_responseCompletionHandler);
362     NetworkProcess::singleton().downloadManager().convertNetworkLoadToDownload(downloadID, std::exchange(m_networkLoad, nullptr), WTFMove(m_responseCompletionHandler), WTFMove(m_fileReferences), request, response);
363 }
364
365 void NetworkResourceLoader::abort()
366 {
367     ASSERT(RunLoop::isMain());
368
369     RELEASE_LOG_IF_ALLOWED("abort: Canceling resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")",
370         m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
371
372     if (m_networkLoad) {
373         if (canUseCache(m_networkLoad->currentRequest())) {
374             // We might already have used data from this incomplete load. Ensure older versions don't remain in the cache after cancel.
375             if (!m_response.isNull())
376                 m_cache->remove(m_networkLoad->currentRequest());
377         }
378         m_networkLoad->cancel();
379     }
380
381     cleanup(LoadResult::Cancel);
382 }
383
384 bool NetworkResourceLoader::shouldInterruptLoadForXFrameOptions(const String& xFrameOptions, const URL& url)
385 {
386     if (isMainFrameLoad())
387         return false;
388
389     switch (parseXFrameOptionsHeader(xFrameOptions)) {
390     case XFrameOptionsNone:
391     case XFrameOptionsAllowAll:
392         return false;
393     case XFrameOptionsDeny:
394         return true;
395     case XFrameOptionsSameOrigin: {
396         auto origin = SecurityOrigin::create(url);
397         auto topFrameOrigin = m_parameters.frameAncestorOrigins.last();
398         if (!origin->isSameSchemeHostPort(*topFrameOrigin))
399             return true;
400         for (auto& ancestorOrigin : m_parameters.frameAncestorOrigins) {
401             if (!origin->isSameSchemeHostPort(*ancestorOrigin))
402                 return true;
403         }
404         return false;
405     }
406     case XFrameOptionsConflict: {
407         String errorMessage = "Multiple 'X-Frame-Options' headers with conflicting values ('" + xFrameOptions + "') encountered when loading '" + url.stringCenterEllipsizedToLength() + "'. Falling back to 'DENY'.";
408         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::JS, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
409         return true;
410     }
411     case XFrameOptionsInvalid: {
412         String errorMessage = "Invalid 'X-Frame-Options' header encountered when loading '" + url.stringCenterEllipsizedToLength() + "': '" + xFrameOptions + "' is not a recognized directive. The header will be ignored.";
413         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::JS, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
414         return false;
415     }
416     }
417     ASSERT_NOT_REACHED();
418     return false;
419 }
420
421 bool NetworkResourceLoader::shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(const ResourceResponse& response)
422 {
423     ASSERT(isMainResource());
424
425 #if USE(QUICK_LOOK)
426     if (PreviewLoader::shouldCreateForMIMEType(response.mimeType()))
427         return false;
428 #endif
429
430     auto url = response.url();
431     ContentSecurityPolicy contentSecurityPolicy { URL { url }, this };
432     contentSecurityPolicy.didReceiveHeaders(ContentSecurityPolicyResponseHeaders { response }, originalRequest().httpReferrer());
433     if (!contentSecurityPolicy.allowFrameAncestors(m_parameters.frameAncestorOrigins, url))
434         return true;
435     String xFrameOptions = m_response.httpHeaderField(HTTPHeaderName::XFrameOptions);
436     if (!xFrameOptions.isNull() && shouldInterruptLoadForXFrameOptions(xFrameOptions, response.url())) {
437         String errorMessage = "Refused to display '" + response.url().stringCenterEllipsizedToLength() + "' in a frame because it set 'X-Frame-Options' to '" + xFrameOptions + "'.";
438         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::Security, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
439         return true;
440     }
441     return false;
442 }
443
444 void NetworkResourceLoader::didReceiveResponse(ResourceResponse&& receivedResponse, ResponseCompletionHandler&& completionHandler)
445 {
446     RELEASE_LOG_IF_ALLOWED("didReceiveResponse: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", httpStatusCode = %d, length = %" PRId64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, receivedResponse.httpStatusCode(), receivedResponse.expectedContentLength());
447
448     m_response = WTFMove(receivedResponse);
449
450     if (shouldCaptureExtraNetworkLoadMetrics() && m_networkLoadChecker) {
451         auto information = m_networkLoadChecker->takeNetworkLoadInformation();
452         information.response = m_response;
453         m_connection->addNetworkLoadInformation(identifier(), WTFMove(information));
454     }
455
456     // For multipart/x-mixed-replace didReceiveResponseAsync gets called multiple times and buffering would require special handling.
457     if (!isSynchronous() && m_response.isMultipart())
458         m_bufferedData = nullptr;
459
460     if (m_response.isMultipart())
461         m_bufferedDataForCache = nullptr;
462
463     if (m_cacheEntryForValidation) {
464         bool validationSucceeded = m_response.httpStatusCode() == 304; // 304 Not Modified
465         if (validationSucceeded) {
466             m_cacheEntryForValidation = m_cache->update(originalRequest(), { m_parameters.webPageID, m_parameters.webFrameID }, *m_cacheEntryForValidation, m_response);
467             // If the request was conditional then this revalidation was not triggered by the network cache and we pass the 304 response to WebCore.
468             if (originalRequest().isConditional())
469                 m_cacheEntryForValidation = nullptr;
470         } else
471             m_cacheEntryForValidation = nullptr;
472     }
473     if (m_cacheEntryForValidation)
474         return completionHandler(PolicyAction::Use);
475
476     if (isMainResource() && shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(m_response)) {
477         auto response = sanitizeResponseIfPossible(ResourceResponse { m_response }, ResourceResponse::SanitizationType::CrossOriginSafe);
478         send(Messages::WebResourceLoader::StopLoadingAfterXFrameOptionsOrContentSecurityPolicyDenied { response });
479         return completionHandler(PolicyAction::Ignore);
480     }
481
482     if (m_networkLoadChecker) {
483         auto error = m_networkLoadChecker->validateResponse(m_response);
484         if (!error.isNull()) {
485             RunLoop::main().dispatch([protectedThis = makeRef(*this), error = WTFMove(error)] {
486                 if (protectedThis->m_networkLoad)
487                     protectedThis->didFailLoading(error);
488             });
489             return completionHandler(PolicyAction::Ignore);
490         }
491     }
492
493     auto response = sanitizeResponseIfPossible(ResourceResponse { m_response }, ResourceResponse::SanitizationType::CrossOriginSafe);
494     if (isSynchronous()) {
495         m_synchronousLoadData->response = WTFMove(response);
496         return completionHandler(PolicyAction::Use);
497     }
498
499     // We wait to receive message NetworkResourceLoader::ContinueDidReceiveResponse before continuing a load for
500     // a main resource because the embedding client must decide whether to allow the load.
501     bool willWaitForContinueDidReceiveResponse = isMainResource();
502     send(Messages::WebResourceLoader::DidReceiveResponse { response, willWaitForContinueDidReceiveResponse });
503     if (willWaitForContinueDidReceiveResponse)
504         m_responseCompletionHandler = WTFMove(completionHandler);
505     else
506         completionHandler(PolicyAction::Use);
507 }
508
509 void NetworkResourceLoader::didReceiveBuffer(Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength)
510 {
511     if (!m_numBytesReceived) {
512         RELEASE_LOG_IF_ALLOWED("didReceiveBuffer: Started receiving data (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
513     }
514     m_numBytesReceived += buffer->size();
515
516     ASSERT(!m_cacheEntryForValidation);
517
518     if (m_bufferedDataForCache) {
519         // Prevent memory growth in case of streaming data.
520         const size_t maximumCacheBufferSize = 10 * 1024 * 1024;
521         if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize)
522             m_bufferedDataForCache->append(buffer.get());
523         else
524             m_bufferedDataForCache = nullptr;
525     }
526     // FIXME: At least on OS X Yosemite we always get -1 from the resource handle.
527     unsigned encodedDataLength = reportedEncodedDataLength >= 0 ? reportedEncodedDataLength : buffer->size();
528
529     m_bytesReceived += buffer->size();
530     if (m_bufferedData) {
531         m_bufferedData->append(buffer.get());
532         m_bufferedDataEncodedDataLength += encodedDataLength;
533         startBufferingTimerIfNeeded();
534         return;
535     }
536     sendBuffer(buffer, encodedDataLength);
537 }
538
539 void NetworkResourceLoader::didFinishLoading(const NetworkLoadMetrics& networkLoadMetrics)
540 {
541     RELEASE_LOG_IF_ALLOWED("didFinishLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", length = %zd)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, m_numBytesReceived);
542
543     if (shouldCaptureExtraNetworkLoadMetrics())
544         m_connection->addNetworkLoadInformationMetrics(identifier(), networkLoadMetrics);
545
546     if (m_cacheEntryForValidation) {
547         // 304 Not Modified
548         ASSERT(m_response.httpStatusCode() == 304);
549         LOG(NetworkCache, "(NetworkProcess) revalidated");
550         didRetrieveCacheEntry(WTFMove(m_cacheEntryForValidation));
551         return;
552     }
553
554 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
555     if (shouldLogCookieInformation(sessionID()))
556         logCookieInformation();
557 #endif
558
559     if (isSynchronous())
560         sendReplyToSynchronousRequest(*m_synchronousLoadData, m_bufferedData.get());
561     else {
562         if (m_bufferedData && !m_bufferedData->isEmpty()) {
563             // FIXME: Pass a real value or remove the encoded data size feature.
564             sendBuffer(*m_bufferedData, -1);
565         }
566         send(Messages::WebResourceLoader::DidFinishResourceLoad(networkLoadMetrics));
567     }
568
569     tryStoreAsCacheEntry();
570
571     cleanup(LoadResult::Success);
572 }
573
574 void NetworkResourceLoader::didFailLoading(const ResourceError& error)
575 {
576     RELEASE_LOG_IF_ALLOWED("didFailLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isTimeout = %d, isCancellation = %d, isAccessControl = %d, errCode = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, error.isTimeout(), error.isCancellation(), error.isAccessControl(), error.errorCode());
577
578     if (shouldCaptureExtraNetworkLoadMetrics())
579         m_connection->removeNetworkLoadInformation(identifier());
580
581     ASSERT(!error.isNull());
582
583     m_cacheEntryForValidation = nullptr;
584
585     if (isSynchronous()) {
586         m_synchronousLoadData->error = error;
587         sendReplyToSynchronousRequest(*m_synchronousLoadData, nullptr);
588     } else if (auto* connection = messageSenderConnection())
589         connection->send(Messages::WebResourceLoader::DidFailResourceLoad(error), messageSenderDestinationID());
590
591     cleanup(LoadResult::Failure);
592 }
593
594 void NetworkResourceLoader::didBlockAuthenticationChallenge()
595 {
596     send(Messages::WebResourceLoader::DidBlockAuthenticationChallenge());
597 }
598
599 std::optional<Seconds> NetworkResourceLoader::validateCacheEntryForMaxAgeCapValidation(const ResourceRequest& request, const ResourceRequest& redirectRequest, const ResourceResponse& redirectResponse)
600 {
601 #if ENABLE(RESOURCE_LOAD_STATISTICS)
602     bool existingCacheEntryMatchesNewResponse = false;
603     if (m_cacheEntryForMaxAgeCapValidation) {
604         ASSERT(redirectResponse.source() == ResourceResponse::Source::Network);
605         ASSERT(redirectResponse.isRedirection());
606         if (redirectResponse.httpHeaderField(WebCore::HTTPHeaderName::Location) == m_cacheEntryForMaxAgeCapValidation->response().httpHeaderField(WebCore::HTTPHeaderName::Location))
607             existingCacheEntryMatchesNewResponse = true;
608
609         m_cache->remove(m_cacheEntryForMaxAgeCapValidation->key());
610         m_cacheEntryForMaxAgeCapValidation = nullptr;
611     }
612     
613     if (!existingCacheEntryMatchesNewResponse) {
614         if (auto networkStorageSession = WebCore::NetworkStorageSession::storageSession(sessionID()))
615             return networkStorageSession->maxAgeCacheCap(request);
616     }
617 #endif
618     return std::nullopt;
619 }
620
621 void NetworkResourceLoader::willSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse)
622 {
623     ++m_redirectCount;
624
625     auto maxAgeCap = validateCacheEntryForMaxAgeCapValidation(request, redirectRequest, redirectResponse);
626     if (redirectResponse.source() == ResourceResponse::Source::Network && canUseCachedRedirect(request))
627         m_cache->storeRedirect(request, redirectResponse, redirectRequest, maxAgeCap);
628
629     if (m_networkLoadChecker) {
630         m_networkLoadChecker->storeRedirectionIfNeeded(request, redirectResponse);
631         m_networkLoadChecker->checkRedirection(WTFMove(request), WTFMove(redirectRequest), WTFMove(redirectResponse), this, [protectedThis = makeRef(*this), this, storedCredentialsPolicy = m_networkLoadChecker->storedCredentialsPolicy()](auto&& result) mutable {
632             if (!result.has_value()) {
633                 if (result.error().isCancellation())
634                     return;
635
636                 this->didFailLoading(result.error());
637                 return;
638             }
639
640             if (m_parameters.options.redirect == FetchOptions::Redirect::Manual) {
641                 this->didFinishWithRedirectResponse(WTFMove(result->redirectResponse));
642                 return;
643             }
644
645             if (this->isSynchronous()) {
646                 if (storedCredentialsPolicy != m_networkLoadChecker->storedCredentialsPolicy()) {
647                     // We need to restart the load to update the session according the new credential policy.
648                     this->restartNetworkLoad(WTFMove(result->redirectRequest));
649                     return;
650                 }
651
652                 // We do not support prompting for credentials for synchronous loads. If we ever change this policy then
653                 // we need to take care to prompt if and only if request and redirectRequest are not mixed content.
654                 this->continueWillSendRequest(WTFMove(result->redirectRequest), false);
655                 return;
656             }
657
658             m_shouldRestartLoad = storedCredentialsPolicy != m_networkLoadChecker->storedCredentialsPolicy();
659             this->continueWillSendRedirectedRequest(WTFMove(result->request), WTFMove(result->redirectRequest), WTFMove(result->redirectResponse));
660         });
661         return;
662     }
663     continueWillSendRedirectedRequest(WTFMove(request), WTFMove(redirectRequest), WTFMove(redirectResponse));
664 }
665
666 void NetworkResourceLoader::continueWillSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse)
667 {
668     ASSERT(!isSynchronous());
669
670     send(Messages::WebResourceLoader::WillSendRequest(redirectRequest, sanitizeResponseIfPossible(WTFMove(redirectResponse), ResourceResponse::SanitizationType::Redirection)));
671 }
672
673 void NetworkResourceLoader::didFinishWithRedirectResponse(ResourceResponse&& redirectResponse)
674 {
675     redirectResponse.setType(ResourceResponse::Type::Opaqueredirect);
676     didReceiveResponse(WTFMove(redirectResponse), [] (auto) { });
677
678     WebCore::NetworkLoadMetrics networkLoadMetrics;
679     networkLoadMetrics.markComplete();
680     networkLoadMetrics.responseBodyBytesReceived = 0;
681     networkLoadMetrics.responseBodyDecodedSize = 0;
682     send(Messages::WebResourceLoader::DidFinishResourceLoad { networkLoadMetrics });
683
684     cleanup(LoadResult::Success);
685 }
686
687 ResourceResponse NetworkResourceLoader::sanitizeResponseIfPossible(ResourceResponse&& response, ResourceResponse::SanitizationType type)
688 {
689     if (m_parameters.shouldRestrictHTTPResponseAccess)
690         response.sanitizeHTTPHeaderFields(type);
691
692     return WTFMove(response);
693 }
694
695 void NetworkResourceLoader::restartNetworkLoad(WebCore::ResourceRequest&& newRequest)
696 {
697     if (m_networkLoad)
698         m_networkLoad->cancel();
699     if (m_networkLoadChecker)
700         m_networkLoadChecker->prepareRedirectedRequest(newRequest);
701
702     startNetworkLoad(WTFMove(newRequest), FirstLoad::No);
703 }
704
705 void NetworkResourceLoader::continueWillSendRequest(ResourceRequest&& newRequest, bool isAllowedToAskUserForCredentials)
706 {
707     if (m_shouldRestartLoad) {
708         m_shouldRestartLoad = false;
709         restartNetworkLoad(WTFMove(newRequest));
710         return;
711     }
712
713     if (m_networkLoadChecker) {
714         // FIXME: We should be doing this check when receiving the redirection and not allow about protocol as per fetch spec.
715         if (!newRequest.url().protocolIsInHTTPFamily() && !newRequest.url().protocolIsAbout() && m_redirectCount) {
716             didFailLoading(ResourceError { String { }, 0, newRequest.url(), "Redirection to URL with a scheme that is not HTTP(S)"_s, ResourceError::Type::AccessControl });
717             return;
718         }
719     }
720
721     RELEASE_LOG_IF_ALLOWED("continueWillSendRequest: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
722
723     if (m_networkLoadChecker)
724         m_networkLoadChecker->prepareRedirectedRequest(newRequest);
725
726     m_isAllowedToAskUserForCredentials = isAllowedToAskUserForCredentials;
727
728     // If there is a match in the network cache, we need to reuse the original cache policy and partition.
729     newRequest.setCachePolicy(originalRequest().cachePolicy());
730     newRequest.setCachePartition(originalRequest().cachePartition());
731
732     if (m_isWaitingContinueWillSendRequestForCachedRedirect) {
733         m_isWaitingContinueWillSendRequestForCachedRedirect = false;
734
735         LOG(NetworkCache, "(NetworkProcess) Retrieving cached redirect");
736
737         if (canUseCachedRedirect(newRequest))
738             retrieveCacheEntry(newRequest);
739         else
740             startNetworkLoad(WTFMove(newRequest), FirstLoad::Yes);
741
742         return;
743     }
744
745     if (m_networkLoad)
746         m_networkLoad->continueWillSendRequest(WTFMove(newRequest));
747 }
748
749 void NetworkResourceLoader::continueDidReceiveResponse()
750 {
751     if (m_cacheEntryWaitingForContinueDidReceiveResponse) {
752         continueProcessingCachedEntryAfterDidReceiveResponse(WTFMove(m_cacheEntryWaitingForContinueDidReceiveResponse));
753         return;
754     }
755
756     if (m_responseCompletionHandler)
757         m_responseCompletionHandler(PolicyAction::Use);
758 }
759
760 void NetworkResourceLoader::didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent)
761 {
762     if (!isSynchronous())
763         send(Messages::WebResourceLoader::DidSendData(bytesSent, totalBytesToBeSent));
764 }
765
766 void NetworkResourceLoader::startBufferingTimerIfNeeded()
767 {
768     if (isSynchronous())
769         return;
770     if (m_bufferingTimer.isActive())
771         return;
772     m_bufferingTimer.startOneShot(m_parameters.maximumBufferingTime);
773 }
774
775 void NetworkResourceLoader::bufferingTimerFired()
776 {
777     ASSERT(m_bufferedData);
778     ASSERT(m_networkLoad);
779
780     if (m_bufferedData->isEmpty())
781         return;
782
783     send(Messages::WebResourceLoader::DidReceiveData({ *m_bufferedData }, m_bufferedDataEncodedDataLength));
784
785     m_bufferedData = SharedBuffer::create();
786     m_bufferedDataEncodedDataLength = 0;
787 }
788
789 void NetworkResourceLoader::sendBuffer(SharedBuffer& buffer, size_t encodedDataLength)
790 {
791     ASSERT(!isSynchronous());
792
793     send(Messages::WebResourceLoader::DidReceiveData({ buffer }, encodedDataLength));
794 }
795
796 void NetworkResourceLoader::tryStoreAsCacheEntry()
797 {
798     if (!canUseCache(m_networkLoad->currentRequest()))
799         return;
800     if (!m_bufferedDataForCache)
801         return;
802
803     m_cache->store(m_networkLoad->currentRequest(), m_response, WTFMove(m_bufferedDataForCache), [loader = makeRef(*this)](auto& mappedBody) mutable {
804 #if ENABLE(SHAREABLE_RESOURCE)
805         if (mappedBody.shareableResourceHandle.isNull())
806             return;
807         LOG(NetworkCache, "(NetworkProcess) sending DidCacheResource");
808         loader->send(Messages::NetworkProcessConnection::DidCacheResource(loader->originalRequest(), mappedBody.shareableResourceHandle, loader->sessionID()));
809 #endif
810     });
811 }
812
813 void NetworkResourceLoader::didRetrieveCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
814 {
815     auto response = entry->response();
816
817     if (isMainResource() && shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(response)) {
818         response = sanitizeResponseIfPossible(WTFMove(response), ResourceResponse::SanitizationType::CrossOriginSafe);
819         send(Messages::WebResourceLoader::StopLoadingAfterXFrameOptionsOrContentSecurityPolicyDenied { response });
820         return;
821     }
822     if (m_networkLoadChecker) {
823         auto error = m_networkLoadChecker->validateResponse(response);
824         if (!error.isNull()) {
825             didFailLoading(error);
826             return;
827         }
828     }
829
830     response = sanitizeResponseIfPossible(WTFMove(response), ResourceResponse::SanitizationType::CrossOriginSafe);
831     if (isSynchronous()) {
832         m_synchronousLoadData->response = WTFMove(response);
833         sendReplyToSynchronousRequest(*m_synchronousLoadData, entry->buffer());
834         cleanup(LoadResult::Success);
835         return;
836     }
837
838     bool needsContinueDidReceiveResponseMessage = isMainResource();
839     send(Messages::WebResourceLoader::DidReceiveResponse { response, needsContinueDidReceiveResponseMessage });
840
841     if (needsContinueDidReceiveResponseMessage)
842         m_cacheEntryWaitingForContinueDidReceiveResponse = WTFMove(entry);
843     else
844         continueProcessingCachedEntryAfterDidReceiveResponse(WTFMove(entry));
845 }
846
847 void NetworkResourceLoader::continueProcessingCachedEntryAfterDidReceiveResponse(std::unique_ptr<NetworkCache::Entry> entry)
848 {
849     if (entry->sourceStorageRecord().bodyHash && !m_parameters.derivedCachedDataTypesToRetrieve.isEmpty()) {
850         auto bodyHash = *entry->sourceStorageRecord().bodyHash;
851         auto* entryPtr = entry.release();
852         auto retrieveCount = m_parameters.derivedCachedDataTypesToRetrieve.size();
853
854         for (auto& type : m_parameters.derivedCachedDataTypesToRetrieve) {
855             NetworkCache::DataKey key { originalRequest().cachePartition(), type, bodyHash };
856             m_cache->retrieveData(key, [loader = makeRef(*this), entryPtr, type, retrieveCount] (const uint8_t* data, size_t size) mutable {
857                 loader->m_retrievedDerivedDataCount++;
858                 bool retrievedAll = loader->m_retrievedDerivedDataCount == retrieveCount;
859                 std::unique_ptr<NetworkCache::Entry> entry(retrievedAll ? entryPtr : nullptr);
860                 if (loader->hasOneRef())
861                     return;
862                 if (data) {
863                     IPC::DataReference dataReference(data, size);
864                     loader->send(Messages::WebResourceLoader::DidRetrieveDerivedData(type, dataReference));
865                 }
866                 if (retrievedAll) {
867                     loader->sendResultForCacheEntry(WTFMove(entry));
868                     loader->cleanup(LoadResult::Success);
869                 }
870             });
871         }
872         return;
873     }
874
875     sendResultForCacheEntry(WTFMove(entry));
876
877     cleanup(LoadResult::Success);
878 }
879
880 void NetworkResourceLoader::sendResultForCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
881 {
882 #if ENABLE(SHAREABLE_RESOURCE)
883     if (!entry->shareableResourceHandle().isNull()) {
884         send(Messages::WebResourceLoader::DidReceiveResource(entry->shareableResourceHandle()));
885         return;
886     }
887 #endif
888
889 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
890     if (shouldLogCookieInformation(sessionID()))
891         logCookieInformation();
892 #endif
893
894     WebCore::NetworkLoadMetrics networkLoadMetrics;
895     networkLoadMetrics.markComplete();
896     networkLoadMetrics.requestHeaderBytesSent = 0;
897     networkLoadMetrics.requestBodyBytesSent = 0;
898     networkLoadMetrics.responseHeaderBytesReceived = 0;
899     networkLoadMetrics.responseBodyBytesReceived = 0;
900     networkLoadMetrics.responseBodyDecodedSize = 0;
901
902     sendBuffer(*entry->buffer(), entry->buffer()->size());
903     send(Messages::WebResourceLoader::DidFinishResourceLoad(networkLoadMetrics));
904 }
905
906 void NetworkResourceLoader::validateCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
907 {
908     ASSERT(!m_networkLoad);
909
910     // If the request is already conditional then the revalidation was not triggered by the disk cache
911     // and we should not overwrite the existing conditional headers.
912     ResourceRequest revalidationRequest = originalRequest();
913     if (!revalidationRequest.isConditional()) {
914         String eTag = entry->response().httpHeaderField(HTTPHeaderName::ETag);
915         String lastModified = entry->response().httpHeaderField(HTTPHeaderName::LastModified);
916         if (!eTag.isEmpty())
917             revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfNoneMatch, eTag);
918         if (!lastModified.isEmpty())
919             revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfModifiedSince, lastModified);
920     }
921
922     m_cacheEntryForValidation = WTFMove(entry);
923
924     startNetworkLoad(WTFMove(revalidationRequest), FirstLoad::Yes);
925 }
926
927 void NetworkResourceLoader::dispatchWillSendRequestForCacheEntry(ResourceRequest&& request, std::unique_ptr<NetworkCache::Entry>&& entry)
928 {
929     ASSERT(entry->redirectRequest());
930     ASSERT(!m_isWaitingContinueWillSendRequestForCachedRedirect);
931
932     LOG(NetworkCache, "(NetworkProcess) Executing cached redirect");
933
934     m_isWaitingContinueWillSendRequestForCachedRedirect = true;
935     willSendRedirectedRequest(WTFMove(request), ResourceRequest { *entry->redirectRequest() }, ResourceResponse { entry->response() });
936 }
937
938 IPC::Connection* NetworkResourceLoader::messageSenderConnection()
939 {
940     return &connectionToWebProcess().connection();
941 }
942
943 void NetworkResourceLoader::consumeSandboxExtensions()
944 {
945     ASSERT(!m_didConsumeSandboxExtensions);
946
947     for (auto& extension : m_parameters.requestBodySandboxExtensions)
948         extension->consume();
949
950     if (auto& extension = m_parameters.resourceSandboxExtension)
951         extension->consume();
952
953     for (auto& fileReference : m_fileReferences)
954         fileReference->prepareForFileAccess();
955
956     m_didConsumeSandboxExtensions = true;
957 }
958
959 void NetworkResourceLoader::invalidateSandboxExtensions()
960 {
961     if (m_didConsumeSandboxExtensions) {
962         for (auto& extension : m_parameters.requestBodySandboxExtensions)
963             extension->revoke();
964         if (auto& extension = m_parameters.resourceSandboxExtension)
965             extension->revoke();
966         for (auto& fileReference : m_fileReferences)
967             fileReference->revokeFileAccess();
968
969         m_didConsumeSandboxExtensions = false;
970     }
971
972     m_fileReferences.clear();
973 }
974
975 bool NetworkResourceLoader::isAlwaysOnLoggingAllowed() const
976 {
977     if (NetworkProcess::singleton().sessionIsControlledByAutomation(sessionID()))
978         return true;
979
980     return sessionID().isAlwaysOnLoggingAllowed();
981 }
982
983 bool NetworkResourceLoader::shouldCaptureExtraNetworkLoadMetrics() const
984 {
985     return m_connection->captureExtraNetworkLoadMetricsEnabled();
986 }
987
988 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
989 bool NetworkResourceLoader::shouldLogCookieInformation(const PAL::SessionID& sessionID)
990 {
991     if (auto session = SessionTracker::networkSession(sessionID))
992         return session->shouldLogCookieInformation();
993     return false;
994 }
995
996 static String escapeForJSON(String s)
997 {
998     return s.replace('\\', "\\\\").replace('"', "\\\"");
999 }
1000
1001 static String escapeIDForJSON(const std::optional<uint64_t>& value)
1002 {
1003     return value ? String::number(value.value()) : String("None");
1004 };
1005
1006 void NetworkResourceLoader::logCookieInformation() const
1007 {
1008     ASSERT(shouldLogCookieInformation(sessionID()));
1009
1010     auto networkStorageSession = WebCore::NetworkStorageSession::storageSession(sessionID());
1011     ASSERT(networkStorageSession);
1012
1013     logCookieInformation("NetworkResourceLoader", reinterpret_cast<const void*>(this), *networkStorageSession, originalRequest().firstPartyForCookies(), SameSiteInfo::create(originalRequest()), originalRequest().url(), originalRequest().httpReferrer(), frameID(), pageID(), identifier());
1014 }
1015
1016 static void logBlockedCookieInformation(const String& label, const void* loggedObject, const WebCore::NetworkStorageSession& networkStorageSession, const URL& firstParty, const SameSiteInfo& sameSiteInfo, const URL& url, const String& referrer, std::optional<uint64_t> frameID, std::optional<uint64_t> pageID, std::optional<uint64_t> identifier)
1017 {
1018     ASSERT(NetworkResourceLoader::shouldLogCookieInformation(networkStorageSession.sessionID()));
1019
1020     auto escapedURL = escapeForJSON(url.string());
1021     auto escapedFirstParty = escapeForJSON(firstParty.string());
1022     auto escapedFrameID = escapeIDForJSON(frameID);
1023     auto escapedPageID = escapeIDForJSON(pageID);
1024     auto escapedIdentifier = escapeIDForJSON(identifier);
1025     auto escapedReferrer = escapeForJSON(referrer);
1026
1027 #define LOCAL_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(networkStorageSession.sessionID().isAlwaysOnLoggingAllowed(), Network, "%p - %s::" fmt, loggedObject, label.utf8().data(), ##__VA_ARGS__)
1028 #define LOCAL_LOG(str, ...) \
1029     LOCAL_LOG_IF_ALLOWED("logCookieInformation: BLOCKED cookie access for pageID = %s, frameID = %s, resourceID = %s, firstParty = %s: " str, escapedPageID.utf8().data(), escapedFrameID.utf8().data(), escapedIdentifier.utf8().data(), escapedFirstParty.utf8().data(), ##__VA_ARGS__)
1030
1031     LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
1032     LOCAL_LOG(R"(  "partition": "%{public}s",)", "BLOCKED");
1033     LOCAL_LOG(R"(  "hasStorageAccess": %{public}s,)", "false");
1034     LOCAL_LOG(R"(  "referer": "%{public}s",)", escapedReferrer.utf8().data());
1035     LOCAL_LOG(R"(  "isSameSite": "%{public}s",)", sameSiteInfo.isSameSite ? "true" : "false");
1036     LOCAL_LOG(R"(  "isTopSite": "%{public}s",)", sameSiteInfo.isTopSite ? "true" : "false");
1037     LOCAL_LOG(R"(  "cookies": [])");
1038     LOCAL_LOG(R"(  })");
1039 #undef LOCAL_LOG
1040 #undef LOCAL_LOG_IF_ALLOWED
1041 }
1042
1043 static void logCookieInformationInternal(const String& label, const void* loggedObject, const WebCore::NetworkStorageSession& networkStorageSession, const URL& firstParty, const WebCore::SameSiteInfo& sameSiteInfo, const URL& url, const String& referrer, std::optional<uint64_t> frameID, std::optional<uint64_t> pageID, std::optional<uint64_t> identifier)
1044 {
1045     ASSERT(NetworkResourceLoader::shouldLogCookieInformation(networkStorageSession.sessionID()));
1046
1047     Vector<WebCore::Cookie> cookies;
1048     if (!networkStorageSession.getRawCookies(firstParty, sameSiteInfo, url, frameID, pageID, cookies))
1049         return;
1050
1051     auto escapedURL = escapeForJSON(url.string());
1052     auto escapedPartition = escapeForJSON(emptyString());
1053     auto escapedReferrer = escapeForJSON(referrer);
1054     auto escapedFrameID = escapeIDForJSON(frameID);
1055     auto escapedPageID = escapeIDForJSON(pageID);
1056     auto escapedIdentifier = escapeIDForJSON(identifier);
1057     bool hasStorageAccess = (frameID && pageID) ? networkStorageSession.hasStorageAccess(url.string(), firstParty.string(), frameID.value(), pageID.value()) : false;
1058
1059 #define LOCAL_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(networkStorageSession.sessionID().isAlwaysOnLoggingAllowed(), Network, "%p - %s::" fmt, loggedObject, label.utf8().data(), ##__VA_ARGS__)
1060 #define LOCAL_LOG(str, ...) \
1061     LOCAL_LOG_IF_ALLOWED("logCookieInformation: pageID = %s, frameID = %s, resourceID = %s: " str, escapedPageID.utf8().data(), escapedFrameID.utf8().data(), escapedIdentifier.utf8().data(), ##__VA_ARGS__)
1062
1063     LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
1064     LOCAL_LOG(R"(  "partition": "%{public}s",)", escapedPartition.utf8().data());
1065     LOCAL_LOG(R"(  "hasStorageAccess": %{public}s,)", hasStorageAccess ? "true" : "false");
1066     LOCAL_LOG(R"(  "referer": "%{public}s",)", escapedReferrer.utf8().data());
1067     LOCAL_LOG(R"(  "isSameSite": "%{public}s",)", sameSiteInfo.isSameSite ? "true" : "false");
1068     LOCAL_LOG(R"(  "isTopSite": "%{public}s",)", sameSiteInfo.isTopSite ? "true" : "false");
1069     LOCAL_LOG(R"(  "cookies": [)");
1070
1071     auto size = cookies.size();
1072     decltype(size) count = 0;
1073     for (const auto& cookie : cookies) {
1074         const char* trailingComma = ",";
1075         if (++count == size)
1076             trailingComma = "";
1077
1078         auto escapedName = escapeForJSON(cookie.name);
1079         auto escapedValue = escapeForJSON(cookie.value);
1080         auto escapedDomain = escapeForJSON(cookie.domain);
1081         auto escapedPath = escapeForJSON(cookie.path);
1082         auto escapedComment = escapeForJSON(cookie.comment);
1083         auto escapedCommentURL = escapeForJSON(cookie.commentURL.string());
1084         // FIXME: Log Same-Site policy for each cookie. See <https://bugs.webkit.org/show_bug.cgi?id=184894>.
1085
1086         LOCAL_LOG(R"(  { "name": "%{public}s",)", escapedName.utf8().data());
1087         LOCAL_LOG(R"(    "value": "%{public}s",)", escapedValue.utf8().data());
1088         LOCAL_LOG(R"(    "domain": "%{public}s",)", escapedDomain.utf8().data());
1089         LOCAL_LOG(R"(    "path": "%{public}s",)", escapedPath.utf8().data());
1090         LOCAL_LOG(R"(    "created": %f,)", cookie.created);
1091         LOCAL_LOG(R"(    "expires": %f,)", cookie.expires);
1092         LOCAL_LOG(R"(    "httpOnly": %{public}s,)", cookie.httpOnly ? "true" : "false");
1093         LOCAL_LOG(R"(    "secure": %{public}s,)", cookie.secure ? "true" : "false");
1094         LOCAL_LOG(R"(    "session": %{public}s,)", cookie.session ? "true" : "false");
1095         LOCAL_LOG(R"(    "comment": "%{public}s",)", escapedComment.utf8().data());
1096         LOCAL_LOG(R"(    "commentURL": "%{public}s")", escapedCommentURL.utf8().data());
1097         LOCAL_LOG(R"(  }%{public}s)", trailingComma);
1098     }
1099     LOCAL_LOG(R"(]})");
1100 #undef LOCAL_LOG
1101 #undef LOCAL_LOG_IF_ALLOWED
1102 }
1103
1104 void NetworkResourceLoader::logCookieInformation(const String& label, const void* loggedObject, const NetworkStorageSession& networkStorageSession, const URL& firstParty, const SameSiteInfo& sameSiteInfo, const URL& url, const String& referrer, std::optional<uint64_t> frameID, std::optional<uint64_t> pageID, std::optional<uint64_t> identifier)
1105 {
1106     ASSERT(shouldLogCookieInformation(networkStorageSession.sessionID()));
1107
1108     if (networkStorageSession.shouldBlockCookies(firstParty, url, frameID, pageID))
1109         logBlockedCookieInformation(label, loggedObject, networkStorageSession, firstParty, sameSiteInfo, url, referrer, frameID, pageID, identifier);
1110     else
1111         logCookieInformationInternal(label, loggedObject, networkStorageSession, firstParty, sameSiteInfo, url, referrer, frameID, pageID, identifier);
1112 }
1113 #endif
1114
1115 void NetworkResourceLoader::addConsoleMessage(MessageSource messageSource, MessageLevel messageLevel, const String& message, unsigned long)
1116 {
1117     send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  messageSource, messageLevel, message, identifier() }, m_parameters.webPageID);
1118 }
1119
1120 void NetworkResourceLoader::sendCSPViolationReport(URL&& reportURL, Ref<FormData>&& report)
1121 {
1122     send(Messages::WebPage::SendCSPViolationReport { m_parameters.webFrameID, WTFMove(reportURL), IPC::FormDataReference { WTFMove(report) } }, m_parameters.webPageID);
1123 }
1124
1125 void NetworkResourceLoader::enqueueSecurityPolicyViolationEvent(WebCore::SecurityPolicyViolationEvent::Init&& eventInit)
1126 {
1127     send(Messages::WebPage::EnqueueSecurityPolicyViolationEvent { m_parameters.webFrameID, WTFMove(eventInit) }, m_parameters.webPageID);
1128 }
1129
1130 void NetworkResourceLoader::logSlowCacheRetrieveIfNeeded(const NetworkCache::Cache::RetrieveInfo& info)
1131 {
1132 #if RELEASE_LOG_DISABLED
1133     UNUSED_PARAM(info);
1134 #else
1135     if (!isAlwaysOnLoggingAllowed())
1136         return;
1137     auto duration = info.completionTime - info.startTime;
1138     if (duration < 1_s)
1139         return;
1140     RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Took %.0fms, priority %d", duration.milliseconds(), info.priority);
1141     if (info.wasSpeculativeLoad)
1142         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Was speculative load");
1143     if (!info.storageTimings.startTime)
1144         return;
1145     RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Storage retrieve time %.0fms", (info.storageTimings.completionTime - info.storageTimings.startTime).milliseconds());
1146     if (info.storageTimings.dispatchTime) {
1147         auto time = (info.storageTimings.dispatchTime - info.storageTimings.startTime).milliseconds();
1148         auto count = info.storageTimings.dispatchCountAtDispatch - info.storageTimings.dispatchCountAtStart;
1149         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Dispatch delay %.0fms, dispatched %lu resources first", time, count);
1150     }
1151     if (info.storageTimings.recordIOStartTime)
1152         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Record I/O time %.0fms", (info.storageTimings.recordIOEndTime - info.storageTimings.recordIOStartTime).milliseconds());
1153     if (info.storageTimings.blobIOStartTime)
1154         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Blob I/O time %.0fms", (info.storageTimings.blobIOEndTime - info.storageTimings.blobIOStartTime).milliseconds());
1155     if (info.storageTimings.synchronizationInProgressAtDispatch)
1156         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Synchronization was in progress");
1157     if (info.storageTimings.shrinkInProgressAtDispatch)
1158         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Shrink was in progress");
1159     if (info.storageTimings.wasCanceled)
1160         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Retrieve was canceled");
1161 #endif
1162 }
1163
1164 } // namespace WebKit
1165
1166 #undef RELEASE_LOG_IF_ALLOWED
1167 #undef RELEASE_LOG_ERROR_IF_ALLOWED