Make fetch() use "same-origin" credentials by default
[WebKit-https.git] / Source / WebKit / NetworkProcess / NetworkResourceLoader.cpp
1 /*
2  * Copyright (C) 2012-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkResourceLoader.h"
28
29 #include "DataReference.h"
30 #include "FormDataReference.h"
31 #include "Logging.h"
32 #include "NetworkBlobRegistry.h"
33 #include "NetworkCache.h"
34 #include "NetworkConnectionToWebProcess.h"
35 #include "NetworkLoad.h"
36 #include "NetworkLoadChecker.h"
37 #include "NetworkProcess.h"
38 #include "NetworkProcessConnectionMessages.h"
39 #include "SessionTracker.h"
40 #include "WebCoreArgumentCoders.h"
41 #include "WebErrors.h"
42 #include "WebPageMessages.h"
43 #include "WebResourceLoaderMessages.h"
44 #include "WebsiteDataStoreParameters.h"
45 #include <JavaScriptCore/ConsoleTypes.h>
46 #include <WebCore/BlobDataFileReference.h>
47 #include <WebCore/CertificateInfo.h>
48 #include <WebCore/ContentSecurityPolicy.h>
49 #include <WebCore/DiagnosticLoggingKeys.h>
50 #include <WebCore/HTTPHeaderNames.h>
51 #include <WebCore/HTTPParsers.h>
52 #include <WebCore/NetworkLoadMetrics.h>
53 #include <WebCore/ProtectionSpace.h>
54 #include <WebCore/SameSiteInfo.h>
55 #include <WebCore/SecurityOrigin.h>
56 #include <WebCore/SharedBuffer.h>
57 #include <WebCore/SynchronousLoaderClient.h>
58 #include <wtf/RunLoop.h>
59
60 #if HAVE(CFNETWORK_STORAGE_PARTITIONING) && !RELEASE_LOG_DISABLED
61 #include <WebCore/NetworkStorageSession.h>
62 #include <WebCore/PlatformCookieJar.h>
63 #endif
64
65 #if USE(QUICK_LOOK)
66 #include <WebCore/PreviewLoader.h>
67 #endif
68
69 using namespace WebCore;
70
71 #define RELEASE_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__)
72 #define RELEASE_LOG_ERROR_IF_ALLOWED(fmt, ...) RELEASE_LOG_ERROR_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__)
73
74 namespace WebKit {
75
76 struct NetworkResourceLoader::SynchronousLoadData {
77     SynchronousLoadData(Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply&& reply)
78         : delayedReply(WTFMove(reply))
79     {
80         ASSERT(delayedReply);
81     }
82     ResourceRequest currentRequest;
83     Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply delayedReply;
84     ResourceResponse response;
85     ResourceError error;
86 };
87
88 static void sendReplyToSynchronousRequest(NetworkResourceLoader::SynchronousLoadData& data, const SharedBuffer* buffer)
89 {
90     ASSERT(data.delayedReply);
91     ASSERT(!data.response.isNull() || !data.error.isNull());
92
93     Vector<char> responseBuffer;
94     if (buffer && buffer->size())
95         responseBuffer.append(buffer->data(), buffer->size());
96
97     data.delayedReply(data.error, data.response, responseBuffer);
98     data.delayedReply = nullptr;
99 }
100
101 NetworkResourceLoader::NetworkResourceLoader(NetworkResourceLoadParameters&& parameters, NetworkConnectionToWebProcess& connection, Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply&& synchronousReply)
102     : m_parameters { WTFMove(parameters) }
103     , m_connection { connection }
104     , m_defersLoading { parameters.defersLoading }
105     , m_isAllowedToAskUserForCredentials { m_parameters.clientCredentialPolicy == ClientCredentialPolicy::MayAskClientForCredentials }
106     , m_bufferingTimer { *this, &NetworkResourceLoader::bufferingTimerFired }
107     , m_cache { sessionID().isEphemeral() ? nullptr : NetworkProcess::singleton().cache() }
108 {
109     ASSERT(RunLoop::isMain());
110     // FIXME: This is necessary because of the existence of EmptyFrameLoaderClient in WebCore.
111     //        Once bug 116233 is resolved, this ASSERT can just be "m_webPageID && m_webFrameID"
112     ASSERT((m_parameters.webPageID && m_parameters.webFrameID) || m_parameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials);
113
114     if (originalRequest().httpBody()) {
115         for (const auto& element : originalRequest().httpBody()->elements()) {
116             if (element.m_type == FormDataElement::Type::EncodedBlob)
117                 m_fileReferences.appendVector(NetworkBlobRegistry::singleton().filesInBlob(connection, element.m_url));
118         }
119     }
120
121     if (synchronousReply || parameters.shouldRestrictHTTPResponseAccess) {
122         m_networkLoadChecker = std::make_unique<NetworkLoadChecker>(FetchOptions { m_parameters.options }, m_parameters.sessionID, HTTPHeaderMap { m_parameters.originalRequestHeaders }, URL { m_parameters.request.url() }, m_parameters.sourceOrigin.copyRef(), m_parameters.preflightPolicy, originalRequest().httpReferrer(), shouldCaptureExtraNetworkLoadMetrics());
123         if (m_parameters.cspResponseHeaders)
124             m_networkLoadChecker->setCSPResponseHeaders(ContentSecurityPolicyResponseHeaders { m_parameters.cspResponseHeaders.value() });
125 #if ENABLE(CONTENT_EXTENSIONS)
126         m_networkLoadChecker->setContentExtensionController(URL { m_parameters.mainDocumentURL }, m_parameters.userContentControllerIdentifier);
127 #endif
128     }
129     if (synchronousReply)
130         m_synchronousLoadData = std::make_unique<SynchronousLoadData>(WTFMove(synchronousReply));
131 }
132
133 NetworkResourceLoader::~NetworkResourceLoader()
134 {
135     ASSERT(RunLoop::isMain());
136     ASSERT(!m_networkLoad);
137     ASSERT(!isSynchronous() || !m_synchronousLoadData->delayedReply);
138 }
139
140 bool NetworkResourceLoader::canUseCache(const ResourceRequest& request) const
141 {
142     if (!m_cache)
143         return false;
144     ASSERT(!sessionID().isEphemeral());
145
146     if (!request.url().protocolIsInHTTPFamily())
147         return false;
148     if (originalRequest().cachePolicy() == WebCore::ResourceRequestCachePolicy::DoNotUseAnyCache)
149         return false;
150
151     return true;
152 }
153
154 bool NetworkResourceLoader::canUseCachedRedirect(const ResourceRequest& request) const
155 {
156     if (!canUseCache(request))
157         return false;
158     // Limit cached redirects to avoid cycles and other trouble.
159     // Networking layer follows over 30 redirects but caching that many seems unnecessary.
160     static const unsigned maximumCachedRedirectCount { 5 };
161     if (m_redirectCount > maximumCachedRedirectCount)
162         return false;
163
164     return true;
165 }
166
167 bool NetworkResourceLoader::isSynchronous() const
168 {
169     return !!m_synchronousLoadData;
170 }
171
172 void NetworkResourceLoader::start()
173 {
174     ASSERT(RunLoop::isMain());
175
176     m_networkActivityTracker = m_connection->startTrackingResourceLoad(m_parameters.webPageID, m_parameters.identifier, isMainResource(), sessionID());
177
178     if (m_defersLoading) {
179         RELEASE_LOG_IF_ALLOWED("start: Loading is deferred (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
180         return;
181     }
182
183     ASSERT(!m_wasStarted);
184     m_wasStarted = true;
185
186     if (m_networkLoadChecker) {
187         m_networkLoadChecker->check(ResourceRequest { originalRequest() }, this, [this] (auto&& result) {
188             if (!result.has_value()) {
189                 if (!result.error().isCancellation())
190                     this->didFailLoading(result.error());
191                 return;
192             }
193             if (this->canUseCache(this->originalRequest())) {
194                 RELEASE_LOG_IF_ALLOWED("start: Checking cache for resource (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, this->isMainResource(), this->isSynchronous());
195                 this->retrieveCacheEntry(this->originalRequest());
196                 return;
197             }
198
199             this->startNetworkLoad(WTFMove(result.value()), FirstLoad::Yes);
200         });
201         return;
202     }
203     // FIXME: Remove that code path once m_networkLoadChecker is used for all network loads.
204     if (canUseCache(originalRequest())) {
205         RELEASE_LOG_IF_ALLOWED("start: Checking cache for resource (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
206         retrieveCacheEntry(originalRequest());
207         return;
208     }
209
210     startNetworkLoad(ResourceRequest { originalRequest() }, FirstLoad::Yes);
211 }
212
213 void NetworkResourceLoader::retrieveCacheEntry(const ResourceRequest& request)
214 {
215     ASSERT(canUseCache(request));
216
217     RefPtr<NetworkResourceLoader> loader(this);
218     m_cache->retrieve(request, { m_parameters.webPageID, m_parameters.webFrameID }, [this, loader = WTFMove(loader), request = ResourceRequest { request }](auto entry, auto info) mutable {
219         if (loader->hasOneRef()) {
220             // The loader has been aborted and is only held alive by this lambda.
221             return;
222         }
223
224         loader->logSlowCacheRetrieveIfNeeded(info);
225
226         if (!entry) {
227             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource not in cache (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
228             loader->startNetworkLoad(WTFMove(request), FirstLoad::Yes);
229             return;
230         }
231         if (entry->redirectRequest()) {
232             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Handling redirect (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
233             loader->dispatchWillSendRequestForCacheEntry(WTFMove(request), WTFMove(entry));
234             return;
235         }
236         if (loader->m_parameters.needsCertificateInfo && !entry->response().certificateInfo()) {
237             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource does not have required certificate (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
238             loader->startNetworkLoad(WTFMove(request), FirstLoad::Yes);
239             return;
240         }
241         if (entry->needsValidation() || request.cachePolicy() == WebCore::ResourceRequestCachePolicy::RefreshAnyCacheData) {
242             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Validating cache entry (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
243             loader->validateCacheEntry(WTFMove(entry));
244             return;
245         }
246         RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Retrieved resource from cache (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
247         loader->didRetrieveCacheEntry(WTFMove(entry));
248     });
249 }
250
251 void NetworkResourceLoader::startNetworkLoad(ResourceRequest&& request, FirstLoad load)
252 {
253     if (load == FirstLoad::Yes) {
254         RELEASE_LOG_IF_ALLOWED("startNetworkLoad: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous());
255
256         consumeSandboxExtensions();
257
258         if (isSynchronous() || m_parameters.maximumBufferingTime > 0_s)
259             m_bufferedData = SharedBuffer::create();
260
261         if (canUseCache(request))
262             m_bufferedDataForCache = SharedBuffer::create();
263     }
264
265     NetworkLoadParameters parameters = m_parameters;
266     parameters.defersLoading = m_defersLoading;
267     parameters.networkActivityTracker = m_networkActivityTracker;
268     if (m_networkLoadChecker)
269         parameters.storedCredentialsPolicy = m_networkLoadChecker->storedCredentialsPolicy();
270
271     if (request.url().protocolIsBlob())
272         parameters.blobFileReferences = NetworkBlobRegistry::singleton().filesInBlob(m_connection, originalRequest().url());
273
274     auto* networkSession = SessionTracker::networkSession(parameters.sessionID);
275     if (!networkSession && parameters.sessionID.isEphemeral()) {
276         NetworkProcess::singleton().addWebsiteDataStore(WebsiteDataStoreParameters::privateSessionParameters(parameters.sessionID));
277         networkSession = SessionTracker::networkSession(parameters.sessionID);
278     }
279     if (!networkSession) {
280         WTFLogAlways("Attempted to create a NetworkLoad with a session (id=%" PRIu64 ") that does not exist.", parameters.sessionID.sessionID());
281         RELEASE_LOG_ERROR_IF_ALLOWED("startNetworkLoad: Attempted to create a NetworkLoad with a session that does not exist (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", sessionID=%" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, parameters.sessionID.sessionID());
282         NetworkProcess::singleton().logDiagnosticMessage(m_parameters.webPageID, WebCore::DiagnosticLoggingKeys::internalErrorKey(), WebCore::DiagnosticLoggingKeys::invalidSessionIDKey(), WebCore::ShouldSample::No);
283         didFailLoading(internalError(request.url()));
284         return;
285     }
286
287     parameters.request = WTFMove(request);
288     m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters), *networkSession);
289
290     RELEASE_LOG_IF_ALLOWED("startNetworkLoad: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", description = %{public}s)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, m_networkLoad->description().utf8().data());
291
292     if (m_defersLoading) {
293         RELEASE_LOG_IF_ALLOWED("startNetworkLoad: Created, but deferred (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")",
294             m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
295     }
296 }
297
298 void NetworkResourceLoader::setDefersLoading(bool defers)
299 {
300     if (m_defersLoading == defers)
301         return;
302     m_defersLoading = defers;
303
304     if (defers)
305         RELEASE_LOG_IF_ALLOWED("setDefersLoading: Deferring resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
306     else
307         RELEASE_LOG_IF_ALLOWED("setDefersLoading: Resuming deferred resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
308
309     if (m_networkLoad) {
310         m_networkLoad->setDefersLoading(defers);
311         return;
312     }
313
314     if (!m_defersLoading && !m_wasStarted)
315         start();
316     else
317         RELEASE_LOG_IF_ALLOWED("setDefersLoading: defers = %d, but nothing to do (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_defersLoading, m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
318 }
319
320 void NetworkResourceLoader::cleanup(LoadResult result)
321 {
322     ASSERT(RunLoop::isMain());
323
324     m_connection->stopTrackingResourceLoad(m_parameters.identifier,
325         result == LoadResult::Success ? NetworkActivityTracker::CompletionCode::Success :
326         result == LoadResult::Failure ? NetworkActivityTracker::CompletionCode::Failure :
327         NetworkActivityTracker::CompletionCode::None);
328
329     m_bufferingTimer.stop();
330
331     invalidateSandboxExtensions();
332
333     m_networkLoad = nullptr;
334
335     // This will cause NetworkResourceLoader to be destroyed and therefore we do it last.
336     m_connection->didCleanupResourceLoader(*this);
337 }
338
339 void NetworkResourceLoader::convertToDownload(DownloadID downloadID, const ResourceRequest& request, const ResourceResponse& response)
340 {
341     ASSERT(m_networkLoad);
342     NetworkProcess::singleton().downloadManager().convertNetworkLoadToDownload(downloadID, std::exchange(m_networkLoad, nullptr), WTFMove(m_fileReferences), request, response);
343 }
344
345 void NetworkResourceLoader::abort()
346 {
347     ASSERT(RunLoop::isMain());
348
349     RELEASE_LOG_IF_ALLOWED("abort: Canceling resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")",
350         m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
351
352     if (m_networkLoad) {
353         if (canUseCache(m_networkLoad->currentRequest())) {
354             // We might already have used data from this incomplete load. Ensure older versions don't remain in the cache after cancel.
355             if (!m_response.isNull())
356                 m_cache->remove(m_networkLoad->currentRequest());
357         }
358         m_networkLoad->cancel();
359     }
360
361     cleanup(LoadResult::Cancel);
362 }
363
364 bool NetworkResourceLoader::shouldInterruptLoadForXFrameOptions(const String& xFrameOptions, const URL& url)
365 {
366     if (isMainFrameLoad())
367         return false;
368
369     switch (parseXFrameOptionsHeader(xFrameOptions)) {
370     case XFrameOptionsNone:
371     case XFrameOptionsAllowAll:
372         return false;
373     case XFrameOptionsDeny:
374         return true;
375     case XFrameOptionsSameOrigin: {
376         auto origin = SecurityOrigin::create(url);
377         auto topFrameOrigin = m_parameters.frameAncestorOrigins.last();
378         if (!origin->isSameSchemeHostPort(*topFrameOrigin))
379             return true;
380         for (auto& ancestorOrigin : m_parameters.frameAncestorOrigins) {
381             if (!origin->isSameSchemeHostPort(*ancestorOrigin))
382                 return true;
383         }
384         return false;
385     }
386     case XFrameOptionsConflict: {
387         String errorMessage = "Multiple 'X-Frame-Options' headers with conflicting values ('" + xFrameOptions + "') encountered when loading '" + url.stringCenterEllipsizedToLength() + "'. Falling back to 'DENY'.";
388         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::JS, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
389         return true;
390     }
391     case XFrameOptionsInvalid: {
392         String errorMessage = "Invalid 'X-Frame-Options' header encountered when loading '" + url.stringCenterEllipsizedToLength() + "': '" + xFrameOptions + "' is not a recognized directive. The header will be ignored.";
393         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::JS, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
394         return false;
395     }
396     }
397     ASSERT_NOT_REACHED();
398     return false;
399 }
400
401 bool NetworkResourceLoader::shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(const ResourceResponse& response)
402 {
403     ASSERT(isMainResource());
404
405 #if USE(QUICK_LOOK)
406     if (PreviewLoader::shouldCreateForMIMEType(response.mimeType()))
407         return false;
408 #endif
409
410     auto url = response.url();
411     ContentSecurityPolicy contentSecurityPolicy { URL { url }, this };
412     contentSecurityPolicy.didReceiveHeaders(ContentSecurityPolicyResponseHeaders { response }, originalRequest().httpReferrer());
413     if (!contentSecurityPolicy.allowFrameAncestors(m_parameters.frameAncestorOrigins, url))
414         return true;
415     String xFrameOptions = m_response.httpHeaderField(HTTPHeaderName::XFrameOptions);
416     if (!xFrameOptions.isNull() && shouldInterruptLoadForXFrameOptions(xFrameOptions, response.url())) {
417         String errorMessage = "Refused to display '" + response.url().stringCenterEllipsizedToLength() + "' in a frame because it set 'X-Frame-Options' to '" + xFrameOptions + "'.";
418         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::Security, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
419         return true;
420     }
421     return false;
422 }
423
424 auto NetworkResourceLoader::didReceiveResponse(ResourceResponse&& receivedResponse) -> ShouldContinueDidReceiveResponse
425 {
426     RELEASE_LOG_IF_ALLOWED("didReceiveResponse: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", httpStatusCode = %d, length = %" PRId64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, receivedResponse.httpStatusCode(), receivedResponse.expectedContentLength());
427
428     m_response = WTFMove(receivedResponse);
429
430     if (shouldCaptureExtraNetworkLoadMetrics() && m_networkLoadChecker) {
431         auto information = m_networkLoadChecker->takeNetworkLoadInformation();
432         information.response = m_response;
433         m_connection->addNetworkLoadInformation(identifier(), WTFMove(information));
434     }
435
436     // For multipart/x-mixed-replace didReceiveResponseAsync gets called multiple times and buffering would require special handling.
437     if (!isSynchronous() && m_response.isMultipart())
438         m_bufferedData = nullptr;
439
440     if (m_response.isMultipart())
441         m_bufferedDataForCache = nullptr;
442
443     if (m_cacheEntryForValidation) {
444         bool validationSucceeded = m_response.httpStatusCode() == 304; // 304 Not Modified
445         if (validationSucceeded) {
446             m_cacheEntryForValidation = m_cache->update(originalRequest(), { m_parameters.webPageID, m_parameters.webFrameID }, *m_cacheEntryForValidation, m_response);
447             // If the request was conditional then this revalidation was not triggered by the network cache and we pass the 304 response to WebCore.
448             if (originalRequest().isConditional())
449                 m_cacheEntryForValidation = nullptr;
450         } else
451             m_cacheEntryForValidation = nullptr;
452     }
453     if (m_cacheEntryForValidation)
454         return ShouldContinueDidReceiveResponse::Yes;
455
456     if (isMainResource() && shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(m_response)) {
457         send(Messages::WebResourceLoader::StopLoadingAfterXFrameOptionsOrContentSecurityPolicyDenied { });
458         return ShouldContinueDidReceiveResponse::No;
459     }
460
461     if (m_networkLoadChecker) {
462         auto error = m_networkLoadChecker->validateResponse(m_response);
463         if (!error.isNull()) {
464             RunLoop::main().dispatch([protectedThis = makeRef(*this), error = WTFMove(error)] {
465                 if (protectedThis->m_networkLoad)
466                     protectedThis->didFailLoading(error);
467             });
468             return ShouldContinueDidReceiveResponse::No;
469         }
470     }
471
472     auto response = sanitizeResponseIfPossible(ResourceResponse { m_response }, ResourceResponse::SanitizationType::CrossOriginSafe);
473     if (isSynchronous()) {
474         m_synchronousLoadData->response = WTFMove(response);
475         return ShouldContinueDidReceiveResponse::Yes;
476     }
477
478     // We wait to receive message NetworkResourceLoader::ContinueDidReceiveResponse before continuing a load for
479     // a main resource because the embedding client must decide whether to allow the load.
480     bool willWaitForContinueDidReceiveResponse = isMainResource();
481     send(Messages::WebResourceLoader::DidReceiveResponse { response, willWaitForContinueDidReceiveResponse });
482     return willWaitForContinueDidReceiveResponse ? ShouldContinueDidReceiveResponse::No : ShouldContinueDidReceiveResponse::Yes;
483 }
484
485 void NetworkResourceLoader::didReceiveBuffer(Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength)
486 {
487     if (!m_numBytesReceived) {
488         RELEASE_LOG_IF_ALLOWED("didReceiveBuffer: Started receiving data (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
489     }
490     m_numBytesReceived += buffer->size();
491
492     ASSERT(!m_cacheEntryForValidation);
493
494     if (m_bufferedDataForCache) {
495         // Prevent memory growth in case of streaming data.
496         const size_t maximumCacheBufferSize = 10 * 1024 * 1024;
497         if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize)
498             m_bufferedDataForCache->append(buffer.get());
499         else
500             m_bufferedDataForCache = nullptr;
501     }
502     // FIXME: At least on OS X Yosemite we always get -1 from the resource handle.
503     unsigned encodedDataLength = reportedEncodedDataLength >= 0 ? reportedEncodedDataLength : buffer->size();
504
505     m_bytesReceived += buffer->size();
506     if (m_bufferedData) {
507         m_bufferedData->append(buffer.get());
508         m_bufferedDataEncodedDataLength += encodedDataLength;
509         startBufferingTimerIfNeeded();
510         return;
511     }
512     sendBuffer(buffer, encodedDataLength);
513 }
514
515 void NetworkResourceLoader::didFinishLoading(const NetworkLoadMetrics& networkLoadMetrics)
516 {
517     RELEASE_LOG_IF_ALLOWED("didFinishLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", length = %zd)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, m_numBytesReceived);
518
519     if (shouldCaptureExtraNetworkLoadMetrics())
520         m_connection->addNetworkLoadInformationMetrics(identifier(), networkLoadMetrics);
521
522     if (m_cacheEntryForValidation) {
523         // 304 Not Modified
524         ASSERT(m_response.httpStatusCode() == 304);
525         LOG(NetworkCache, "(NetworkProcess) revalidated");
526         didRetrieveCacheEntry(WTFMove(m_cacheEntryForValidation));
527         return;
528     }
529
530 #if HAVE(CFNETWORK_STORAGE_PARTITIONING) && !RELEASE_LOG_DISABLED
531     if (shouldLogCookieInformation())
532         logCookieInformation();
533 #endif
534
535     if (isSynchronous())
536         sendReplyToSynchronousRequest(*m_synchronousLoadData, m_bufferedData.get());
537     else {
538         if (m_bufferedData && !m_bufferedData->isEmpty()) {
539             // FIXME: Pass a real value or remove the encoded data size feature.
540             sendBuffer(*m_bufferedData, -1);
541         }
542         send(Messages::WebResourceLoader::DidFinishResourceLoad(networkLoadMetrics));
543     }
544
545     tryStoreAsCacheEntry();
546
547     cleanup(LoadResult::Success);
548 }
549
550 void NetworkResourceLoader::didFailLoading(const ResourceError& error)
551 {
552     RELEASE_LOG_IF_ALLOWED("didFailLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isTimeout = %d, isCancellation = %d, isAccessControl = %d, errCode = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, error.isTimeout(), error.isCancellation(), error.isAccessControl(), error.errorCode());
553
554     if (shouldCaptureExtraNetworkLoadMetrics())
555         m_connection->removeNetworkLoadInformation(identifier());
556
557     ASSERT(!error.isNull());
558
559     m_cacheEntryForValidation = nullptr;
560
561     if (isSynchronous()) {
562         m_synchronousLoadData->error = error;
563         sendReplyToSynchronousRequest(*m_synchronousLoadData, nullptr);
564     } else if (auto* connection = messageSenderConnection())
565         connection->send(Messages::WebResourceLoader::DidFailResourceLoad(error), messageSenderDestinationID());
566
567     cleanup(LoadResult::Failure);
568 }
569
570 void NetworkResourceLoader::didBlockAuthenticationChallenge()
571 {
572     send(Messages::WebResourceLoader::DidBlockAuthenticationChallenge());
573 }
574
575 void NetworkResourceLoader::willSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse)
576 {
577     ++m_redirectCount;
578
579     if (redirectResponse.source() == ResourceResponse::Source::Network && canUseCachedRedirect(request))
580         m_cache->storeRedirect(request, redirectResponse, redirectRequest);
581
582     if (m_networkLoadChecker) {
583         m_networkLoadChecker->storeRedirectionIfNeeded(request, redirectResponse);
584         m_networkLoadChecker->checkRedirection(WTFMove(request), WTFMove(redirectRequest), WTFMove(redirectResponse), this, [protectedThis = makeRef(*this), this, storedCredentialsPolicy = m_networkLoadChecker->storedCredentialsPolicy()](auto&& result) mutable {
585             if (!result.has_value()) {
586                 if (result.error().isCancellation())
587                     return;
588
589                 this->didFailLoading(result.error());
590                 return;
591             }
592
593             if (m_parameters.options.redirect == FetchOptions::Redirect::Manual) {
594                 this->didFinishWithRedirectResponse(WTFMove(result->redirectResponse));
595                 return;
596             }
597
598             if (this->isSynchronous()) {
599                 if (storedCredentialsPolicy != m_networkLoadChecker->storedCredentialsPolicy()) {
600                     // We need to restart the load to update the session according the new credential policy.
601                     this->restartNetworkLoad(WTFMove(result->redirectRequest));
602                     return;
603                 }
604
605                 // We do not support prompting for credentials for synchronous loads. If we ever change this policy then
606                 // we need to take care to prompt if and only if request and redirectRequest are not mixed content.
607                 this->continueWillSendRequest(WTFMove(result->redirectRequest), false);
608                 return;
609             }
610
611             m_shouldRestartLoad = storedCredentialsPolicy != m_networkLoadChecker->storedCredentialsPolicy();
612             this->continueWillSendRedirectedRequest(WTFMove(result->request), WTFMove(result->redirectRequest), WTFMove(result->redirectResponse));
613         });
614         return;
615     }
616     continueWillSendRedirectedRequest(WTFMove(request), WTFMove(redirectRequest), WTFMove(redirectResponse));
617 }
618
619 void NetworkResourceLoader::continueWillSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse)
620 {
621     ASSERT(!isSynchronous());
622
623     send(Messages::WebResourceLoader::WillSendRequest(redirectRequest, sanitizeResponseIfPossible(WTFMove(redirectResponse), ResourceResponse::SanitizationType::Redirection)));
624 }
625
626 void NetworkResourceLoader::didFinishWithRedirectResponse(ResourceResponse&& redirectResponse)
627 {
628     redirectResponse.setType(ResourceResponse::Type::Opaqueredirect);
629     didReceiveResponse(WTFMove(redirectResponse));
630
631     WebCore::NetworkLoadMetrics networkLoadMetrics;
632     networkLoadMetrics.markComplete();
633     networkLoadMetrics.responseBodyBytesReceived = 0;
634     networkLoadMetrics.responseBodyDecodedSize = 0;
635     send(Messages::WebResourceLoader::DidFinishResourceLoad { networkLoadMetrics });
636
637     cleanup(LoadResult::Success);
638 }
639
640 ResourceResponse NetworkResourceLoader::sanitizeResponseIfPossible(ResourceResponse&& response, ResourceResponse::SanitizationType type)
641 {
642     if (m_parameters.shouldRestrictHTTPResponseAccess)
643         response.sanitizeHTTPHeaderFields(type);
644
645     return WTFMove(response);
646 }
647
648 void NetworkResourceLoader::restartNetworkLoad(WebCore::ResourceRequest&& newRequest)
649 {
650     if (m_networkLoad)
651         m_networkLoad->cancel();
652     if (m_networkLoadChecker)
653         m_networkLoadChecker->prepareRedirectedRequest(newRequest);
654
655     startNetworkLoad(WTFMove(newRequest), FirstLoad::No);
656 }
657
658 void NetworkResourceLoader::continueWillSendRequest(ResourceRequest&& newRequest, bool isAllowedToAskUserForCredentials)
659 {
660     if (m_shouldRestartLoad) {
661         m_shouldRestartLoad = false;
662         restartNetworkLoad(WTFMove(newRequest));
663         return;
664     }
665
666     if (m_networkLoadChecker) {
667         // FIXME: We should be doing this check when receiving the redirection and not allow about protocol as per fetch spec.
668         if (!newRequest.url().protocolIsInHTTPFamily() && !newRequest.url().isBlankURL() && m_redirectCount) {
669             didFailLoading(ResourceError { String { }, 0, newRequest.url(), "Redirection to URL with a scheme that is not HTTP(S)"_s, ResourceError::Type::AccessControl });
670             return;
671         }
672     }
673
674     RELEASE_LOG_IF_ALLOWED("continueWillSendRequest: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier);
675
676     if (m_networkLoadChecker)
677         m_networkLoadChecker->prepareRedirectedRequest(newRequest);
678
679     m_isAllowedToAskUserForCredentials = isAllowedToAskUserForCredentials;
680
681     // If there is a match in the network cache, we need to reuse the original cache policy and partition.
682     newRequest.setCachePolicy(originalRequest().cachePolicy());
683     newRequest.setCachePartition(originalRequest().cachePartition());
684
685     if (m_isWaitingContinueWillSendRequestForCachedRedirect) {
686         m_isWaitingContinueWillSendRequestForCachedRedirect = false;
687
688         LOG(NetworkCache, "(NetworkProcess) Retrieving cached redirect");
689
690         if (canUseCachedRedirect(newRequest))
691             retrieveCacheEntry(newRequest);
692         else
693             startNetworkLoad(WTFMove(newRequest), FirstLoad::Yes);
694
695         return;
696     }
697
698     if (m_networkLoad)
699         m_networkLoad->continueWillSendRequest(WTFMove(newRequest));
700 }
701
702 void NetworkResourceLoader::continueDidReceiveResponse()
703 {
704     if (m_cacheEntryWaitingForContinueDidReceiveResponse) {
705         continueProcessingCachedEntryAfterDidReceiveResponse(WTFMove(m_cacheEntryWaitingForContinueDidReceiveResponse));
706         return;
707     }
708
709     // FIXME: Remove this check once BlobResourceHandle implements didReceiveResponseAsync correctly.
710     // Currently, it does not wait for response, so the load is likely to finish before continueDidReceiveResponse.
711     if (m_networkLoad)
712         m_networkLoad->continueDidReceiveResponse();
713 }
714
715 void NetworkResourceLoader::didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent)
716 {
717     if (!isSynchronous())
718         send(Messages::WebResourceLoader::DidSendData(bytesSent, totalBytesToBeSent));
719 }
720
721 void NetworkResourceLoader::startBufferingTimerIfNeeded()
722 {
723     if (isSynchronous())
724         return;
725     if (m_bufferingTimer.isActive())
726         return;
727     m_bufferingTimer.startOneShot(m_parameters.maximumBufferingTime);
728 }
729
730 void NetworkResourceLoader::bufferingTimerFired()
731 {
732     ASSERT(m_bufferedData);
733     ASSERT(m_networkLoad);
734
735     if (m_bufferedData->isEmpty())
736         return;
737
738     IPC::SharedBufferDataReference dataReference(m_bufferedData.get());
739     size_t encodedLength = m_bufferedDataEncodedDataLength;
740
741     m_bufferedData = SharedBuffer::create();
742     m_bufferedDataEncodedDataLength = 0;
743
744     send(Messages::WebResourceLoader::DidReceiveData(dataReference, encodedLength));
745 }
746
747 void NetworkResourceLoader::sendBuffer(SharedBuffer& buffer, size_t encodedDataLength)
748 {
749     ASSERT(!isSynchronous());
750
751     IPC::SharedBufferDataReference dataReference(&buffer);
752     send(Messages::WebResourceLoader::DidReceiveData(dataReference, encodedDataLength));
753 }
754
755 void NetworkResourceLoader::tryStoreAsCacheEntry()
756 {
757     if (!canUseCache(m_networkLoad->currentRequest()))
758         return;
759     if (!m_bufferedDataForCache)
760         return;
761
762     m_cache->store(m_networkLoad->currentRequest(), m_response, WTFMove(m_bufferedDataForCache), [loader = makeRef(*this)](auto& mappedBody) mutable {
763 #if ENABLE(SHAREABLE_RESOURCE)
764         if (mappedBody.shareableResourceHandle.isNull())
765             return;
766         LOG(NetworkCache, "(NetworkProcess) sending DidCacheResource");
767         loader->send(Messages::NetworkProcessConnection::DidCacheResource(loader->originalRequest(), mappedBody.shareableResourceHandle, loader->sessionID()));
768 #endif
769     });
770 }
771
772 void NetworkResourceLoader::didRetrieveCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
773 {
774     auto response = entry->response();
775
776     if (isMainResource() && shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(response)) {
777         send(Messages::WebResourceLoader::StopLoadingAfterXFrameOptionsOrContentSecurityPolicyDenied { });
778         return;
779     }
780     if (m_networkLoadChecker) {
781         auto error = m_networkLoadChecker->validateResponse(response);
782         if (!error.isNull()) {
783             didFailLoading(error);
784             return;
785         }
786     }
787
788     response = sanitizeResponseIfPossible(WTFMove(response), ResourceResponse::SanitizationType::CrossOriginSafe);
789     if (isSynchronous()) {
790         m_synchronousLoadData->response = WTFMove(response);
791         sendReplyToSynchronousRequest(*m_synchronousLoadData, entry->buffer());
792         cleanup(LoadResult::Success);
793         return;
794     }
795
796     bool needsContinueDidReceiveResponseMessage = isMainResource();
797     send(Messages::WebResourceLoader::DidReceiveResponse { response, needsContinueDidReceiveResponseMessage });
798
799     if (needsContinueDidReceiveResponseMessage)
800         m_cacheEntryWaitingForContinueDidReceiveResponse = WTFMove(entry);
801     else
802         continueProcessingCachedEntryAfterDidReceiveResponse(WTFMove(entry));
803 }
804
805 void NetworkResourceLoader::continueProcessingCachedEntryAfterDidReceiveResponse(std::unique_ptr<NetworkCache::Entry> entry)
806 {
807     if (entry->sourceStorageRecord().bodyHash && !m_parameters.derivedCachedDataTypesToRetrieve.isEmpty()) {
808         auto bodyHash = *entry->sourceStorageRecord().bodyHash;
809         auto* entryPtr = entry.release();
810         auto retrieveCount = m_parameters.derivedCachedDataTypesToRetrieve.size();
811
812         for (auto& type : m_parameters.derivedCachedDataTypesToRetrieve) {
813             NetworkCache::DataKey key { originalRequest().cachePartition(), type, bodyHash };
814             m_cache->retrieveData(key, [loader = makeRef(*this), entryPtr, type, retrieveCount] (const uint8_t* data, size_t size) mutable {
815                 loader->m_retrievedDerivedDataCount++;
816                 bool retrievedAll = loader->m_retrievedDerivedDataCount == retrieveCount;
817                 std::unique_ptr<NetworkCache::Entry> entry(retrievedAll ? entryPtr : nullptr);
818                 if (loader->hasOneRef())
819                     return;
820                 if (data) {
821                     IPC::DataReference dataReference(data, size);
822                     loader->send(Messages::WebResourceLoader::DidRetrieveDerivedData(type, dataReference));
823                 }
824                 if (retrievedAll) {
825                     loader->sendResultForCacheEntry(WTFMove(entry));
826                     loader->cleanup(LoadResult::Success);
827                 }
828             });
829         }
830         return;
831     }
832
833     sendResultForCacheEntry(WTFMove(entry));
834
835     cleanup(LoadResult::Success);
836 }
837
838 void NetworkResourceLoader::sendResultForCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
839 {
840 #if ENABLE(SHAREABLE_RESOURCE)
841     if (!entry->shareableResourceHandle().isNull()) {
842         send(Messages::WebResourceLoader::DidReceiveResource(entry->shareableResourceHandle()));
843         return;
844     }
845 #endif
846
847 #if HAVE(CFNETWORK_STORAGE_PARTITIONING) && !RELEASE_LOG_DISABLED
848     if (shouldLogCookieInformation())
849         logCookieInformation();
850 #endif
851
852     WebCore::NetworkLoadMetrics networkLoadMetrics;
853     networkLoadMetrics.markComplete();
854     networkLoadMetrics.requestHeaderBytesSent = 0;
855     networkLoadMetrics.requestBodyBytesSent = 0;
856     networkLoadMetrics.responseHeaderBytesReceived = 0;
857     networkLoadMetrics.responseBodyBytesReceived = 0;
858     networkLoadMetrics.responseBodyDecodedSize = 0;
859
860     sendBuffer(*entry->buffer(), entry->buffer()->size());
861     send(Messages::WebResourceLoader::DidFinishResourceLoad(networkLoadMetrics));
862 }
863
864 void NetworkResourceLoader::validateCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
865 {
866     ASSERT(!m_networkLoad);
867
868     // If the request is already conditional then the revalidation was not triggered by the disk cache
869     // and we should not overwrite the existing conditional headers.
870     ResourceRequest revalidationRequest = originalRequest();
871     if (!revalidationRequest.isConditional()) {
872         String eTag = entry->response().httpHeaderField(HTTPHeaderName::ETag);
873         String lastModified = entry->response().httpHeaderField(HTTPHeaderName::LastModified);
874         if (!eTag.isEmpty())
875             revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfNoneMatch, eTag);
876         if (!lastModified.isEmpty())
877             revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfModifiedSince, lastModified);
878     }
879
880     m_cacheEntryForValidation = WTFMove(entry);
881
882     startNetworkLoad(WTFMove(revalidationRequest), FirstLoad::Yes);
883 }
884
885 void NetworkResourceLoader::dispatchWillSendRequestForCacheEntry(ResourceRequest&& request, std::unique_ptr<NetworkCache::Entry>&& entry)
886 {
887     ASSERT(entry->redirectRequest());
888     ASSERT(!m_isWaitingContinueWillSendRequestForCachedRedirect);
889
890     LOG(NetworkCache, "(NetworkProcess) Executing cached redirect");
891
892     m_isWaitingContinueWillSendRequestForCachedRedirect = true;
893     willSendRedirectedRequest(WTFMove(request), ResourceRequest { *entry->redirectRequest() }, ResourceResponse { entry->response() });
894 }
895
896 IPC::Connection* NetworkResourceLoader::messageSenderConnection()
897 {
898     return &connectionToWebProcess().connection();
899 }
900
901 void NetworkResourceLoader::consumeSandboxExtensions()
902 {
903     ASSERT(!m_didConsumeSandboxExtensions);
904
905     for (auto& extension : m_parameters.requestBodySandboxExtensions)
906         extension->consume();
907
908     if (auto& extension = m_parameters.resourceSandboxExtension)
909         extension->consume();
910
911     for (auto& fileReference : m_fileReferences)
912         fileReference->prepareForFileAccess();
913
914     m_didConsumeSandboxExtensions = true;
915 }
916
917 void NetworkResourceLoader::invalidateSandboxExtensions()
918 {
919     if (m_didConsumeSandboxExtensions) {
920         for (auto& extension : m_parameters.requestBodySandboxExtensions)
921             extension->revoke();
922         if (auto& extension = m_parameters.resourceSandboxExtension)
923             extension->revoke();
924         for (auto& fileReference : m_fileReferences)
925             fileReference->revokeFileAccess();
926
927         m_didConsumeSandboxExtensions = false;
928     }
929
930     m_fileReferences.clear();
931 }
932
933 #if USE(PROTECTION_SPACE_AUTH_CALLBACK)
934 void NetworkResourceLoader::canAuthenticateAgainstProtectionSpaceAsync(const ProtectionSpace& protectionSpace)
935 {
936     NetworkProcess::singleton().canAuthenticateAgainstProtectionSpace(*this, protectionSpace);
937 }
938
939 void NetworkResourceLoader::continueCanAuthenticateAgainstProtectionSpace(bool result)
940 {
941     if (m_networkLoad)
942         m_networkLoad->continueCanAuthenticateAgainstProtectionSpace(result);
943 }
944 #endif
945
946 bool NetworkResourceLoader::isAlwaysOnLoggingAllowed() const
947 {
948     if (NetworkProcess::singleton().sessionIsControlledByAutomation(sessionID()))
949         return true;
950
951     return sessionID().isAlwaysOnLoggingAllowed();
952 }
953
954 bool NetworkResourceLoader::shouldCaptureExtraNetworkLoadMetrics() const
955 {
956     return m_connection->captureExtraNetworkLoadMetricsEnabled();
957 }
958
959 #if HAVE(CFNETWORK_STORAGE_PARTITIONING) && !RELEASE_LOG_DISABLED
960 bool NetworkResourceLoader::shouldLogCookieInformation()
961 {
962     return NetworkProcess::singleton().shouldLogCookieInformation();
963 }
964
965 static String escapeForJSON(String s)
966 {
967     return s.replace('\\', "\\\\").replace('"', "\\\"");
968 }
969
970 static String escapeIDForJSON(const std::optional<uint64_t>& value)
971 {
972     return value ? String::number(value.value()) : String("None");
973 };
974
975 void NetworkResourceLoader::logCookieInformation() const
976 {
977     ASSERT(shouldLogCookieInformation());
978
979     auto networkStorageSession = WebCore::NetworkStorageSession::storageSession(sessionID());
980     ASSERT(networkStorageSession);
981
982     logCookieInformation("NetworkResourceLoader", reinterpret_cast<const void*>(this), *networkStorageSession, originalRequest().firstPartyForCookies(), SameSiteInfo::create(originalRequest()), originalRequest().url(), originalRequest().httpReferrer(), frameID(), pageID(), identifier());
983 }
984
985 static void logBlockedCookieInformation(const String& label, const void* loggedObject, const WebCore::NetworkStorageSession& networkStorageSession, const WebCore::URL& firstParty, const SameSiteInfo& sameSiteInfo, const WebCore::URL& url, const String& referrer, std::optional<uint64_t> frameID, std::optional<uint64_t> pageID, std::optional<uint64_t> identifier)
986 {
987     ASSERT(NetworkResourceLoader::shouldLogCookieInformation());
988
989     auto escapedURL = escapeForJSON(url.string());
990     auto escapedFirstParty = escapeForJSON(firstParty.string());
991     auto escapedFrameID = escapeIDForJSON(frameID);
992     auto escapedPageID = escapeIDForJSON(pageID);
993     auto escapedIdentifier = escapeIDForJSON(identifier);
994     auto escapedReferrer = escapeForJSON(referrer);
995
996 #define LOCAL_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(networkStorageSession.sessionID().isAlwaysOnLoggingAllowed(), Network, "%p - %s::" fmt, loggedObject, label.utf8().data(), ##__VA_ARGS__)
997 #define LOCAL_LOG(str, ...) \
998     LOCAL_LOG_IF_ALLOWED("logCookieInformation: BLOCKED cookie access for pageID = %s, frameID = %s, resourceID = %s, firstParty = %s: " str, escapedPageID.utf8().data(), escapedFrameID.utf8().data(), escapedIdentifier.utf8().data(), escapedFirstParty.utf8().data(), ##__VA_ARGS__)
999
1000     LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
1001     LOCAL_LOG(R"(  "partition": "%{public}s",)", "BLOCKED");
1002     LOCAL_LOG(R"(  "hasStorageAccess": %{public}s,)", "false");
1003     LOCAL_LOG(R"(  "referer": "%{public}s",)", escapedReferrer.utf8().data());
1004     LOCAL_LOG(R"(  "isSameSite": "%{public}s",)", sameSiteInfo.isSameSite ? "true" : "false");
1005     LOCAL_LOG(R"(  "isTopSite": "%{public}s",)", sameSiteInfo.isTopSite ? "true" : "false");
1006     LOCAL_LOG(R"(  "cookies": [])");
1007     LOCAL_LOG(R"(  })");
1008 #undef LOCAL_LOG
1009 #undef LOCAL_LOG_IF_ALLOWED
1010 }
1011
1012 static void logCookieInformationInternal(const String& label, const void* loggedObject, const WebCore::NetworkStorageSession& networkStorageSession, const WebCore::URL& partition, const WebCore::SameSiteInfo& sameSiteInfo, const WebCore::URL& url, const String& referrer, std::optional<uint64_t> frameID, std::optional<uint64_t> pageID, std::optional<uint64_t> identifier)
1013 {
1014     ASSERT(NetworkResourceLoader::shouldLogCookieInformation());
1015
1016     Vector<WebCore::Cookie> cookies;
1017     if (!WebCore::getRawCookies(networkStorageSession, partition, sameSiteInfo, url, frameID, pageID, cookies))
1018         return;
1019
1020     auto escapedURL = escapeForJSON(url.string());
1021     auto escapedPartition = escapeForJSON(partition.string());
1022     auto escapedReferrer = escapeForJSON(referrer);
1023     auto escapedFrameID = escapeIDForJSON(frameID);
1024     auto escapedPageID = escapeIDForJSON(pageID);
1025     auto escapedIdentifier = escapeIDForJSON(identifier);
1026     bool hasStorageAccess = (frameID && pageID) ? networkStorageSession.hasStorageAccess(url.string(), partition.string(), frameID.value(), pageID.value()) : false;
1027
1028 #define LOCAL_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(networkStorageSession.sessionID().isAlwaysOnLoggingAllowed(), Network, "%p - %s::" fmt, loggedObject, label.utf8().data(), ##__VA_ARGS__)
1029 #define LOCAL_LOG(str, ...) \
1030     LOCAL_LOG_IF_ALLOWED("logCookieInformation: pageID = %s, frameID = %s, resourceID = %s: " str, escapedPageID.utf8().data(), escapedFrameID.utf8().data(), escapedIdentifier.utf8().data(), ##__VA_ARGS__)
1031
1032     LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
1033     LOCAL_LOG(R"(  "partition": "%{public}s",)", escapedPartition.utf8().data());
1034     LOCAL_LOG(R"(  "hasStorageAccess": %{public}s,)", hasStorageAccess ? "true" : "false");
1035     LOCAL_LOG(R"(  "referer": "%{public}s",)", escapedReferrer.utf8().data());
1036     LOCAL_LOG(R"(  "isSameSite": "%{public}s",)", sameSiteInfo.isSameSite ? "true" : "false");
1037     LOCAL_LOG(R"(  "isTopSite": "%{public}s",)", sameSiteInfo.isTopSite ? "true" : "false");
1038     LOCAL_LOG(R"(  "cookies": [)");
1039
1040     auto size = cookies.size();
1041     decltype(size) count = 0;
1042     for (const auto& cookie : cookies) {
1043         const char* trailingComma = ",";
1044         if (++count == size)
1045             trailingComma = "";
1046
1047         auto escapedName = escapeForJSON(cookie.name);
1048         auto escapedValue = escapeForJSON(cookie.value);
1049         auto escapedDomain = escapeForJSON(cookie.domain);
1050         auto escapedPath = escapeForJSON(cookie.path);
1051         auto escapedComment = escapeForJSON(cookie.comment);
1052         auto escapedCommentURL = escapeForJSON(cookie.commentURL.string());
1053         // FIXME: Log Same-Site policy for each cookie. See <https://bugs.webkit.org/show_bug.cgi?id=184894>.
1054
1055         LOCAL_LOG(R"(  { "name": "%{public}s",)", escapedName.utf8().data());
1056         LOCAL_LOG(R"(    "value": "%{public}s",)", escapedValue.utf8().data());
1057         LOCAL_LOG(R"(    "domain": "%{public}s",)", escapedDomain.utf8().data());
1058         LOCAL_LOG(R"(    "path": "%{public}s",)", escapedPath.utf8().data());
1059         LOCAL_LOG(R"(    "created": %f,)", cookie.created);
1060         LOCAL_LOG(R"(    "expires": %f,)", cookie.expires);
1061         LOCAL_LOG(R"(    "httpOnly": %{public}s,)", cookie.httpOnly ? "true" : "false");
1062         LOCAL_LOG(R"(    "secure": %{public}s,)", cookie.secure ? "true" : "false");
1063         LOCAL_LOG(R"(    "session": %{public}s,)", cookie.session ? "true" : "false");
1064         LOCAL_LOG(R"(    "comment": "%{public}s",)", escapedComment.utf8().data());
1065         LOCAL_LOG(R"(    "commentURL": "%{public}s")", escapedCommentURL.utf8().data());
1066         LOCAL_LOG(R"(  }%{public}s)", trailingComma);
1067     }
1068     LOCAL_LOG(R"(]})");
1069 #undef LOCAL_LOG
1070 #undef LOCAL_LOG_IF_ALLOWED
1071 }
1072
1073 void NetworkResourceLoader::logCookieInformation(const String& label, const void* loggedObject, const NetworkStorageSession& networkStorageSession, const URL& firstParty, const SameSiteInfo& sameSiteInfo, const URL& url, const String& referrer, std::optional<uint64_t> frameID, std::optional<uint64_t> pageID, std::optional<uint64_t> identifier)
1074 {
1075     ASSERT(shouldLogCookieInformation());
1076
1077     if (networkStorageSession.shouldBlockCookies(firstParty, url))
1078         logBlockedCookieInformation(label, loggedObject, networkStorageSession, firstParty, sameSiteInfo, url, referrer, frameID, pageID, identifier);
1079     else {
1080         auto partition = URL(ParsedURLString, networkStorageSession.cookieStoragePartition(firstParty, url, frameID, pageID));
1081         logCookieInformationInternal(label, loggedObject, networkStorageSession, partition, sameSiteInfo, url, referrer, frameID, pageID, identifier);
1082     }
1083 }
1084 #endif
1085
1086 void NetworkResourceLoader::addConsoleMessage(MessageSource messageSource, MessageLevel messageLevel, const String& message, unsigned long)
1087 {
1088     send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  messageSource, messageLevel, message, identifier() }, m_parameters.webPageID);
1089 }
1090
1091 void NetworkResourceLoader::sendCSPViolationReport(URL&& reportURL, Ref<FormData>&& report)
1092 {
1093     send(Messages::WebPage::SendCSPViolationReport { m_parameters.webFrameID, WTFMove(reportURL), IPC::FormDataReference { WTFMove(report) } }, m_parameters.webPageID);
1094 }
1095
1096 void NetworkResourceLoader::enqueueSecurityPolicyViolationEvent(WebCore::SecurityPolicyViolationEvent::Init&& eventInit)
1097 {
1098     send(Messages::WebPage::EnqueueSecurityPolicyViolationEvent { m_parameters.webFrameID, WTFMove(eventInit) }, m_parameters.webPageID);
1099 }
1100
1101 void NetworkResourceLoader::logSlowCacheRetrieveIfNeeded(const NetworkCache::Cache::RetrieveInfo& info)
1102 {
1103 #if RELEASE_LOG_DISABLED
1104     UNUSED_PARAM(info);
1105 #else
1106     if (!isAlwaysOnLoggingAllowed())
1107         return;
1108     auto duration = info.completionTime - info.startTime;
1109     if (duration < 1_s)
1110         return;
1111     RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Took %.0fms, priority %d", duration.milliseconds(), info.priority);
1112     if (info.wasSpeculativeLoad)
1113         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Was speculative load");
1114     if (!info.storageTimings.startTime)
1115         return;
1116     RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Storage retrieve time %.0fms", (info.storageTimings.completionTime - info.storageTimings.startTime).milliseconds());
1117     if (info.storageTimings.dispatchTime) {
1118         auto time = (info.storageTimings.dispatchTime - info.storageTimings.startTime).milliseconds();
1119         auto count = info.storageTimings.dispatchCountAtDispatch - info.storageTimings.dispatchCountAtStart;
1120         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Dispatch delay %.0fms, dispatched %lu resources first", time, count);
1121     }
1122     if (info.storageTimings.recordIOStartTime)
1123         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Record I/O time %.0fms", (info.storageTimings.recordIOEndTime - info.storageTimings.recordIOStartTime).milliseconds());
1124     if (info.storageTimings.blobIOStartTime)
1125         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Blob I/O time %.0fms", (info.storageTimings.blobIOEndTime - info.storageTimings.blobIOStartTime).milliseconds());
1126     if (info.storageTimings.synchronizationInProgressAtDispatch)
1127         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Synchronization was in progress");
1128     if (info.storageTimings.shrinkInProgressAtDispatch)
1129         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Shrink was in progress");
1130     if (info.storageTimings.wasCanceled)
1131         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Retrieve was canceled");
1132 #endif
1133 }
1134
1135 } // namespace WebKit