[WTF] Add makeUnique<T>, which ensures T is fast-allocated, makeUnique / makeUniqueWi...
[WebKit-https.git] / Source / WebKit / NetworkProcess / NetworkResourceLoader.cpp
1 /*
2  * Copyright (C) 2012-2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkResourceLoader.h"
28
29 #include "DataReference.h"
30 #include "FormDataReference.h"
31 #include "Logging.h"
32 #include "NetworkCache.h"
33 #include "NetworkConnectionToWebProcess.h"
34 #include "NetworkLoad.h"
35 #include "NetworkLoadChecker.h"
36 #include "NetworkProcess.h"
37 #include "NetworkProcessConnectionMessages.h"
38 #include "NetworkSession.h"
39 #include "SharedBufferDataReference.h"
40 #include "WebCoreArgumentCoders.h"
41 #include "WebErrors.h"
42 #include "WebPageMessages.h"
43 #include "WebResourceLoaderMessages.h"
44 #include "WebsiteDataStoreParameters.h"
45 #include <WebCore/BlobDataFileReference.h>
46 #include <WebCore/CertificateInfo.h>
47 #include <WebCore/ContentSecurityPolicy.h>
48 #include <WebCore/DiagnosticLoggingKeys.h>
49 #include <WebCore/HTTPParsers.h>
50 #include <WebCore/NetworkLoadMetrics.h>
51 #include <WebCore/NetworkStorageSession.h>
52 #include <WebCore/RegistrableDomain.h>
53 #include <WebCore/SameSiteInfo.h>
54 #include <WebCore/SecurityOrigin.h>
55 #include <WebCore/SharedBuffer.h>
56 #include <wtf/RunLoop.h>
57
58 #if USE(QUICK_LOOK)
59 #include <WebCore/PreviewConverter.h>
60 #endif
61
62 #define RELEASE_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__)
63 #define RELEASE_LOG_ERROR_IF_ALLOWED(fmt, ...) RELEASE_LOG_ERROR_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__)
64
65 namespace WebKit {
66 using namespace WebCore;
67
68 struct NetworkResourceLoader::SynchronousLoadData {
69     WTF_MAKE_STRUCT_FAST_ALLOCATED;
70
71     SynchronousLoadData(Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply&& reply)
72         : delayedReply(WTFMove(reply))
73     {
74         ASSERT(delayedReply);
75     }
76     ResourceRequest currentRequest;
77     Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply delayedReply;
78     ResourceResponse response;
79     ResourceError error;
80 };
81
82 static void sendReplyToSynchronousRequest(NetworkResourceLoader::SynchronousLoadData& data, const SharedBuffer* buffer)
83 {
84     ASSERT(data.delayedReply);
85     ASSERT(!data.response.isNull() || !data.error.isNull());
86
87     Vector<char> responseBuffer;
88     if (buffer && buffer->size())
89         responseBuffer.append(buffer->data(), buffer->size());
90
91     data.delayedReply(data.error, data.response, responseBuffer);
92     data.delayedReply = nullptr;
93 }
94
95 NetworkResourceLoader::NetworkResourceLoader(NetworkResourceLoadParameters&& parameters, NetworkConnectionToWebProcess& connection, Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply&& synchronousReply)
96     : m_parameters { WTFMove(parameters) }
97     , m_connection { connection }
98     , m_fileReferences(connection.resolveBlobReferences(m_parameters))
99     , m_isAllowedToAskUserForCredentials { m_parameters.clientCredentialPolicy == ClientCredentialPolicy::MayAskClientForCredentials }
100     , m_bufferingTimer { *this, &NetworkResourceLoader::bufferingTimerFired }
101     , m_shouldCaptureExtraNetworkLoadMetrics(m_connection->captureExtraNetworkLoadMetricsEnabled())
102 {
103     ASSERT(RunLoop::isMain());
104
105     if (auto* session = connection.networkProcess().networkSession(sessionID()))
106         m_cache = session->cache();
107
108     // FIXME: This is necessary because of the existence of EmptyFrameLoaderClient in WebCore.
109     //        Once bug 116233 is resolved, this ASSERT can just be "m_webPageID && m_webFrameID"
110     ASSERT((m_parameters.webPageID && m_parameters.webFrameID) || m_parameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials);
111
112     if (synchronousReply || parameters.shouldRestrictHTTPResponseAccess || parameters.options.keepAlive) {
113         NetworkLoadChecker::LoadType requestLoadType = isMainFrameLoad() ? NetworkLoadChecker::LoadType::MainFrame : NetworkLoadChecker::LoadType::Other;
114         m_networkLoadChecker = makeUnique<NetworkLoadChecker>(connection.networkProcess(), FetchOptions { m_parameters.options }, m_parameters.sessionID, m_parameters.webPageID, m_parameters.webFrameID, HTTPHeaderMap { m_parameters.originalRequestHeaders }, URL { m_parameters.request.url() }, m_parameters.sourceOrigin.copyRef(), m_parameters.preflightPolicy, originalRequest().httpReferrer(), m_parameters.isHTTPSUpgradeEnabled, shouldCaptureExtraNetworkLoadMetrics(), requestLoadType);
115         if (m_parameters.cspResponseHeaders)
116             m_networkLoadChecker->setCSPResponseHeaders(ContentSecurityPolicyResponseHeaders { m_parameters.cspResponseHeaders.value() });
117 #if ENABLE(CONTENT_EXTENSIONS)
118         m_networkLoadChecker->setContentExtensionController(URL { m_parameters.mainDocumentURL }, m_parameters.userContentControllerIdentifier);
119 #endif
120     }
121     if (synchronousReply)
122         m_synchronousLoadData = makeUnique<SynchronousLoadData>(WTFMove(synchronousReply));
123 }
124
125 NetworkResourceLoader::~NetworkResourceLoader()
126 {
127     ASSERT(RunLoop::isMain());
128     ASSERT(!m_networkLoad);
129     ASSERT(!isSynchronous() || !m_synchronousLoadData->delayedReply);
130     ASSERT(m_fileReferences.isEmpty());
131     if (m_responseCompletionHandler)
132         m_responseCompletionHandler(PolicyAction::Ignore);
133 }
134
135 bool NetworkResourceLoader::canUseCache(const ResourceRequest& request) const
136 {
137     if (!m_cache)
138         return false;
139     ASSERT(!sessionID().isEphemeral());
140
141     if (!request.url().protocolIsInHTTPFamily())
142         return false;
143     if (originalRequest().cachePolicy() == WebCore::ResourceRequestCachePolicy::DoNotUseAnyCache)
144         return false;
145
146     return true;
147 }
148
149 bool NetworkResourceLoader::canUseCachedRedirect(const ResourceRequest& request) const
150 {
151     if (!canUseCache(request) || m_cacheEntryForMaxAgeCapValidation)
152         return false;
153     // Limit cached redirects to avoid cycles and other trouble.
154     // Networking layer follows over 30 redirects but caching that many seems unnecessary.
155     static const unsigned maximumCachedRedirectCount { 5 };
156     if (m_redirectCount > maximumCachedRedirectCount)
157         return false;
158
159     return true;
160 }
161
162 bool NetworkResourceLoader::isSynchronous() const
163 {
164     return !!m_synchronousLoadData;
165 }
166
167 void NetworkResourceLoader::start()
168 {
169     ASSERT(RunLoop::isMain());
170
171     m_networkActivityTracker = m_connection->startTrackingResourceLoad(m_parameters.webPageID, m_parameters.identifier, isMainResource(), sessionID());
172
173     ASSERT(!m_wasStarted);
174     m_wasStarted = true;
175
176     if (m_networkLoadChecker) {
177         m_networkLoadChecker->check(ResourceRequest { originalRequest() }, this, [this, weakThis = makeWeakPtr(*this)] (auto&& result) {
178             if (!weakThis)
179                 return;
180
181             WTF::switchOn(result,
182                 [this] (ResourceError& error) {
183                     RELEASE_LOG_IF_ALLOWED("start: error checking (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d, parentPID = %d, error.domain = %{public}s, error.code = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, this->isMainResource(), this->isSynchronous(), m_parameters.parentPID, error.domain().utf8().data(), error.errorCode());
184                     if (!error.isCancellation())
185                         this->didFailLoading(error);
186                 },
187                 [this] (NetworkLoadChecker::RedirectionTriplet& triplet) {
188                     this->m_isWaitingContinueWillSendRequestForCachedRedirect = true;
189                     this->willSendRedirectedRequest(WTFMove(triplet.request), WTFMove(triplet.redirectRequest), WTFMove(triplet.redirectResponse));
190                     RELEASE_LOG_IF_ALLOWED("start: synthetic redirect sent because request URL was modified (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d, parentPID = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, this->isMainResource(), this->isSynchronous(), m_parameters.parentPID);
191                 },
192                 [this] (ResourceRequest& request) {
193                     if (this->canUseCache(request)) {
194                         RELEASE_LOG_IF_ALLOWED("start: Checking cache for resource (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d, parentPID = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, this->isMainResource(), this->isSynchronous(), m_parameters.parentPID);
195                         this->retrieveCacheEntry(request);
196                         return;
197                     }
198
199                     this->startNetworkLoad(WTFMove(request), FirstLoad::Yes);
200                 }
201             );
202         });
203         return;
204     }
205     // FIXME: Remove that code path once m_networkLoadChecker is used for all network loads.
206     if (canUseCache(originalRequest())) {
207         RELEASE_LOG_IF_ALLOWED("start: Checking cache for resource (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d, parentPID = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, isMainResource(), isSynchronous(), m_parameters.parentPID);
208         retrieveCacheEntry(originalRequest());
209         return;
210     }
211
212     startNetworkLoad(ResourceRequest { originalRequest() }, FirstLoad::Yes);
213 }
214
215 void NetworkResourceLoader::retrieveCacheEntry(const ResourceRequest& request)
216 {
217     ASSERT(canUseCache(request));
218
219     auto protectedThis = makeRef(*this);
220     if (isMainFrameLoad()) {
221         ASSERT(m_parameters.options.mode == FetchOptions::Mode::Navigate);
222         if (auto* session = m_connection->networkProcess().networkSession(sessionID())) {
223             if (auto entry = session->prefetchCache().take(request.url())) {
224                 // FIXME: Deal with credentials (https://bugs.webkit.org/show_bug.cgi?id=200000)
225                 if (!entry->redirectRequest.isNull()) {
226                     auto cacheEntry = m_cache->makeRedirectEntry(request, entry->response, entry->redirectRequest);
227                     retrieveCacheEntryInternal(WTFMove(cacheEntry), ResourceRequest { request });
228                     auto maxAgeCap = validateCacheEntryForMaxAgeCapValidation(request, entry->redirectRequest, entry->response);
229                     m_cache->storeRedirect(request, entry->response, entry->redirectRequest, maxAgeCap);
230                     return;
231                 }
232                 auto buffer = entry->releaseBuffer();
233                 auto cacheEntry = m_cache->makeEntry(request, entry->response, buffer.copyRef());
234                 retrieveCacheEntryInternal(WTFMove(cacheEntry), ResourceRequest { request });
235                 m_cache->store(request, entry->response, WTFMove(buffer), nullptr);
236                 return;
237             }
238         }
239     }
240     m_cache->retrieve(request, { m_parameters.webPageID, m_parameters.webFrameID }, [this, weakThis = makeWeakPtr(*this), request = ResourceRequest { request }](auto entry, auto info) mutable {
241         if (!weakThis)
242             return;
243
244         logSlowCacheRetrieveIfNeeded(info);
245
246         if (!entry) {
247             RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource not in cache (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, isMainResource(), isSynchronous());
248             startNetworkLoad(WTFMove(request), FirstLoad::Yes);
249             return;
250         }
251         retrieveCacheEntryInternal(WTFMove(entry), WTFMove(request));
252     });
253 }
254
255 void NetworkResourceLoader::retrieveCacheEntryInternal(std::unique_ptr<NetworkCache::Entry>&& entry, WebCore::ResourceRequest&& request)
256 {
257 #if ENABLE(RESOURCE_LOAD_STATISTICS)
258     if (entry->hasReachedPrevalentResourceAgeCap()) {
259         RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource has reached prevalent resource age cap (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, isMainResource(), isSynchronous());
260         m_cacheEntryForMaxAgeCapValidation = WTFMove(entry);
261         ResourceRequest revalidationRequest = originalRequest();
262         startNetworkLoad(WTFMove(revalidationRequest), FirstLoad::Yes);
263         return;
264     }
265 #endif
266     if (entry->redirectRequest()) {
267         RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Handling redirect (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, isMainResource(), isSynchronous());
268         dispatchWillSendRequestForCacheEntry(WTFMove(request), WTFMove(entry));
269         return;
270     }
271     if (m_parameters.needsCertificateInfo && !entry->response().certificateInfo()) {
272         RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Resource does not have required certificate (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, isMainResource(), isSynchronous());
273         startNetworkLoad(WTFMove(request), FirstLoad::Yes);
274         return;
275     }
276     if (entry->needsValidation() || request.cachePolicy() == WebCore::ResourceRequestCachePolicy::RefreshAnyCacheData) {
277         RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Validating cache entry (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, isMainResource(), isSynchronous());
278         validateCacheEntry(WTFMove(entry));
279         return;
280     }
281     RELEASE_LOG_IF_ALLOWED("retrieveCacheEntry: Retrieved resource from cache (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, isMainResource(), isSynchronous());
282     didRetrieveCacheEntry(WTFMove(entry));
283 }
284
285 void NetworkResourceLoader::startNetworkLoad(ResourceRequest&& request, FirstLoad load)
286 {
287     if (load == FirstLoad::Yes) {
288         RELEASE_LOG_IF_ALLOWED("startNetworkLoad: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, isMainResource(), isSynchronous());
289
290         consumeSandboxExtensions();
291
292         if (isSynchronous() || m_parameters.maximumBufferingTime > 0_s)
293             m_bufferedData = SharedBuffer::create();
294
295         if (canUseCache(request))
296             m_bufferedDataForCache = SharedBuffer::create();
297     }
298
299     NetworkLoadParameters parameters = m_parameters;
300     parameters.networkActivityTracker = m_networkActivityTracker;
301     if (parameters.storedCredentialsPolicy == WebCore::StoredCredentialsPolicy::Use && m_networkLoadChecker)
302         parameters.storedCredentialsPolicy = m_networkLoadChecker->storedCredentialsPolicy();
303
304     auto* networkSession = m_connection->networkProcess().networkSession(parameters.sessionID);
305     if (!networkSession && parameters.sessionID.isEphemeral()) {
306         m_connection->networkProcess().addWebsiteDataStore(WebsiteDataStoreParameters::privateSessionParameters(parameters.sessionID));
307         networkSession = m_connection->networkProcess().networkSession(parameters.sessionID);
308     }
309     if (!networkSession) {
310         WTFLogAlways("Attempted to create a NetworkLoad with a session (id=%" PRIu64 ") that does not exist.", parameters.sessionID.sessionID());
311         RELEASE_LOG_ERROR_IF_ALLOWED("startNetworkLoad: Attempted to create a NetworkLoad with a session that does not exist (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", sessionID=%" PRIu64 ")", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, parameters.sessionID.sessionID());
312         m_connection->networkProcess().logDiagnosticMessage(m_parameters.webPageID, WebCore::DiagnosticLoggingKeys::internalErrorKey(), WebCore::DiagnosticLoggingKeys::invalidSessionIDKey(), WebCore::ShouldSample::No);
313         didFailLoading(internalError(request.url()));
314         return;
315     }
316
317     if (request.url().protocolIsBlob())
318         parameters.blobFileReferences = networkSession->blobRegistry().filesInBlob(originalRequest().url());
319
320     parameters.request = WTFMove(request);
321     m_networkLoad = makeUnique<NetworkLoad>(*this, &networkSession->blobRegistry(), WTFMove(parameters), *networkSession);
322
323     RELEASE_LOG_IF_ALLOWED("startNetworkLoad: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", description = %{public}s)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, m_networkLoad->description().utf8().data());
324 }
325
326 void NetworkResourceLoader::cleanup(LoadResult result)
327 {
328     ASSERT(RunLoop::isMain());
329
330     m_connection->stopTrackingResourceLoad(m_parameters.identifier,
331         result == LoadResult::Success ? NetworkActivityTracker::CompletionCode::Success :
332         result == LoadResult::Failure ? NetworkActivityTracker::CompletionCode::Failure :
333         NetworkActivityTracker::CompletionCode::None);
334
335     m_bufferingTimer.stop();
336
337     invalidateSandboxExtensions();
338
339     m_networkLoad = nullptr;
340
341     // This will cause NetworkResourceLoader to be destroyed and therefore we do it last.
342     m_connection->didCleanupResourceLoader(*this);
343 }
344
345 void NetworkResourceLoader::convertToDownload(DownloadID downloadID, const ResourceRequest& request, const ResourceResponse& response)
346 {
347     RELEASE_LOG(Loading, "Converting NetworkResourceLoader %p to download %" PRIu64 " (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", this, downloadID.downloadID(), m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier);
348     
349     // This can happen if the resource came from the disk cache.
350     if (!m_networkLoad) {
351         m_connection->networkProcess().downloadManager().startDownload(m_parameters.sessionID, downloadID, request);
352         abort();
353         return;
354     }
355
356     if (m_responseCompletionHandler)
357         m_connection->networkProcess().downloadManager().convertNetworkLoadToDownload(downloadID, std::exchange(m_networkLoad, nullptr), WTFMove(m_responseCompletionHandler), WTFMove(m_fileReferences), request, response);
358 }
359
360 void NetworkResourceLoader::abort()
361 {
362     ASSERT(RunLoop::isMain());
363
364     RELEASE_LOG_IF_ALLOWED("abort: Canceling resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")",
365         m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier);
366
367     if (m_parameters.options.keepAlive && m_response.isNull() && !m_isKeptAlive) {
368         m_isKeptAlive = true;
369         m_connection->transferKeptAliveLoad(*this);
370         return;
371     }
372
373     if (m_networkLoad) {
374         if (canUseCache(m_networkLoad->currentRequest())) {
375             // We might already have used data from this incomplete load. Ensure older versions don't remain in the cache after cancel.
376             if (!m_response.isNull())
377                 m_cache->remove(m_networkLoad->currentRequest());
378         }
379         m_networkLoad->cancel();
380     }
381
382     cleanup(LoadResult::Cancel);
383 }
384
385 bool NetworkResourceLoader::shouldInterruptLoadForXFrameOptions(const String& xFrameOptions, const URL& url)
386 {
387     if (isMainFrameLoad())
388         return false;
389
390     switch (parseXFrameOptionsHeader(xFrameOptions)) {
391     case XFrameOptionsNone:
392     case XFrameOptionsAllowAll:
393         return false;
394     case XFrameOptionsDeny:
395         return true;
396     case XFrameOptionsSameOrigin: {
397         auto origin = SecurityOrigin::create(url);
398         auto topFrameOrigin = m_parameters.frameAncestorOrigins.last();
399         if (!origin->isSameSchemeHostPort(*topFrameOrigin))
400             return true;
401         for (auto& ancestorOrigin : m_parameters.frameAncestorOrigins) {
402             if (!origin->isSameSchemeHostPort(*ancestorOrigin))
403                 return true;
404         }
405         return false;
406     }
407     case XFrameOptionsConflict: {
408         String errorMessage = "Multiple 'X-Frame-Options' headers with conflicting values ('" + xFrameOptions + "') encountered when loading '" + url.stringCenterEllipsizedToLength() + "'. Falling back to 'DENY'.";
409         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::JS, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
410         return true;
411     }
412     case XFrameOptionsInvalid: {
413         String errorMessage = "Invalid 'X-Frame-Options' header encountered when loading '" + url.stringCenterEllipsizedToLength() + "': '" + xFrameOptions + "' is not a recognized directive. The header will be ignored.";
414         send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::JS, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
415         return false;
416     }
417     }
418     ASSERT_NOT_REACHED();
419     return false;
420 }
421
422 bool NetworkResourceLoader::shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(const ResourceResponse& response)
423 {
424     ASSERT(isMainResource());
425
426 #if USE(QUICK_LOOK)
427     if (PreviewConverter::supportsMIMEType(response.mimeType()))
428         return false;
429 #endif
430
431     auto url = response.url();
432     ContentSecurityPolicy contentSecurityPolicy { URL { url }, this };
433     contentSecurityPolicy.didReceiveHeaders(ContentSecurityPolicyResponseHeaders { response }, originalRequest().httpReferrer());
434     if (!contentSecurityPolicy.allowFrameAncestors(m_parameters.frameAncestorOrigins, url))
435         return true;
436
437     if (!contentSecurityPolicy.overridesXFrameOptions()) {
438         String xFrameOptions = m_response.httpHeaderField(HTTPHeaderName::XFrameOptions);
439         if (!xFrameOptions.isNull() && shouldInterruptLoadForXFrameOptions(xFrameOptions, response.url())) {
440             String errorMessage = "Refused to display '" + response.url().stringCenterEllipsizedToLength() + "' in a frame because it set 'X-Frame-Options' to '" + xFrameOptions + "'.";
441             send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  MessageSource::Security, MessageLevel::Error, errorMessage, identifier() }, m_parameters.webPageID);
442             return true;
443         }
444     }
445     return false;
446 }
447
448 void NetworkResourceLoader::didReceiveResponse(ResourceResponse&& receivedResponse, ResponseCompletionHandler&& completionHandler)
449 {
450     RELEASE_LOG_IF_ALLOWED("didReceiveResponse: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", httpStatusCode = %d, length = %" PRId64 ")", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, receivedResponse.httpStatusCode(), receivedResponse.expectedContentLength());
451
452     m_response = WTFMove(receivedResponse);
453
454     if (shouldCaptureExtraNetworkLoadMetrics() && m_networkLoadChecker) {
455         auto information = m_networkLoadChecker->takeNetworkLoadInformation();
456         information.response = m_response;
457         m_connection->addNetworkLoadInformation(identifier(), WTFMove(information));
458     }
459
460     // For multipart/x-mixed-replace didReceiveResponseAsync gets called multiple times and buffering would require special handling.
461     if (!isSynchronous() && m_response.isMultipart())
462         m_bufferedData = nullptr;
463
464     if (m_response.isMultipart())
465         m_bufferedDataForCache = nullptr;
466
467     if (m_cacheEntryForValidation) {
468         bool validationSucceeded = m_response.httpStatusCode() == 304; // 304 Not Modified
469         if (validationSucceeded) {
470             m_cacheEntryForValidation = m_cache->update(originalRequest(), { m_parameters.webPageID, m_parameters.webFrameID }, *m_cacheEntryForValidation, m_response);
471             // If the request was conditional then this revalidation was not triggered by the network cache and we pass the 304 response to WebCore.
472             if (originalRequest().isConditional())
473                 m_cacheEntryForValidation = nullptr;
474         } else
475             m_cacheEntryForValidation = nullptr;
476     }
477     if (m_cacheEntryForValidation)
478         return completionHandler(PolicyAction::Use);
479
480     if (isMainResource() && shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(m_response)) {
481         auto response = sanitizeResponseIfPossible(ResourceResponse { m_response }, ResourceResponse::SanitizationType::CrossOriginSafe);
482         send(Messages::WebResourceLoader::StopLoadingAfterXFrameOptionsOrContentSecurityPolicyDenied { response });
483         return completionHandler(PolicyAction::Ignore);
484     }
485
486     if (m_networkLoadChecker) {
487         auto error = m_networkLoadChecker->validateResponse(m_response);
488         if (!error.isNull()) {
489             RunLoop::main().dispatch([protectedThis = makeRef(*this), error = WTFMove(error)] {
490                 if (protectedThis->m_networkLoad)
491                     protectedThis->didFailLoading(error);
492             });
493             return completionHandler(PolicyAction::Ignore);
494         }
495     }
496
497     auto response = sanitizeResponseIfPossible(ResourceResponse { m_response }, ResourceResponse::SanitizationType::CrossOriginSafe);
498     if (isSynchronous()) {
499         m_synchronousLoadData->response = WTFMove(response);
500         return completionHandler(PolicyAction::Use);
501     }
502
503     if (isCrossOriginPrefetch())
504         return completionHandler(PolicyAction::Use);
505
506     // We wait to receive message NetworkResourceLoader::ContinueDidReceiveResponse before continuing a load for
507     // a main resource because the embedding client must decide whether to allow the load.
508     bool willWaitForContinueDidReceiveResponse = isMainResource();
509     send(Messages::WebResourceLoader::DidReceiveResponse { response, willWaitForContinueDidReceiveResponse });
510
511     if (willWaitForContinueDidReceiveResponse) {
512         m_responseCompletionHandler = WTFMove(completionHandler);
513         return;
514     }
515
516     if (m_isKeptAlive) {
517         m_responseCompletionHandler = WTFMove(completionHandler);
518         RunLoop::main().dispatch([protectedThis = makeRef(*this)] {
519             protectedThis->didFinishLoading(NetworkLoadMetrics { });
520         });
521         return;
522     }
523
524     completionHandler(PolicyAction::Use);
525 }
526
527 void NetworkResourceLoader::didReceiveBuffer(Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength)
528 {
529     if (!m_numBytesReceived)
530         RELEASE_LOG_IF_ALLOWED("didReceiveBuffer: Started receiving data (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier);
531     m_numBytesReceived += buffer->size();
532
533     ASSERT(!m_cacheEntryForValidation);
534
535     if (m_bufferedDataForCache) {
536         // Prevent memory growth in case of streaming data.
537         const size_t maximumCacheBufferSize = 10 * 1024 * 1024;
538         if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize)
539             m_bufferedDataForCache->append(buffer.get());
540         else
541             m_bufferedDataForCache = nullptr;
542     }
543     if (isCrossOriginPrefetch())
544         return;
545     // FIXME: At least on OS X Yosemite we always get -1 from the resource handle.
546     unsigned encodedDataLength = reportedEncodedDataLength >= 0 ? reportedEncodedDataLength : buffer->size();
547
548     if (m_bufferedData) {
549         m_bufferedData->append(buffer.get());
550         m_bufferedDataEncodedDataLength += encodedDataLength;
551         startBufferingTimerIfNeeded();
552         return;
553     }
554     sendBuffer(buffer, encodedDataLength);
555 }
556
557 void NetworkResourceLoader::didFinishLoading(const NetworkLoadMetrics& networkLoadMetrics)
558 {
559     RELEASE_LOG_IF_ALLOWED("didFinishLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", length = %zd)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, m_numBytesReceived);
560
561     if (shouldCaptureExtraNetworkLoadMetrics())
562         m_connection->addNetworkLoadInformationMetrics(identifier(), networkLoadMetrics);
563
564     if (m_cacheEntryForValidation) {
565         // 304 Not Modified
566         ASSERT(m_response.httpStatusCode() == 304);
567         LOG(NetworkCache, "(NetworkProcess) revalidated");
568         didRetrieveCacheEntry(WTFMove(m_cacheEntryForValidation));
569         return;
570     }
571
572 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
573     if (shouldLogCookieInformation(m_connection, sessionID()))
574         logCookieInformation();
575 #endif
576
577     if (isSynchronous())
578         sendReplyToSynchronousRequest(*m_synchronousLoadData, m_bufferedData.get());
579     else {
580         if (m_bufferedData && !m_bufferedData->isEmpty()) {
581             // FIXME: Pass a real value or remove the encoded data size feature.
582             sendBuffer(*m_bufferedData, -1);
583         }
584         send(Messages::WebResourceLoader::DidFinishResourceLoad(networkLoadMetrics));
585     }
586
587     tryStoreAsCacheEntry();
588
589     cleanup(LoadResult::Success);
590 }
591
592 void NetworkResourceLoader::didFailLoading(const ResourceError& error)
593 {
594     RELEASE_LOG_IF_ALLOWED("didFailLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isTimeout = %d, isCancellation = %d, isAccessControl = %d, errCode = %d)", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier, error.isTimeout(), error.isCancellation(), error.isAccessControl(), error.errorCode());
595
596     if (shouldCaptureExtraNetworkLoadMetrics())
597         m_connection->removeNetworkLoadInformation(identifier());
598
599     ASSERT(!error.isNull());
600
601     m_cacheEntryForValidation = nullptr;
602
603     if (isSynchronous()) {
604         m_synchronousLoadData->error = error;
605         sendReplyToSynchronousRequest(*m_synchronousLoadData, nullptr);
606     } else if (auto* connection = messageSenderConnection())
607         connection->send(Messages::WebResourceLoader::DidFailResourceLoad(error), messageSenderDestinationID());
608
609     cleanup(LoadResult::Failure);
610 }
611
612 void NetworkResourceLoader::didBlockAuthenticationChallenge()
613 {
614     send(Messages::WebResourceLoader::DidBlockAuthenticationChallenge());
615 }
616
617 Optional<Seconds> NetworkResourceLoader::validateCacheEntryForMaxAgeCapValidation(const ResourceRequest& request, const ResourceRequest& redirectRequest, const ResourceResponse& redirectResponse)
618 {
619 #if ENABLE(RESOURCE_LOAD_STATISTICS)
620     bool existingCacheEntryMatchesNewResponse = false;
621     if (m_cacheEntryForMaxAgeCapValidation) {
622         ASSERT(redirectResponse.source() == ResourceResponse::Source::Network);
623         ASSERT(redirectResponse.isRedirection());
624         if (redirectResponse.httpHeaderField(WebCore::HTTPHeaderName::Location) == m_cacheEntryForMaxAgeCapValidation->response().httpHeaderField(WebCore::HTTPHeaderName::Location))
625             existingCacheEntryMatchesNewResponse = true;
626
627         m_cache->remove(m_cacheEntryForMaxAgeCapValidation->key());
628         m_cacheEntryForMaxAgeCapValidation = nullptr;
629     }
630     
631     if (!existingCacheEntryMatchesNewResponse) {
632         if (auto* networkStorageSession = m_connection->networkProcess().storageSession(sessionID()))
633             return networkStorageSession->maxAgeCacheCap(request);
634     }
635 #endif
636     return WTF::nullopt;
637 }
638
639 void NetworkResourceLoader::willSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse)
640 {
641     ++m_redirectCount;
642
643     Optional<AdClickAttribution::Conversion> adClickConversion;
644     if (!sessionID().isEphemeral())
645         adClickConversion = AdClickAttribution::parseConversionRequest(redirectRequest.url());
646
647     auto maxAgeCap = validateCacheEntryForMaxAgeCapValidation(request, redirectRequest, redirectResponse);
648     if (redirectResponse.source() == ResourceResponse::Source::Network && canUseCachedRedirect(request))
649         m_cache->storeRedirect(request, redirectResponse, redirectRequest, maxAgeCap);
650
651     if (m_networkLoadChecker) {
652         if (adClickConversion)
653             m_networkLoadChecker->enableContentExtensionsCheck();
654         m_networkLoadChecker->storeRedirectionIfNeeded(request, redirectResponse);
655         m_networkLoadChecker->checkRedirection(WTFMove(request), WTFMove(redirectRequest), WTFMove(redirectResponse), this, [protectedThis = makeRef(*this), this, storedCredentialsPolicy = m_networkLoadChecker->storedCredentialsPolicy(), adClickConversion = WTFMove(adClickConversion)](auto&& result) mutable {
656             if (!result.has_value()) {
657                 if (result.error().isCancellation())
658                     return;
659
660                 this->didFailLoading(result.error());
661                 return;
662             }
663
664             if (m_parameters.options.redirect == FetchOptions::Redirect::Manual) {
665                 this->didFinishWithRedirectResponse(WTFMove(result->request), WTFMove(result->redirectRequest), WTFMove(result->redirectResponse));
666                 return;
667             }
668
669             if (this->isSynchronous()) {
670                 if (storedCredentialsPolicy != m_networkLoadChecker->storedCredentialsPolicy()) {
671                     // We need to restart the load to update the session according the new credential policy.
672                     this->restartNetworkLoad(WTFMove(result->redirectRequest));
673                     return;
674                 }
675
676                 // We do not support prompting for credentials for synchronous loads. If we ever change this policy then
677                 // we need to take care to prompt if and only if request and redirectRequest are not mixed content.
678                 this->continueWillSendRequest(WTFMove(result->redirectRequest), false);
679                 return;
680             }
681
682             m_shouldRestartLoad = storedCredentialsPolicy != m_networkLoadChecker->storedCredentialsPolicy();
683             this->continueWillSendRedirectedRequest(WTFMove(result->request), WTFMove(result->redirectRequest), WTFMove(result->redirectResponse), WTFMove(adClickConversion));
684         });
685         return;
686     }
687     continueWillSendRedirectedRequest(WTFMove(request), WTFMove(redirectRequest), WTFMove(redirectResponse), WTFMove(adClickConversion));
688 }
689
690 void NetworkResourceLoader::continueWillSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse, Optional<AdClickAttribution::Conversion>&& adClickConversion)
691 {
692     ASSERT(!isSynchronous());
693
694     if (m_isKeptAlive) {
695         continueWillSendRequest(WTFMove(redirectRequest), false);
696         return;
697     }
698
699     NetworkSession* networkSession = nullptr;
700     if (adClickConversion && (networkSession = m_connection->networkProcess().networkSession(sessionID())))
701         networkSession->handleAdClickAttributionConversion(WTFMove(*adClickConversion), request.url(), redirectRequest);
702
703     send(Messages::WebResourceLoader::WillSendRequest(redirectRequest, sanitizeResponseIfPossible(WTFMove(redirectResponse), ResourceResponse::SanitizationType::Redirection)));
704 }
705
706 void NetworkResourceLoader::didFinishWithRedirectResponse(WebCore::ResourceRequest&& request, WebCore::ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse)
707 {
708     redirectResponse.setType(ResourceResponse::Type::Opaqueredirect);
709     if (!isCrossOriginPrefetch())
710         didReceiveResponse(WTFMove(redirectResponse), [] (auto) { });
711     else if (auto* session = m_connection->networkProcess().networkSession(sessionID()))
712         session->prefetchCache().storeRedirect(m_networkLoad->currentRequest().url(), WTFMove(redirectResponse), WTFMove(redirectRequest));
713
714     WebCore::NetworkLoadMetrics networkLoadMetrics;
715     networkLoadMetrics.markComplete();
716     networkLoadMetrics.responseBodyBytesReceived = 0;
717     networkLoadMetrics.responseBodyDecodedSize = 0;
718     send(Messages::WebResourceLoader::DidFinishResourceLoad { networkLoadMetrics });
719
720     cleanup(LoadResult::Success);
721 }
722
723 ResourceResponse NetworkResourceLoader::sanitizeResponseIfPossible(ResourceResponse&& response, ResourceResponse::SanitizationType type)
724 {
725     if (m_parameters.shouldRestrictHTTPResponseAccess)
726         response.sanitizeHTTPHeaderFields(type);
727
728     return WTFMove(response);
729 }
730
731 void NetworkResourceLoader::restartNetworkLoad(WebCore::ResourceRequest&& newRequest)
732 {
733     if (m_networkLoad)
734         m_networkLoad->cancel();
735
736     startNetworkLoad(WTFMove(newRequest), FirstLoad::No);
737 }
738
739 void NetworkResourceLoader::continueWillSendRequest(ResourceRequest&& newRequest, bool isAllowedToAskUserForCredentials)
740 {
741     if (m_shouldRestartLoad) {
742         m_shouldRestartLoad = false;
743
744         if (m_networkLoad)
745             m_networkLoad->updateRequestAfterRedirection(newRequest);
746
747         restartNetworkLoad(WTFMove(newRequest));
748         return;
749     }
750
751     if (m_networkLoadChecker) {
752         // FIXME: We should be doing this check when receiving the redirection and not allow about protocol as per fetch spec.
753         if (!newRequest.url().protocolIsInHTTPFamily() && !newRequest.url().protocolIsAbout() && m_redirectCount) {
754             didFailLoading(ResourceError { String { }, 0, newRequest.url(), "Redirection to URL with a scheme that is not HTTP(S)"_s, ResourceError::Type::AccessControl });
755             return;
756         }
757     }
758
759     RELEASE_LOG_IF_ALLOWED("continueWillSendRequest: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID.toUInt64(), m_parameters.webFrameID.toUInt64(), m_parameters.identifier);
760
761     m_isAllowedToAskUserForCredentials = isAllowedToAskUserForCredentials;
762
763     // If there is a match in the network cache, we need to reuse the original cache policy and partition.
764     newRequest.setCachePolicy(originalRequest().cachePolicy());
765     newRequest.setCachePartition(originalRequest().cachePartition());
766
767     if (m_isWaitingContinueWillSendRequestForCachedRedirect) {
768         m_isWaitingContinueWillSendRequestForCachedRedirect = false;
769
770         LOG(NetworkCache, "(NetworkProcess) Retrieving cached redirect");
771
772         if (canUseCachedRedirect(newRequest))
773             retrieveCacheEntry(newRequest);
774         else
775             startNetworkLoad(WTFMove(newRequest), FirstLoad::Yes);
776
777         return;
778     }
779
780     if (m_networkLoad)
781         m_networkLoad->continueWillSendRequest(WTFMove(newRequest));
782 }
783
784 void NetworkResourceLoader::continueDidReceiveResponse()
785 {
786     if (m_cacheEntryWaitingForContinueDidReceiveResponse) {
787         sendResultForCacheEntry(WTFMove(m_cacheEntryWaitingForContinueDidReceiveResponse));
788         cleanup(LoadResult::Success);
789         return;
790     }
791
792     if (m_responseCompletionHandler)
793         m_responseCompletionHandler(PolicyAction::Use);
794 }
795
796 void NetworkResourceLoader::didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent)
797 {
798     if (!isSynchronous())
799         send(Messages::WebResourceLoader::DidSendData(bytesSent, totalBytesToBeSent));
800 }
801
802 void NetworkResourceLoader::startBufferingTimerIfNeeded()
803 {
804     if (isSynchronous())
805         return;
806     if (m_bufferingTimer.isActive())
807         return;
808     m_bufferingTimer.startOneShot(m_parameters.maximumBufferingTime);
809 }
810
811 void NetworkResourceLoader::bufferingTimerFired()
812 {
813     ASSERT(m_bufferedData);
814     ASSERT(m_networkLoad);
815
816     if (m_bufferedData->isEmpty())
817         return;
818
819     send(Messages::WebResourceLoader::DidReceiveData({ *m_bufferedData }, m_bufferedDataEncodedDataLength));
820
821     m_bufferedData = SharedBuffer::create();
822     m_bufferedDataEncodedDataLength = 0;
823 }
824
825 void NetworkResourceLoader::sendBuffer(SharedBuffer& buffer, size_t encodedDataLength)
826 {
827     ASSERT(!isSynchronous());
828
829     send(Messages::WebResourceLoader::DidReceiveData({ buffer }, encodedDataLength));
830 }
831
832 void NetworkResourceLoader::tryStoreAsCacheEntry()
833 {
834     if (!canUseCache(m_networkLoad->currentRequest()))
835         return;
836     if (!m_bufferedDataForCache)
837         return;
838
839     if (isCrossOriginPrefetch()) {
840         if (auto* session = m_connection->networkProcess().networkSession(sessionID()))
841             session->prefetchCache().store(m_networkLoad->currentRequest().url(), WTFMove(m_response), WTFMove(m_bufferedDataForCache));
842         return;
843     }
844     m_cache->store(m_networkLoad->currentRequest(), m_response, WTFMove(m_bufferedDataForCache), [loader = makeRef(*this)](auto& mappedBody) mutable {
845 #if ENABLE(SHAREABLE_RESOURCE)
846         if (mappedBody.shareableResourceHandle.isNull())
847             return;
848         LOG(NetworkCache, "(NetworkProcess) sending DidCacheResource");
849         loader->send(Messages::NetworkProcessConnection::DidCacheResource(loader->originalRequest(), mappedBody.shareableResourceHandle, loader->sessionID()));
850 #endif
851     });
852 }
853
854 void NetworkResourceLoader::didRetrieveCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
855 {
856     auto response = entry->response();
857
858     if (isMainResource() && shouldInterruptLoadForCSPFrameAncestorsOrXFrameOptions(response)) {
859         response = sanitizeResponseIfPossible(WTFMove(response), ResourceResponse::SanitizationType::CrossOriginSafe);
860         send(Messages::WebResourceLoader::StopLoadingAfterXFrameOptionsOrContentSecurityPolicyDenied { response });
861         return;
862     }
863     if (m_networkLoadChecker) {
864         auto error = m_networkLoadChecker->validateResponse(response);
865         if (!error.isNull()) {
866             didFailLoading(error);
867             return;
868         }
869     }
870
871     response = sanitizeResponseIfPossible(WTFMove(response), ResourceResponse::SanitizationType::CrossOriginSafe);
872     if (isSynchronous()) {
873         m_synchronousLoadData->response = WTFMove(response);
874         sendReplyToSynchronousRequest(*m_synchronousLoadData, entry->buffer());
875         cleanup(LoadResult::Success);
876         return;
877     }
878
879     bool needsContinueDidReceiveResponseMessage = isMainResource();
880     send(Messages::WebResourceLoader::DidReceiveResponse { response, needsContinueDidReceiveResponseMessage });
881
882     if (needsContinueDidReceiveResponseMessage)
883         m_cacheEntryWaitingForContinueDidReceiveResponse = WTFMove(entry);
884     else {
885         sendResultForCacheEntry(WTFMove(entry));
886         cleanup(LoadResult::Success);
887     }
888 }
889
890 void NetworkResourceLoader::sendResultForCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
891 {
892 #if ENABLE(SHAREABLE_RESOURCE)
893     if (!entry->shareableResourceHandle().isNull()) {
894         send(Messages::WebResourceLoader::DidReceiveResource(entry->shareableResourceHandle()));
895         return;
896     }
897 #endif
898
899 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
900     if (shouldLogCookieInformation(m_connection, sessionID()))
901         logCookieInformation();
902 #endif
903
904     WebCore::NetworkLoadMetrics networkLoadMetrics;
905     networkLoadMetrics.markComplete();
906     networkLoadMetrics.requestHeaderBytesSent = 0;
907     networkLoadMetrics.requestBodyBytesSent = 0;
908     networkLoadMetrics.responseHeaderBytesReceived = 0;
909     networkLoadMetrics.responseBodyBytesReceived = 0;
910     networkLoadMetrics.responseBodyDecodedSize = 0;
911
912     sendBuffer(*entry->buffer(), entry->buffer()->size());
913     send(Messages::WebResourceLoader::DidFinishResourceLoad(networkLoadMetrics));
914 }
915
916 void NetworkResourceLoader::validateCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
917 {
918     ASSERT(!m_networkLoad);
919
920     // If the request is already conditional then the revalidation was not triggered by the disk cache
921     // and we should not overwrite the existing conditional headers.
922     ResourceRequest revalidationRequest = originalRequest();
923     if (!revalidationRequest.isConditional()) {
924         String eTag = entry->response().httpHeaderField(HTTPHeaderName::ETag);
925         String lastModified = entry->response().httpHeaderField(HTTPHeaderName::LastModified);
926         if (!eTag.isEmpty())
927             revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfNoneMatch, eTag);
928         if (!lastModified.isEmpty())
929             revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfModifiedSince, lastModified);
930     }
931
932     m_cacheEntryForValidation = WTFMove(entry);
933
934     startNetworkLoad(WTFMove(revalidationRequest), FirstLoad::Yes);
935 }
936
937 void NetworkResourceLoader::dispatchWillSendRequestForCacheEntry(ResourceRequest&& request, std::unique_ptr<NetworkCache::Entry>&& entry)
938 {
939     ASSERT(entry->redirectRequest());
940     ASSERT(!m_isWaitingContinueWillSendRequestForCachedRedirect);
941
942     LOG(NetworkCache, "(NetworkProcess) Executing cached redirect");
943
944     m_isWaitingContinueWillSendRequestForCachedRedirect = true;
945     willSendRedirectedRequest(WTFMove(request), ResourceRequest { *entry->redirectRequest() }, ResourceResponse { entry->response() });
946 }
947
948 IPC::Connection* NetworkResourceLoader::messageSenderConnection() const
949 {
950     return &connectionToWebProcess().connection();
951 }
952
953 void NetworkResourceLoader::consumeSandboxExtensions()
954 {
955     ASSERT(!m_didConsumeSandboxExtensions);
956
957     for (auto& extension : m_parameters.requestBodySandboxExtensions)
958         extension->consume();
959
960     if (auto& extension = m_parameters.resourceSandboxExtension)
961         extension->consume();
962
963     for (auto& fileReference : m_fileReferences)
964         fileReference->prepareForFileAccess();
965
966     m_didConsumeSandboxExtensions = true;
967 }
968
969 void NetworkResourceLoader::invalidateSandboxExtensions()
970 {
971     if (m_didConsumeSandboxExtensions) {
972         for (auto& extension : m_parameters.requestBodySandboxExtensions)
973             extension->revoke();
974         if (auto& extension = m_parameters.resourceSandboxExtension)
975             extension->revoke();
976         for (auto& fileReference : m_fileReferences)
977             fileReference->revokeFileAccess();
978
979         m_didConsumeSandboxExtensions = false;
980     }
981
982     m_fileReferences.clear();
983 }
984
985 bool NetworkResourceLoader::isAlwaysOnLoggingAllowed() const
986 {
987     if (m_connection->networkProcess().sessionIsControlledByAutomation(sessionID()))
988         return true;
989
990     return sessionID().isAlwaysOnLoggingAllowed();
991 }
992
993 bool NetworkResourceLoader::shouldCaptureExtraNetworkLoadMetrics() const
994 {
995     return m_shouldCaptureExtraNetworkLoadMetrics;
996 }
997
998 #if ENABLE(RESOURCE_LOAD_STATISTICS) && !RELEASE_LOG_DISABLED
999 bool NetworkResourceLoader::shouldLogCookieInformation(NetworkConnectionToWebProcess& connection, const PAL::SessionID& sessionID)
1000 {
1001     if (auto* session = connection.networkProcess().networkSession(sessionID))
1002         return session->shouldLogCookieInformation();
1003     return false;
1004 }
1005
1006 static String escapeForJSON(String s)
1007 {
1008     return s.replace('\\', "\\\\").replace('"', "\\\"");
1009 }
1010
1011 static String escapeIDForJSON(const Optional<uint64_t>& value)
1012 {
1013     return value ? String::number(value.value()) : String("None"_s);
1014 }
1015
1016 static String escapeIDForJSON(const Optional<FrameIdentifier>& value)
1017 {
1018     return value ? String::number(value->toUInt64()) : String("None"_s);
1019 }
1020
1021 static String escapeIDForJSON(const Optional<PageIdentifier>& value)
1022 {
1023     return value ? String::number(value->toUInt64()) : String("None"_s);
1024 }
1025
1026 void NetworkResourceLoader::logCookieInformation() const
1027 {
1028     ASSERT(shouldLogCookieInformation(m_connection, sessionID()));
1029
1030     auto* networkStorageSession = m_connection->networkProcess().storageSession(sessionID());
1031     ASSERT(networkStorageSession);
1032
1033     logCookieInformation(m_connection, "NetworkResourceLoader", reinterpret_cast<const void*>(this), *networkStorageSession, originalRequest().firstPartyForCookies(), SameSiteInfo::create(originalRequest()), originalRequest().url(), originalRequest().httpReferrer(), frameID(), pageID(), identifier());
1034 }
1035
1036 static void logBlockedCookieInformation(NetworkConnectionToWebProcess& connection, const String& label, const void* loggedObject, const WebCore::NetworkStorageSession& networkStorageSession, const URL& firstParty, const SameSiteInfo& sameSiteInfo, const URL& url, const String& referrer, Optional<FrameIdentifier> frameID, Optional<PageIdentifier> pageID, Optional<uint64_t> identifier)
1037 {
1038     ASSERT(NetworkResourceLoader::shouldLogCookieInformation(connection, networkStorageSession.sessionID()));
1039
1040     auto escapedURL = escapeForJSON(url.string());
1041     auto escapedFirstParty = escapeForJSON(firstParty.string());
1042     auto escapedFrameID = escapeIDForJSON(frameID);
1043     auto escapedPageID = escapeIDForJSON(pageID);
1044     auto escapedIdentifier = escapeIDForJSON(identifier);
1045     auto escapedReferrer = escapeForJSON(referrer);
1046
1047 #define LOCAL_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(networkStorageSession.sessionID().isAlwaysOnLoggingAllowed(), Network, "%p - %s::" fmt, loggedObject, label.utf8().data(), ##__VA_ARGS__)
1048 #define LOCAL_LOG(str, ...) \
1049     LOCAL_LOG_IF_ALLOWED("logCookieInformation: BLOCKED cookie access for pageID = %s, frameID = %s, resourceID = %s, firstParty = %s: " str, escapedPageID.utf8().data(), escapedFrameID.utf8().data(), escapedIdentifier.utf8().data(), escapedFirstParty.utf8().data(), ##__VA_ARGS__)
1050
1051     LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
1052     LOCAL_LOG(R"(  "partition": "%{public}s",)", "BLOCKED");
1053     LOCAL_LOG(R"(  "hasStorageAccess": %{public}s,)", "false");
1054     LOCAL_LOG(R"(  "referer": "%{public}s",)", escapedReferrer.utf8().data());
1055     LOCAL_LOG(R"(  "isSameSite": "%{public}s",)", sameSiteInfo.isSameSite ? "true" : "false");
1056     LOCAL_LOG(R"(  "isTopSite": "%{public}s",)", sameSiteInfo.isTopSite ? "true" : "false");
1057     LOCAL_LOG(R"(  "cookies": [])");
1058     LOCAL_LOG(R"(  })");
1059 #undef LOCAL_LOG
1060 #undef LOCAL_LOG_IF_ALLOWED
1061 }
1062
1063 static void logCookieInformationInternal(NetworkConnectionToWebProcess& connection, const String& label, const void* loggedObject, const WebCore::NetworkStorageSession& networkStorageSession, const URL& firstParty, const WebCore::SameSiteInfo& sameSiteInfo, const URL& url, const String& referrer, Optional<FrameIdentifier> frameID, Optional<PageIdentifier> pageID, Optional<uint64_t> identifier)
1064 {
1065     ASSERT(NetworkResourceLoader::shouldLogCookieInformation(connection, networkStorageSession.sessionID()));
1066
1067     Vector<WebCore::Cookie> cookies;
1068     if (!networkStorageSession.getRawCookies(firstParty, sameSiteInfo, url, frameID, pageID, cookies))
1069         return;
1070
1071     auto escapedURL = escapeForJSON(url.string());
1072     auto escapedPartition = escapeForJSON(emptyString());
1073     auto escapedReferrer = escapeForJSON(referrer);
1074     auto escapedFrameID = escapeIDForJSON(frameID);
1075     auto escapedPageID = escapeIDForJSON(pageID);
1076     auto escapedIdentifier = escapeIDForJSON(identifier);
1077     bool hasStorageAccess = (frameID && pageID) ? networkStorageSession.hasStorageAccess(WebCore::RegistrableDomain { url }, WebCore::RegistrableDomain { firstParty }, frameID.value(), pageID.value()) : false;
1078
1079 #define LOCAL_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(networkStorageSession.sessionID().isAlwaysOnLoggingAllowed(), Network, "%p - %s::" fmt, loggedObject, label.utf8().data(), ##__VA_ARGS__)
1080 #define LOCAL_LOG(str, ...) \
1081     LOCAL_LOG_IF_ALLOWED("logCookieInformation: pageID = %s, frameID = %s, resourceID = %s: " str, escapedPageID.utf8().data(), escapedFrameID.utf8().data(), escapedIdentifier.utf8().data(), ##__VA_ARGS__)
1082
1083     LOCAL_LOG(R"({ "url": "%{public}s",)", escapedURL.utf8().data());
1084     LOCAL_LOG(R"(  "partition": "%{public}s",)", escapedPartition.utf8().data());
1085     LOCAL_LOG(R"(  "hasStorageAccess": %{public}s,)", hasStorageAccess ? "true" : "false");
1086     LOCAL_LOG(R"(  "referer": "%{public}s",)", escapedReferrer.utf8().data());
1087     LOCAL_LOG(R"(  "isSameSite": "%{public}s",)", sameSiteInfo.isSameSite ? "true" : "false");
1088     LOCAL_LOG(R"(  "isTopSite": "%{public}s",)", sameSiteInfo.isTopSite ? "true" : "false");
1089     LOCAL_LOG(R"(  "cookies": [)");
1090
1091     auto size = cookies.size();
1092     decltype(size) count = 0;
1093     for (const auto& cookie : cookies) {
1094         const char* trailingComma = ",";
1095         if (++count == size)
1096             trailingComma = "";
1097
1098         auto escapedName = escapeForJSON(cookie.name);
1099         auto escapedValue = escapeForJSON(cookie.value);
1100         auto escapedDomain = escapeForJSON(cookie.domain);
1101         auto escapedPath = escapeForJSON(cookie.path);
1102         auto escapedComment = escapeForJSON(cookie.comment);
1103         auto escapedCommentURL = escapeForJSON(cookie.commentURL.string());
1104         // FIXME: Log Same-Site policy for each cookie. See <https://bugs.webkit.org/show_bug.cgi?id=184894>.
1105
1106         LOCAL_LOG(R"(  { "name": "%{public}s",)", escapedName.utf8().data());
1107         LOCAL_LOG(R"(    "value": "%{public}s",)", escapedValue.utf8().data());
1108         LOCAL_LOG(R"(    "domain": "%{public}s",)", escapedDomain.utf8().data());
1109         LOCAL_LOG(R"(    "path": "%{public}s",)", escapedPath.utf8().data());
1110         LOCAL_LOG(R"(    "created": %f,)", cookie.created);
1111         LOCAL_LOG(R"(    "expires": %f,)", cookie.expires);
1112         LOCAL_LOG(R"(    "httpOnly": %{public}s,)", cookie.httpOnly ? "true" : "false");
1113         LOCAL_LOG(R"(    "secure": %{public}s,)", cookie.secure ? "true" : "false");
1114         LOCAL_LOG(R"(    "session": %{public}s,)", cookie.session ? "true" : "false");
1115         LOCAL_LOG(R"(    "comment": "%{public}s",)", escapedComment.utf8().data());
1116         LOCAL_LOG(R"(    "commentURL": "%{public}s")", escapedCommentURL.utf8().data());
1117         LOCAL_LOG(R"(  }%{public}s)", trailingComma);
1118     }
1119     LOCAL_LOG(R"(]})");
1120 #undef LOCAL_LOG
1121 #undef LOCAL_LOG_IF_ALLOWED
1122 }
1123
1124 void NetworkResourceLoader::logCookieInformation(NetworkConnectionToWebProcess& connection, const String& label, const void* loggedObject, const NetworkStorageSession& networkStorageSession, const URL& firstParty, const SameSiteInfo& sameSiteInfo, const URL& url, const String& referrer, Optional<FrameIdentifier> frameID, Optional<PageIdentifier> pageID, Optional<uint64_t> identifier)
1125 {
1126     ASSERT(shouldLogCookieInformation(connection, networkStorageSession.sessionID()));
1127
1128     if (networkStorageSession.shouldBlockCookies(firstParty, url, frameID, pageID))
1129         logBlockedCookieInformation(connection, label, loggedObject, networkStorageSession, firstParty, sameSiteInfo, url, referrer, frameID, pageID, identifier);
1130     else
1131         logCookieInformationInternal(connection, label, loggedObject, networkStorageSession, firstParty, sameSiteInfo, url, referrer, frameID, pageID, identifier);
1132 }
1133 #endif
1134
1135 void NetworkResourceLoader::addConsoleMessage(MessageSource messageSource, MessageLevel messageLevel, const String& message, unsigned long)
1136 {
1137     send(Messages::WebPage::AddConsoleMessage { m_parameters.webFrameID,  messageSource, messageLevel, message, identifier() }, m_parameters.webPageID);
1138 }
1139
1140 void NetworkResourceLoader::sendCSPViolationReport(URL&& reportURL, Ref<FormData>&& report)
1141 {
1142     send(Messages::WebPage::SendCSPViolationReport { m_parameters.webFrameID, WTFMove(reportURL), IPC::FormDataReference { WTFMove(report) } }, m_parameters.webPageID);
1143 }
1144
1145 void NetworkResourceLoader::enqueueSecurityPolicyViolationEvent(WebCore::SecurityPolicyViolationEvent::Init&& eventInit)
1146 {
1147     send(Messages::WebPage::EnqueueSecurityPolicyViolationEvent { m_parameters.webFrameID, WTFMove(eventInit) }, m_parameters.webPageID);
1148 }
1149
1150 void NetworkResourceLoader::logSlowCacheRetrieveIfNeeded(const NetworkCache::Cache::RetrieveInfo& info)
1151 {
1152 #if RELEASE_LOG_DISABLED
1153     UNUSED_PARAM(info);
1154 #else
1155     if (!isAlwaysOnLoggingAllowed())
1156         return;
1157     auto duration = info.completionTime - info.startTime;
1158     if (duration < 1_s)
1159         return;
1160     RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Took %.0fms, priority %d", duration.milliseconds(), info.priority);
1161     if (info.wasSpeculativeLoad)
1162         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Was speculative load");
1163     if (!info.storageTimings.startTime)
1164         return;
1165     RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Storage retrieve time %.0fms", (info.storageTimings.completionTime - info.storageTimings.startTime).milliseconds());
1166     if (info.storageTimings.dispatchTime) {
1167         auto time = (info.storageTimings.dispatchTime - info.storageTimings.startTime).milliseconds();
1168         auto count = info.storageTimings.dispatchCountAtDispatch - info.storageTimings.dispatchCountAtStart;
1169         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Dispatch delay %.0fms, dispatched %lu resources first", time, count);
1170     }
1171     if (info.storageTimings.recordIOStartTime)
1172         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Record I/O time %.0fms", (info.storageTimings.recordIOEndTime - info.storageTimings.recordIOStartTime).milliseconds());
1173     if (info.storageTimings.blobIOStartTime)
1174         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Blob I/O time %.0fms", (info.storageTimings.blobIOEndTime - info.storageTimings.blobIOStartTime).milliseconds());
1175     if (info.storageTimings.synchronizationInProgressAtDispatch)
1176         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Synchronization was in progress");
1177     if (info.storageTimings.shrinkInProgressAtDispatch)
1178         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Shrink was in progress");
1179     if (info.storageTimings.wasCanceled)
1180         RELEASE_LOG_IF_ALLOWED("logSlowCacheRetrieveIfNeeded: Retrieve was canceled");
1181 #endif
1182 }
1183
1184 bool NetworkResourceLoader::isCrossOriginPrefetch() const
1185 {
1186     auto& request = originalRequest();
1187     return request.httpHeaderField(HTTPHeaderName::Purpose) == "prefetch" && !m_parameters.sourceOrigin->canRequest(request.url());
1188 }
1189
1190 } // namespace WebKit
1191
1192 #undef RELEASE_LOG_IF_ALLOWED
1193 #undef RELEASE_LOG_ERROR_IF_ALLOWED