FrameLoader::FrameProgressTracker::progressCompleted() does not need a pageID
[WebKit-https.git] / Source / WebKit / WebProcess / Network / WebLoaderStrategy.cpp
1 /*
2  * Copyright (C) 2012, 2015, 2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebLoaderStrategy.h"
28
29 #include "DataReference.h"
30 #include "HangDetectionDisabler.h"
31 #include "Logging.h"
32 #include "NetworkConnectionToWebProcessMessages.h"
33 #include "NetworkProcessConnection.h"
34 #include "NetworkResourceLoadParameters.h"
35 #include "SharedBufferDataReference.h"
36 #include "WebCompiledContentRuleList.h"
37 #include "WebCoreArgumentCoders.h"
38 #include "WebErrors.h"
39 #include "WebFrame.h"
40 #include "WebFrameLoaderClient.h"
41 #include "WebPage.h"
42 #include "WebPageProxyMessages.h"
43 #include "WebProcess.h"
44 #include "WebProcessPoolMessages.h"
45 #include "WebResourceLoader.h"
46 #include "WebServiceWorkerProvider.h"
47 #include "WebURLSchemeHandlerProxy.h"
48 #include "WebURLSchemeTaskProxy.h"
49 #include <WebCore/ApplicationCacheHost.h>
50 #include <WebCore/CachedResource.h>
51 #include <WebCore/ContentSecurityPolicy.h>
52 #include <WebCore/DiagnosticLoggingClient.h>
53 #include <WebCore/DiagnosticLoggingKeys.h>
54 #include <WebCore/Document.h>
55 #include <WebCore/DocumentLoader.h>
56 #include <WebCore/FetchOptions.h>
57 #include <WebCore/Frame.h>
58 #include <WebCore/FrameLoader.h>
59 #include <WebCore/NetscapePlugInStreamLoader.h>
60 #include <WebCore/NetworkLoadInformation.h>
61 #include <WebCore/PlatformStrategies.h>
62 #include <WebCore/ReferrerPolicy.h>
63 #include <WebCore/ResourceLoader.h>
64 #include <WebCore/RuntimeApplicationChecks.h>
65 #include <WebCore/RuntimeEnabledFeatures.h>
66 #include <WebCore/SecurityOrigin.h>
67 #include <WebCore/Settings.h>
68 #include <WebCore/SubresourceLoader.h>
69 #include <WebCore/UserContentProvider.h>
70 #include <pal/SessionID.h>
71 #include <wtf/CompletionHandler.h>
72 #include <wtf/text/CString.h>
73
74 #if USE(QUICK_LOOK)
75 #include <WebCore/QuickLook.h>
76 #endif
77
78 #define RELEASE_LOG_IF_ALLOWED(permissionChecker, fmt, ...) RELEASE_LOG_IF(permissionChecker.isAlwaysOnLoggingAllowed(), Network, "%p - WebLoaderStrategy::" fmt, this, ##__VA_ARGS__)
79 #define RELEASE_LOG_ERROR_IF_ALLOWED(permissionChecker, fmt, ...) RELEASE_LOG_ERROR_IF(permissionChecker.isAlwaysOnLoggingAllowed(), Network, "%p - WebLoaderStrategy::" fmt, this, ##__VA_ARGS__)
80
81 namespace WebKit {
82 using namespace WebCore;
83
84 WebLoaderStrategy::WebLoaderStrategy()
85     : m_internallyFailedLoadTimer(RunLoop::main(), this, &WebLoaderStrategy::internallyFailedLoadTimerFired)
86 {
87 }
88
89 WebLoaderStrategy::~WebLoaderStrategy()
90 {
91 }
92
93 void WebLoaderStrategy::loadResource(Frame& frame, CachedResource& resource, ResourceRequest&& request, const ResourceLoaderOptions& options, CompletionHandler<void(RefPtr<SubresourceLoader>&&)>&& completionHandler)
94 {
95     SubresourceLoader::create(frame, resource, WTFMove(request), options, [this, referrerPolicy = options.referrerPolicy, completionHandler = WTFMove(completionHandler), resource = CachedResourceHandle<CachedResource>(&resource), frame = makeRef(frame)] (RefPtr<SubresourceLoader>&& loader) mutable {
96         if (loader)
97             scheduleLoad(*loader, resource.get(), referrerPolicy == ReferrerPolicy::NoReferrerWhenDowngrade);
98         else
99             RELEASE_LOG_IF_ALLOWED(frame.get(), "loadResource: Unable to create SubresourceLoader (frame = %p", &frame);
100         completionHandler(WTFMove(loader));
101     });
102 }
103
104 void WebLoaderStrategy::schedulePluginStreamLoad(Frame& frame, NetscapePlugInStreamLoaderClient& client, ResourceRequest&& request, CompletionHandler<void(RefPtr<NetscapePlugInStreamLoader>&&)>&& completionHandler)
105 {
106     NetscapePlugInStreamLoader::create(frame, client, WTFMove(request), [this, completionHandler = WTFMove(completionHandler), frame = makeRef(frame)] (RefPtr<NetscapePlugInStreamLoader>&& loader) mutable {
107         if (loader)
108             scheduleLoad(*loader, 0, frame->document()->referrerPolicy() == ReferrerPolicy::NoReferrerWhenDowngrade);
109         completionHandler(WTFMove(loader));
110     });
111 }
112
113 static Seconds maximumBufferingTime(CachedResource* resource)
114 {
115     if (!resource)
116         return 0_s;
117
118     switch (resource->type()) {
119     case CachedResource::Type::Beacon:
120     case CachedResource::Type::Ping:
121     case CachedResource::Type::CSSStyleSheet:
122     case CachedResource::Type::Script:
123 #if ENABLE(SVG_FONTS)
124     case CachedResource::Type::SVGFontResource:
125 #endif
126     case CachedResource::Type::FontResource:
127 #if ENABLE(APPLICATION_MANIFEST)
128     case CachedResource::Type::ApplicationManifest:
129 #endif
130         return Seconds::infinity();
131     case CachedResource::Type::ImageResource:
132         return 500_ms;
133     case CachedResource::Type::MediaResource:
134         return 50_ms;
135     case CachedResource::Type::MainResource:
136     case CachedResource::Type::Icon:
137     case CachedResource::Type::RawResource:
138     case CachedResource::Type::SVGDocumentResource:
139     case CachedResource::Type::LinkPrefetch:
140 #if ENABLE(VIDEO_TRACK)
141     case CachedResource::Type::TextTrackResource:
142 #endif
143 #if ENABLE(XSLT)
144     case CachedResource::Type::XSLStyleSheet:
145 #endif
146         return 0_s;
147     }
148
149     ASSERT_NOT_REACHED();
150     return 0_s;
151 }
152
153 void WebLoaderStrategy::scheduleLoad(ResourceLoader& resourceLoader, CachedResource* resource, bool shouldClearReferrerOnHTTPSToHTTPRedirect)
154 {
155     ResourceLoadIdentifier identifier = resourceLoader.identifier();
156     ASSERT(identifier);
157
158     auto& frameLoaderClient = resourceLoader.frameLoader()->client();
159
160     WebResourceLoader::TrackingParameters trackingParameters;
161     trackingParameters.pageID = frameLoaderClient.pageID().valueOr(PageIdentifier { });
162     trackingParameters.frameID = frameLoaderClient.frameID().valueOr(FrameIdentifier { });
163     trackingParameters.resourceID = identifier;
164     auto sessionID = frameLoaderClient.sessionID();
165
166 #if ENABLE(WEB_ARCHIVE) || ENABLE(MHTML)
167     // If the DocumentLoader schedules this as an archive resource load,
168     // then we should remember the ResourceLoader in our records but not schedule it in the NetworkProcess.
169     if (resourceLoader.documentLoader()->scheduleArchiveLoad(resourceLoader, resourceLoader.request())) {
170         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be handled as an archive resource.", resourceLoader.url().string().utf8().data());
171         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled as an archive resource (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID.toUInt64(), trackingParameters.frameID.toUInt64(), identifier);
172         m_webResourceLoaders.set(identifier, WebResourceLoader::create(resourceLoader, trackingParameters));
173         return;
174     }
175 #endif
176
177     if (resourceLoader.documentLoader()->applicationCacheHost().maybeLoadResource(resourceLoader, resourceLoader.request(), resourceLoader.request().url())) {
178         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be loaded from application cache.", resourceLoader.url().string().utf8().data());
179         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be loaded from application cache (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID.toUInt64(), trackingParameters.frameID.toUInt64(), identifier);
180         m_webResourceLoaders.set(identifier, WebResourceLoader::create(resourceLoader, trackingParameters));
181         return;
182     }
183
184     if (resourceLoader.request().url().protocolIsData()) {
185         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be loaded as data.", resourceLoader.url().string().utf8().data());
186         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be loaded as data (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID.toUInt64(), trackingParameters.frameID.toUInt64(), identifier);
187         startLocalLoad(resourceLoader);
188         return;
189     }
190
191 #if USE(QUICK_LOOK)
192     if (isQuickLookPreviewURL(resourceLoader.request().url())) {
193         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be handled as a QuickLook resource.", resourceLoader.url().string().utf8().data());
194         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled as a QuickLook resource (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID.toUInt64(), trackingParameters.frameID.toUInt64(), identifier);
195         startLocalLoad(resourceLoader);
196         return;
197     }
198 #endif
199
200 #if USE(SOUP)
201     // For apps that call g_resource_load in a web extension.
202     // https://blogs.gnome.org/alexl/2012/01/26/resources-in-glib/
203     if (resourceLoader.request().url().protocolIs("resource")) {
204         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be handled as a GResource.", resourceLoader.url().string().utf8().data());
205         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled as a GResource (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID.toUInt64(), trackingParameters.frameID.toUInt64(), identifier);
206         startLocalLoad(resourceLoader);
207         return;
208     }
209 #endif
210
211 #if ENABLE(SERVICE_WORKER)
212     WebServiceWorkerProvider::singleton().handleFetch(resourceLoader, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, [this, trackingParameters, identifier, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, maximumBufferingTime = maximumBufferingTime(resource), resourceLoader = makeRef(resourceLoader)] (ServiceWorkerClientFetch::Result result) mutable {
213         if (result != ServiceWorkerClientFetch::Result::Unhandled) {
214             LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be scheduled through ServiceWorker handle fetch algorithm", resourceLoader->url().string().latin1().data());
215             RELEASE_LOG_IF_ALLOWED(resourceLoader.get(), "scheduleLoad: URL will be scheduled through ServiceWorker handle fetch algorithm (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader->frame(), trackingParameters.pageID.toUInt64(), trackingParameters.frameID.toUInt64(), identifier);
216             return;
217         }
218         if (resourceLoader->options().serviceWorkersMode == ServiceWorkersMode::Only) {
219             RELEASE_LOG_ERROR_IF_ALLOWED(resourceLoader.get(), "scheduleLoad: unable to schedule URL through ServiceWorker handle fetch algorithm (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader->frame(), trackingParameters.pageID.toUInt64(), trackingParameters.frameID.toUInt64(), identifier);
220             callOnMainThread([resourceLoader = WTFMove(resourceLoader)] {
221                 auto error = internalError(resourceLoader->request().url());
222                 error.setType(ResourceError::Type::Cancellation);
223                 resourceLoader->didFail(error);
224             });
225             return;
226         }
227
228         if (!WebProcess::singleton().webLoaderStrategy().tryLoadingUsingURLSchemeHandler(resourceLoader))
229             WebProcess::singleton().webLoaderStrategy().scheduleLoadFromNetworkProcess(resourceLoader.get(), resourceLoader->request(), trackingParameters, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, maximumBufferingTime);
230         else
231             RELEASE_LOG_IF_ALLOWED(resourceLoader.get(), "scheduleLoad: URL not handled by any handlers (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader->frame(), trackingParameters.pageID.toUInt64(), trackingParameters.frameID.toUInt64(), identifier);
232     });
233 #else
234     if (!tryLoadingUsingURLSchemeHandler(resourceLoader))
235         scheduleLoadFromNetworkProcess(resourceLoader, resourceLoader.request(), trackingParameters, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, maximumBufferingTime(resource));
236     else
237         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL not handled by any handlers (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID.toUInt64(), trackingParameters.frameID.toUInt64(), identifier);
238 #endif
239 }
240
241 bool WebLoaderStrategy::tryLoadingUsingURLSchemeHandler(ResourceLoader& resourceLoader)
242 {
243     auto* webFrameLoaderClient = toWebFrameLoaderClient(resourceLoader.frameLoader()->client());
244     auto* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
245     auto* webPage = webFrame ? webFrame->page() : nullptr;
246     if (webPage) {
247         if (auto* handler = webPage->urlSchemeHandlerForScheme(resourceLoader.request().url().protocol().toStringWithoutCopying())) {
248             LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, URL '%s' will be handled by a UIProcess URL scheme handler.", resourceLoader.url().string().utf8().data());
249             RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled by a UIProcess URL scheme handler (frame = %p, resourceID = %lu)", resourceLoader.frame(), resourceLoader.identifier());
250
251             handler->startNewTask(resourceLoader);
252             return true;
253         }
254     }
255     return false;
256 }
257
258 void WebLoaderStrategy::scheduleLoadFromNetworkProcess(ResourceLoader& resourceLoader, const ResourceRequest& request, const WebResourceLoader::TrackingParameters& trackingParameters, PAL::SessionID sessionID, bool shouldClearReferrerOnHTTPSToHTTPRedirect, Seconds maximumBufferingTime)
259 {
260     ResourceLoadIdentifier identifier = resourceLoader.identifier();
261     ASSERT(identifier);
262
263     LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be scheduled with the NetworkProcess with priority %d", resourceLoader.url().string().latin1().data(), static_cast<int>(resourceLoader.request().priority()));
264
265     ContentSniffingPolicy contentSniffingPolicy = resourceLoader.shouldSniffContent() ? ContentSniffingPolicy::SniffContent : ContentSniffingPolicy::DoNotSniffContent;
266     ContentEncodingSniffingPolicy contentEncodingSniffingPolicy = resourceLoader.shouldSniffContentEncoding() ? ContentEncodingSniffingPolicy::Sniff : ContentEncodingSniffingPolicy::DoNotSniff;
267     StoredCredentialsPolicy storedCredentialsPolicy = resourceLoader.shouldUseCredentialStorage() ? StoredCredentialsPolicy::Use : StoredCredentialsPolicy::DoNotUse;
268
269     NetworkResourceLoadParameters loadParameters { sessionID };
270     loadParameters.identifier = identifier;
271     loadParameters.webPageID = trackingParameters.pageID;
272     loadParameters.webFrameID = trackingParameters.frameID;
273     loadParameters.parentPID = presentingApplicationPID();
274     loadParameters.request = request;
275     loadParameters.contentSniffingPolicy = contentSniffingPolicy;
276     loadParameters.contentEncodingSniffingPolicy = contentEncodingSniffingPolicy;
277     loadParameters.storedCredentialsPolicy = storedCredentialsPolicy;
278     // If there is no WebFrame then this resource cannot be authenticated with the client.
279     loadParameters.clientCredentialPolicy = (loadParameters.webFrameID && loadParameters.webPageID && resourceLoader.isAllowedToAskUserForCredentials()) ? ClientCredentialPolicy::MayAskClientForCredentials : ClientCredentialPolicy::CannotAskClientForCredentials;
280     loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect = shouldClearReferrerOnHTTPSToHTTPRedirect;
281     loadParameters.needsCertificateInfo = resourceLoader.shouldIncludeCertificateInfo();
282     loadParameters.maximumBufferingTime = maximumBufferingTime;
283     loadParameters.options = resourceLoader.options();
284     loadParameters.preflightPolicy = resourceLoader.options().preflightPolicy;
285     loadParameters.isHTTPSUpgradeEnabled = resourceLoader.frame() ? resourceLoader.frame()->settings().HTTPSUpgradeEnabled() : false;
286
287     auto* document = resourceLoader.frame() ? resourceLoader.frame()->document() : nullptr;
288     if (resourceLoader.options().cspResponseHeaders)
289         loadParameters.cspResponseHeaders = resourceLoader.options().cspResponseHeaders;
290     else if (document && !document->shouldBypassMainWorldContentSecurityPolicy() && resourceLoader.options().contentSecurityPolicyImposition == ContentSecurityPolicyImposition::DoPolicyCheck) {
291         if (auto* contentSecurityPolicy = document->contentSecurityPolicy())
292             loadParameters.cspResponseHeaders = contentSecurityPolicy->responseHeaders();
293     }
294
295 #if ENABLE(CONTENT_EXTENSIONS)
296     if (document) {
297         loadParameters.mainDocumentURL = document->topDocument().url();
298         // FIXME: Instead of passing userContentControllerIdentifier, the NetworkProcess should be able to get it using webPageId.
299         auto* webFrameLoaderClient = toWebFrameLoaderClient(resourceLoader.frame()->loader().client());
300         auto* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
301         auto* webPage = webFrame ? webFrame->page() : nullptr;
302         if (webPage)
303             loadParameters.userContentControllerIdentifier = webPage->userContentControllerIdentifier();
304     }
305 #endif
306
307     // FIXME: All loaders should provide their origin if navigation mode is cors/no-cors/same-origin.
308     // As a temporary approach, we use the document origin if available or the HTTP Origin header otherwise.
309     if (is<SubresourceLoader>(resourceLoader)) {
310         auto& loader = downcast<SubresourceLoader>(resourceLoader);
311         loadParameters.sourceOrigin = loader.origin();
312
313         if (auto* headers = loader.originalHeaders())
314             loadParameters.originalRequestHeaders = *headers;
315     }
316
317     if (!loadParameters.sourceOrigin && document)
318         loadParameters.sourceOrigin = &document->securityOrigin();
319     if (!loadParameters.sourceOrigin) {
320         auto origin = request.httpOrigin();
321         if (!origin.isNull())
322             loadParameters.sourceOrigin = SecurityOrigin::createFromString(origin);
323     }
324     if (document)
325         loadParameters.topOrigin = &document->topOrigin();
326
327     if (loadParameters.options.mode != FetchOptions::Mode::Navigate) {
328         ASSERT(loadParameters.sourceOrigin);
329         if (!loadParameters.sourceOrigin) {
330             RELEASE_LOG_ERROR_IF_ALLOWED(resourceLoader, "scheduleLoad: no sourceOrigin (frame = %p, priority = %d, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), static_cast<int>(resourceLoader.request().priority()), loadParameters.webPageID.toUInt64(), loadParameters.webFrameID.toUInt64(), loadParameters.identifier);
331             scheduleInternallyFailedLoad(resourceLoader);
332             return;
333         }
334     }
335
336     loadParameters.shouldRestrictHTTPResponseAccess = shouldPerformSecurityChecks();
337
338     loadParameters.isMainFrameNavigation = resourceLoader.frame() && resourceLoader.frame()->isMainFrame() && resourceLoader.options().mode == FetchOptions::Mode::Navigate;
339
340     loadParameters.isMainResourceNavigationForAnyFrame = resourceLoader.frame() && resourceLoader.options().mode == FetchOptions::Mode::Navigate;
341
342     loadParameters.shouldEnableCrossOriginResourcePolicy = RuntimeEnabledFeatures::sharedFeatures().crossOriginResourcePolicyEnabled() && !loadParameters.isMainFrameNavigation;
343
344     if (resourceLoader.options().mode == FetchOptions::Mode::Navigate) {
345         Vector<RefPtr<SecurityOrigin>> frameAncestorOrigins;
346         for (auto* frame = resourceLoader.frame()->tree().parent(); frame; frame = frame->tree().parent())
347             frameAncestorOrigins.append(makeRefPtr(frame->document()->securityOrigin()));
348         loadParameters.frameAncestorOrigins = WTFMove(frameAncestorOrigins);
349     }
350
351     ASSERT((loadParameters.webPageID && loadParameters.webFrameID) || loadParameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials);
352
353     RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: Resource is being scheduled with the NetworkProcess (frame = %p, priority = %d, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), static_cast<int>(resourceLoader.request().priority()), loadParameters.webPageID.toUInt64(), loadParameters.webFrameID.toUInt64(), loadParameters.identifier);
354     if (!WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::ScheduleResourceLoad(loadParameters), 0)) {
355         RELEASE_LOG_ERROR_IF_ALLOWED(resourceLoader, "scheduleLoad: Unable to schedule resource with the NetworkProcess (frame = %p, priority = %d, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), static_cast<int>(resourceLoader.request().priority()), loadParameters.webPageID.toUInt64(), loadParameters.webFrameID.toUInt64(), loadParameters.identifier);
356         // We probably failed to schedule this load with the NetworkProcess because it had crashed.
357         // This load will never succeed so we will schedule it to fail asynchronously.
358         scheduleInternallyFailedLoad(resourceLoader);
359         return;
360     }
361
362     auto loader = WebResourceLoader::create(resourceLoader, trackingParameters);
363     if (resourceLoader.originalRequest().hasUpload()) {
364         if (m_loadersWithUploads.isEmpty())
365             WebProcess::singleton().parentProcessConnection()->send(Messages::WebProcessPool::SetWebProcessHasUploads(Process::identifier()), 0);
366         m_loadersWithUploads.add(loader.ptr());
367     }
368
369     m_webResourceLoaders.set(identifier, WTFMove(loader));
370 }
371
372 void WebLoaderStrategy::scheduleInternallyFailedLoad(WebCore::ResourceLoader& resourceLoader)
373 {
374     m_internallyFailedResourceLoaders.add(&resourceLoader);
375     m_internallyFailedLoadTimer.startOneShot(0_s);
376 }
377
378 void WebLoaderStrategy::internallyFailedLoadTimerFired()
379 {
380     for (auto& resourceLoader : copyToVector(m_internallyFailedResourceLoaders))
381         resourceLoader->didFail(internalError(resourceLoader->url()));
382 }
383
384 void WebLoaderStrategy::startLocalLoad(WebCore::ResourceLoader& resourceLoader)
385 {
386     resourceLoader.start();
387     m_webResourceLoaders.set(resourceLoader.identifier(), WebResourceLoader::create(resourceLoader, { }));
388 }
389
390 void WebLoaderStrategy::addURLSchemeTaskProxy(WebURLSchemeTaskProxy& task)
391 {
392     auto result = m_urlSchemeTasks.add(task.identifier(), &task);
393     ASSERT_UNUSED(result, result.isNewEntry);
394 }
395
396 void WebLoaderStrategy::removeURLSchemeTaskProxy(WebURLSchemeTaskProxy& task)
397 {
398     m_urlSchemeTasks.remove(task.identifier());
399 }
400
401 void WebLoaderStrategy::remove(ResourceLoader* resourceLoader)
402 {
403     ASSERT(resourceLoader);
404     LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::remove, url '%s'", resourceLoader->url().string().utf8().data());
405
406     if (auto task = m_urlSchemeTasks.take(resourceLoader->identifier())) {
407         ASSERT(!m_internallyFailedResourceLoaders.contains(resourceLoader));
408         task->stopLoading();
409         return;
410     }
411
412     if (m_internallyFailedResourceLoaders.contains(resourceLoader)) {
413         m_internallyFailedResourceLoaders.remove(resourceLoader);
414         return;
415     }
416     
417     ResourceLoadIdentifier identifier = resourceLoader->identifier();
418     if (!identifier) {
419         LOG_ERROR("WebLoaderStrategy removing a ResourceLoader that has no identifier.");
420         return;
421     }
422
423 #if ENABLE(SERVICE_WORKER)
424     if (WebServiceWorkerProvider::singleton().cancelFetch(makeObjectIdentifier<FetchIdentifierType>(identifier)))
425         return;
426 #endif
427
428     RefPtr<WebResourceLoader> loader = m_webResourceLoaders.take(identifier);
429     // Loader may not be registered if we created it, but haven't scheduled yet (a bundle client can decide to cancel such request via willSendRequest).
430     if (!loader)
431         return;
432
433     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::RemoveLoadIdentifier(identifier), 0);
434
435     if (m_loadersWithUploads.remove(loader.get()) && m_loadersWithUploads.isEmpty())
436         WebProcess::singleton().parentProcessConnection()->send(Messages::WebProcessPool::ClearWebProcessHasUploads { Process::identifier() }, 0);
437
438     // It's possible that this WebResourceLoader might be just about to message back to the NetworkProcess (e.g. ContinueWillSendRequest)
439     // but there's no point in doing so anymore.
440     loader->detachFromCoreLoader();
441 }
442
443 void WebLoaderStrategy::setDefersLoading(ResourceLoader&, bool)
444 {
445 }
446
447 void WebLoaderStrategy::crossOriginRedirectReceived(ResourceLoader*, const URL&)
448 {
449     // We handle cross origin redirects entirely within the NetworkProcess.
450     // We override this call in the WebProcess to make it a no-op.
451 }
452
453 void WebLoaderStrategy::servePendingRequests(ResourceLoadPriority)
454 {
455     // This overrides the base class version.
456     // We don't need to do anything as this is handled by the network process.
457 }
458
459 void WebLoaderStrategy::suspendPendingRequests()
460 {
461     // Network process does keep requests in pending state.
462 }
463
464 void WebLoaderStrategy::resumePendingRequests()
465 {
466     // Network process does keep requests in pending state.
467 }
468
469 void WebLoaderStrategy::networkProcessCrashed()
470 {
471     RELEASE_LOG_ERROR(Network, "WebLoaderStrategy::networkProcessCrashed: failing all pending resource loaders");
472
473     for (auto& loader : m_webResourceLoaders.values()) {
474         scheduleInternallyFailedLoad(*loader->resourceLoader());
475         loader->detachFromCoreLoader();
476     }
477
478     m_webResourceLoaders.clear();
479
480     auto pingLoadCompletionHandlers = WTFMove(m_pingLoadCompletionHandlers);
481     for (auto& pingLoadCompletionHandler : pingLoadCompletionHandlers.values())
482         pingLoadCompletionHandler(internalError(URL()), { });
483
484     auto preconnectCompletionHandlers = WTFMove(m_preconnectCompletionHandlers);
485     for (auto& preconnectCompletionHandler : preconnectCompletionHandlers.values())
486         preconnectCompletionHandler(internalError(URL()));
487 }
488
489 static bool shouldClearReferrerOnHTTPSToHTTPRedirect(Frame* frame)
490 {
491     if (frame) {
492         if (auto* document = frame->document())
493             return document->referrerPolicy() == ReferrerPolicy::NoReferrerWhenDowngrade;
494     }
495     return true;
496 }
497
498 Optional<WebLoaderStrategy::SyncLoadResult> WebLoaderStrategy::tryLoadingSynchronouslyUsingURLSchemeHandler(FrameLoader& frameLoader, ResourceLoadIdentifier identifier, const ResourceRequest& request)
499 {
500     auto* webFrameLoaderClient = toWebFrameLoaderClient(frameLoader.client());
501     auto* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
502     auto* webPage = webFrame ? webFrame->page() : nullptr;
503     if (!webPage)
504         return WTF::nullopt;
505
506     auto* handler = webPage->urlSchemeHandlerForScheme(request.url().protocol().toStringWithoutCopying());
507     if (!handler)
508         return WTF::nullopt;
509
510     LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, sync load to URL '%s' will be handled by a UIProcess URL scheme handler.", request.url().string().utf8().data());
511
512     SyncLoadResult result;
513     handler->loadSynchronously(identifier, request, result.response, result.error, result.data);
514
515     return result;
516 }
517
518 void WebLoaderStrategy::loadResourceSynchronously(FrameLoader& frameLoader, unsigned long resourceLoadIdentifier, const ResourceRequest& request, ClientCredentialPolicy clientCredentialPolicy,  const FetchOptions& options, const HTTPHeaderMap& originalRequestHeaders, ResourceError& error, ResourceResponse& response, Vector<char>& data)
519 {
520     WebFrameLoaderClient* webFrameLoaderClient = toWebFrameLoaderClient(frameLoader.client());
521     WebFrame* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
522     WebPage* webPage = webFrame ? webFrame->page() : nullptr;
523
524     auto pageID = webPage ? webPage->identifier() : PageIdentifier { };
525     auto frameID = webFrame ? webFrame->frameID() : FrameIdentifier { };
526     auto sessionID = webPage ? webPage->sessionID() : PAL::SessionID::defaultSessionID();
527
528     auto* document = frameLoader.frame().document();
529     if (!document) {
530         RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, "loadResourceSynchronously: no document (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %lu)", pageID.toUInt64(), frameID.toUInt64(), resourceLoadIdentifier);
531         error = internalError(request.url());
532         return;
533     }
534
535     if (auto syncLoadResult = tryLoadingSynchronouslyUsingURLSchemeHandler(frameLoader, resourceLoadIdentifier, request)) {
536         RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, "loadResourceSynchronously: failed calling tryLoadingSynchronouslyUsingURLSchemeHandler (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %lu, error = %d)", pageID.toUInt64(), frameID.toUInt64(), resourceLoadIdentifier, syncLoadResult->error.errorCode());
537         error = WTFMove(syncLoadResult->error);
538         response = WTFMove(syncLoadResult->response);
539         data = WTFMove(syncLoadResult->data);
540         return;
541     }
542
543     NetworkResourceLoadParameters loadParameters { sessionID };
544     loadParameters.identifier = resourceLoadIdentifier;
545     loadParameters.webPageID = pageID;
546     loadParameters.webFrameID = frameID;
547     loadParameters.parentPID = presentingApplicationPID();
548     loadParameters.request = request;
549     loadParameters.contentSniffingPolicy = ContentSniffingPolicy::SniffContent;
550     loadParameters.contentEncodingSniffingPolicy = ContentEncodingSniffingPolicy::Sniff;
551     loadParameters.storedCredentialsPolicy = options.credentials == FetchOptions::Credentials::Omit ? StoredCredentialsPolicy::DoNotUse : StoredCredentialsPolicy::Use;
552     loadParameters.clientCredentialPolicy = clientCredentialPolicy;
553     loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect = shouldClearReferrerOnHTTPSToHTTPRedirect(webFrame ? webFrame->coreFrame() : nullptr);
554     loadParameters.shouldRestrictHTTPResponseAccess = shouldPerformSecurityChecks();
555
556     loadParameters.options = options;
557     loadParameters.sourceOrigin = &document->securityOrigin();
558     loadParameters.topOrigin = &document->topOrigin();
559     if (!document->shouldBypassMainWorldContentSecurityPolicy()) {
560         if (auto* contentSecurityPolicy = document->contentSecurityPolicy())
561             loadParameters.cspResponseHeaders = contentSecurityPolicy->responseHeaders();
562     }
563     loadParameters.originalRequestHeaders = originalRequestHeaders;
564
565     data.shrink(0);
566
567     HangDetectionDisabler hangDetectionDisabler;
568     IPC::UnboundedSynchronousIPCScope unboundedSynchronousIPCScope;
569
570     bool shouldNotifyOfUpload = request.hasUpload() && m_loadersWithUploads.isEmpty();
571     if (shouldNotifyOfUpload)
572         WebProcess::singleton().parentProcessConnection()->send(Messages::WebProcessPool::SetWebProcessHasUploads { Process::identifier() }, 0);
573
574     if (!WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad(loadParameters), Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::Reply(error, response, data), 0)) {
575         RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, "loadResourceSynchronously: failed sending synchronous network process message (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %lu)", pageID.toUInt64(), frameID.toUInt64(), resourceLoadIdentifier);
576         if (auto* page = webPage ? webPage->corePage() : nullptr)
577             page->diagnosticLoggingClient().logDiagnosticMessage(WebCore::DiagnosticLoggingKeys::internalErrorKey(), WebCore::DiagnosticLoggingKeys::synchronousMessageFailedKey(), WebCore::ShouldSample::No);
578         response = ResourceResponse();
579         error = internalError(request.url());
580     }
581
582     if (shouldNotifyOfUpload)
583         WebProcess::singleton().parentProcessConnection()->send(Messages::WebProcessPool::ClearWebProcessHasUploads { Process::identifier() }, 0);
584 }
585
586 void WebLoaderStrategy::pageLoadCompleted(Page& page)
587 {
588     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::PageLoadCompleted(WebPage::fromCorePage(page).identifier()), 0);
589 }
590
591 static uint64_t generateLoadIdentifier()
592 {
593     static uint64_t identifier = 0;
594     return ++identifier;
595 }
596
597 bool WebLoaderStrategy::usePingLoad() const
598 {
599     return !RuntimeEnabledFeatures::sharedFeatures().fetchAPIKeepAliveEnabled();
600 }
601
602 void WebLoaderStrategy::startPingLoad(Frame& frame, ResourceRequest& request, const HTTPHeaderMap& originalRequestHeaders, const FetchOptions& options, ContentSecurityPolicyImposition policyCheck, PingLoadCompletionHandler&& completionHandler)
603 {
604     auto* document = frame.document();
605     if (!document) {
606         if (completionHandler)
607             completionHandler(internalError(request.url()), { });
608         return;
609     }
610
611     NetworkResourceLoadParameters loadParameters { frame.page() ? frame.page()->sessionID() : PAL::SessionID::defaultSessionID() };
612     loadParameters.identifier = generateLoadIdentifier();
613     loadParameters.request = request;
614     loadParameters.sourceOrigin = &document->securityOrigin();
615     loadParameters.topOrigin = &document->topOrigin();
616     loadParameters.parentPID = presentingApplicationPID();
617     loadParameters.storedCredentialsPolicy = options.credentials == FetchOptions::Credentials::Omit ? StoredCredentialsPolicy::DoNotUse : StoredCredentialsPolicy::Use;
618     loadParameters.options = options;
619     loadParameters.originalRequestHeaders = originalRequestHeaders;
620     loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect = shouldClearReferrerOnHTTPSToHTTPRedirect(&frame);
621     loadParameters.shouldRestrictHTTPResponseAccess = shouldPerformSecurityChecks();
622     if (policyCheck == ContentSecurityPolicyImposition::DoPolicyCheck && !document->shouldBypassMainWorldContentSecurityPolicy()) {
623         if (auto* contentSecurityPolicy = document->contentSecurityPolicy())
624             loadParameters.cspResponseHeaders = contentSecurityPolicy->responseHeaders();
625     }
626
627 #if ENABLE(CONTENT_EXTENSIONS)
628     loadParameters.mainDocumentURL = document->topDocument().url();
629     // FIXME: Instead of passing userContentControllerIdentifier, we should just pass webPageId to NetworkProcess.
630     WebFrameLoaderClient* webFrameLoaderClient = toWebFrameLoaderClient(frame.loader().client());
631     WebFrame* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
632     WebPage* webPage = webFrame ? webFrame->page() : nullptr;
633     if (webPage)
634         loadParameters.userContentControllerIdentifier = webPage->userContentControllerIdentifier();
635 #endif
636
637     if (completionHandler)
638         m_pingLoadCompletionHandlers.add(loadParameters.identifier, WTFMove(completionHandler));
639
640     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::LoadPing { loadParameters }, 0);
641 }
642
643 void WebLoaderStrategy::didFinishPingLoad(uint64_t pingLoadIdentifier, ResourceError&& error, ResourceResponse&& response)
644 {
645     if (auto completionHandler = m_pingLoadCompletionHandlers.take(pingLoadIdentifier))
646         completionHandler(WTFMove(error), WTFMove(response));
647 }
648
649 void WebLoaderStrategy::preconnectTo(FrameLoader& frameLoader, const URL& url, StoredCredentialsPolicy storedCredentialsPolicy, PreconnectCompletionHandler&& completionHandler)
650 {
651     uint64_t preconnectionIdentifier = generateLoadIdentifier();
652     auto addResult = m_preconnectCompletionHandlers.add(preconnectionIdentifier, WTFMove(completionHandler));
653     ASSERT_UNUSED(addResult, addResult.isNewEntry);
654
655     auto* webFrameLoaderClient = toWebFrameLoaderClient(frameLoader.client());
656     if (!webFrameLoaderClient) {
657         completionHandler(internalError(url));
658         return;
659     }
660     auto* webFrame = webFrameLoaderClient->webFrame();
661     if (!webFrame) {
662         completionHandler(internalError(url));
663         return;
664     }
665     auto* webPage = webFrame->page();
666     if (!webPage) {
667         completionHandler(internalError(url));
668         return;
669     }
670
671     NetworkResourceLoadParameters parameters { webPage->sessionID() };
672     parameters.request = ResourceRequest { url };
673     parameters.webPageID = webPage->identifier();
674     parameters.webFrameID = webFrame->frameID();
675     parameters.parentPID = presentingApplicationPID();
676     parameters.storedCredentialsPolicy = storedCredentialsPolicy;
677     parameters.shouldPreconnectOnly = PreconnectOnly::Yes;
678     parameters.shouldRestrictHTTPResponseAccess = shouldPerformSecurityChecks();
679     // FIXME: Use the proper destination once all fetch options are passed.
680     parameters.options.destination = FetchOptions::Destination::EmptyString;
681
682     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::PreconnectTo(preconnectionIdentifier, WTFMove(parameters)), 0);
683 }
684
685 void WebLoaderStrategy::didFinishPreconnection(uint64_t preconnectionIdentifier, ResourceError&& error)
686 {
687     if (auto completionHandler = m_preconnectCompletionHandlers.take(preconnectionIdentifier))
688         completionHandler(WTFMove(error));
689 }
690
691 bool WebLoaderStrategy::isOnLine() const
692 {
693     return m_isOnLine;
694 }
695
696 void WebLoaderStrategy::addOnlineStateChangeListener(Function<void(bool)>&& listener)
697 {
698     WebProcess::singleton().ensureNetworkProcessConnection();
699     m_onlineStateChangeListeners.append(WTFMove(listener));
700 }
701
702 void WebLoaderStrategy::setOnLineState(bool isOnLine)
703 {
704     if (m_isOnLine == isOnLine)
705         return;
706
707     m_isOnLine = isOnLine;
708     for (auto& listener : m_onlineStateChangeListeners)
709         listener(isOnLine);
710 }
711
712 void WebLoaderStrategy::setCaptureExtraNetworkLoadMetricsEnabled(bool enabled)
713 {
714     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::SetCaptureExtraNetworkLoadMetricsEnabled(enabled), 0);
715 }
716
717 ResourceResponse WebLoaderStrategy::responseFromResourceLoadIdentifier(uint64_t resourceLoadIdentifier)
718 {
719     ResourceResponse response;
720     WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::GetNetworkLoadInformationResponse { resourceLoadIdentifier }, Messages::NetworkConnectionToWebProcess::GetNetworkLoadInformationResponse::Reply { response }, 0);
721     return response;
722 }
723
724 Vector<NetworkTransactionInformation> WebLoaderStrategy::intermediateLoadInformationFromResourceLoadIdentifier(uint64_t resourceLoadIdentifier)
725 {
726     Vector<NetworkTransactionInformation> information;
727     WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::GetNetworkLoadIntermediateInformation { resourceLoadIdentifier }, Messages::NetworkConnectionToWebProcess::GetNetworkLoadIntermediateInformation::Reply { information }, 0);
728     return information;
729 }
730
731 NetworkLoadMetrics WebLoaderStrategy::networkMetricsFromResourceLoadIdentifier(uint64_t resourceLoadIdentifier)
732 {
733     NetworkLoadMetrics networkMetrics;
734     WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::TakeNetworkLoadInformationMetrics { resourceLoadIdentifier }, Messages::NetworkConnectionToWebProcess::TakeNetworkLoadInformationMetrics::Reply { networkMetrics }, 0);
735     return networkMetrics;
736 }
737
738 bool WebLoaderStrategy::shouldPerformSecurityChecks() const
739 {
740     return RuntimeEnabledFeatures::sharedFeatures().restrictedHTTPResponseAccess();
741 }
742
743 bool WebLoaderStrategy::havePerformedSecurityChecks(const ResourceResponse& response) const
744 {
745     if (!shouldPerformSecurityChecks())
746         return false;
747     switch (response.source()) {
748     case ResourceResponse::Source::ApplicationCache:
749     case ResourceResponse::Source::MemoryCache:
750     case ResourceResponse::Source::MemoryCacheAfterValidation:
751     case ResourceResponse::Source::ServiceWorker:
752         return false;
753     case ResourceResponse::Source::DiskCache:
754     case ResourceResponse::Source::DiskCacheAfterValidation:
755     case ResourceResponse::Source::Network:
756     case ResourceResponse::Source::Unknown:
757         return true;
758     }
759     ASSERT_NOT_REACHED();
760     return false;
761 }
762
763 } // namespace WebKit
764
765 #undef RELEASE_LOG_IF_ALLOWED
766 #undef RELEASE_LOG_ERROR_IF_ALLOWED