WKURLSchemeHandler doesn't handle sync XHR.
[WebKit-https.git] / Source / WebKit / WebProcess / Network / WebLoaderStrategy.cpp
1 /*
2  * Copyright (C) 2012, 2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebLoaderStrategy.h"
28
29 #include "DataReference.h"
30 #include "HangDetectionDisabler.h"
31 #include "Logging.h"
32 #include "NetworkConnectionToWebProcessMessages.h"
33 #include "NetworkProcessConnection.h"
34 #include "NetworkResourceLoadParameters.h"
35 #include "SessionTracker.h"
36 #include "WebCompiledContentRuleList.h"
37 #include "WebCoreArgumentCoders.h"
38 #include "WebErrors.h"
39 #include "WebFrame.h"
40 #include "WebFrameLoaderClient.h"
41 #include "WebPage.h"
42 #include "WebPageProxyMessages.h"
43 #include "WebProcess.h"
44 #include "WebResourceLoader.h"
45 #include "WebServiceWorkerProvider.h"
46 #include "WebURLSchemeHandlerProxy.h"
47 #include "WebURLSchemeTaskProxy.h"
48 #include <WebCore/ApplicationCacheHost.h>
49 #include <WebCore/CachedResource.h>
50 #include <WebCore/ContentSecurityPolicy.h>
51 #include <WebCore/DiagnosticLoggingClient.h>
52 #include <WebCore/DiagnosticLoggingKeys.h>
53 #include <WebCore/Document.h>
54 #include <WebCore/DocumentLoader.h>
55 #include <WebCore/FetchOptions.h>
56 #include <WebCore/Frame.h>
57 #include <WebCore/FrameLoader.h>
58 #include <WebCore/NetscapePlugInStreamLoader.h>
59 #include <WebCore/NetworkLoadInformation.h>
60 #include <WebCore/PlatformStrategies.h>
61 #include <WebCore/ReferrerPolicy.h>
62 #include <WebCore/ResourceLoader.h>
63 #include <WebCore/RuntimeEnabledFeatures.h>
64 #include <WebCore/SecurityOrigin.h>
65 #include <WebCore/Settings.h>
66 #include <WebCore/SubresourceLoader.h>
67 #include <WebCore/UserContentProvider.h>
68 #include <pal/SessionID.h>
69 #include <wtf/CompletionHandler.h>
70 #include <wtf/text/CString.h>
71
72 #if USE(QUICK_LOOK)
73 #include <WebCore/QuickLook.h>
74 #endif
75
76 using namespace WebCore;
77
78 #define RELEASE_LOG_IF_ALLOWED(permissionChecker, fmt, ...) RELEASE_LOG_IF(permissionChecker.isAlwaysOnLoggingAllowed(), Network, "%p - WebLoaderStrategy::" fmt, this, ##__VA_ARGS__)
79 #define RELEASE_LOG_ERROR_IF_ALLOWED(permissionChecker, fmt, ...) RELEASE_LOG_ERROR_IF(permissionChecker.isAlwaysOnLoggingAllowed(), Network, "%p - WebLoaderStrategy::" fmt, this, ##__VA_ARGS__)
80
81 namespace WebKit {
82
83 WebLoaderStrategy::WebLoaderStrategy()
84     : m_internallyFailedLoadTimer(RunLoop::main(), this, &WebLoaderStrategy::internallyFailedLoadTimerFired)
85 {
86 }
87
88 WebLoaderStrategy::~WebLoaderStrategy()
89 {
90 }
91
92 void WebLoaderStrategy::loadResource(Frame& frame, CachedResource& resource, ResourceRequest&& request, const ResourceLoaderOptions& options, CompletionHandler<void(RefPtr<SubresourceLoader>&&)>&& completionHandler)
93 {
94     SubresourceLoader::create(frame, resource, WTFMove(request), options, [this, completionHandler = WTFMove(completionHandler), resource = CachedResourceHandle<CachedResource>(&resource), frame = makeRef(frame)] (RefPtr<SubresourceLoader>&& loader) mutable {
95         if (loader)
96             scheduleLoad(*loader, resource.get(), frame->document()->referrerPolicy() == ReferrerPolicy::NoReferrerWhenDowngrade);
97         else
98             RELEASE_LOG_IF_ALLOWED(frame.get(), "loadResource: Unable to create SubresourceLoader (frame = %p", &frame);
99         completionHandler(WTFMove(loader));
100     });
101 }
102
103 void WebLoaderStrategy::schedulePluginStreamLoad(Frame& frame, NetscapePlugInStreamLoaderClient& client, ResourceRequest&& request, CompletionHandler<void(RefPtr<NetscapePlugInStreamLoader>&&)>&& completionHandler)
104 {
105     NetscapePlugInStreamLoader::create(frame, client, WTFMove(request), [this, completionHandler = WTFMove(completionHandler), frame = makeRef(frame)] (RefPtr<NetscapePlugInStreamLoader>&& loader) mutable {
106         if (loader)
107             scheduleLoad(*loader, 0, frame->document()->referrerPolicy() == ReferrerPolicy::NoReferrerWhenDowngrade);
108         completionHandler(WTFMove(loader));
109     });
110 }
111
112 static Seconds maximumBufferingTime(CachedResource* resource)
113 {
114     if (!resource)
115         return 0_s;
116
117     switch (resource->type()) {
118     case CachedResource::Beacon:
119     case CachedResource::CSSStyleSheet:
120     case CachedResource::Script:
121 #if ENABLE(SVG_FONTS)
122     case CachedResource::SVGFontResource:
123 #endif
124     case CachedResource::FontResource:
125 #if ENABLE(APPLICATION_MANIFEST)
126     case CachedResource::ApplicationManifest:
127 #endif
128         return Seconds::infinity();
129     case CachedResource::ImageResource:
130         return 500_ms;
131     case CachedResource::MediaResource:
132         return 50_ms;
133     case CachedResource::MainResource:
134     case CachedResource::Icon:
135     case CachedResource::RawResource:
136     case CachedResource::SVGDocumentResource:
137     case CachedResource::LinkPrefetch:
138 #if ENABLE(VIDEO_TRACK)
139     case CachedResource::TextTrackResource:
140 #endif
141 #if ENABLE(XSLT)
142     case CachedResource::XSLStyleSheet:
143 #endif
144         return 0_s;
145     }
146
147     ASSERT_NOT_REACHED();
148     return 0_s;
149 }
150
151 void WebLoaderStrategy::scheduleLoad(ResourceLoader& resourceLoader, CachedResource* resource, bool shouldClearReferrerOnHTTPSToHTTPRedirect)
152 {
153     ResourceLoadIdentifier identifier = resourceLoader.identifier();
154     ASSERT(identifier);
155
156     auto& frameLoaderClient = resourceLoader.frameLoader()->client();
157
158     WebResourceLoader::TrackingParameters trackingParameters;
159     trackingParameters.pageID = frameLoaderClient.pageID().value_or(0);
160     trackingParameters.frameID = frameLoaderClient.frameID().value_or(0);
161     trackingParameters.resourceID = identifier;
162     auto sessionID = frameLoaderClient.sessionID();
163
164 #if ENABLE(WEB_ARCHIVE) || ENABLE(MHTML)
165     // If the DocumentLoader schedules this as an archive resource load,
166     // then we should remember the ResourceLoader in our records but not schedule it in the NetworkProcess.
167     if (resourceLoader.documentLoader()->scheduleArchiveLoad(resourceLoader, resourceLoader.request())) {
168         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be handled as an archive resource.", resourceLoader.url().string().utf8().data());
169         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled as an archive resource (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
170         m_webResourceLoaders.set(identifier, WebResourceLoader::create(resourceLoader, trackingParameters));
171         return;
172     }
173 #endif
174
175     if (resourceLoader.documentLoader()->applicationCacheHost().maybeLoadResource(resourceLoader, resourceLoader.request(), resourceLoader.request().url())) {
176         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be loaded from application cache.", resourceLoader.url().string().utf8().data());
177         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be loaded from application cache (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
178         m_webResourceLoaders.set(identifier, WebResourceLoader::create(resourceLoader, trackingParameters));
179         return;
180     }
181
182     if (resourceLoader.request().url().protocolIsData()) {
183         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be loaded as data.", resourceLoader.url().string().utf8().data());
184         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be loaded as data (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
185         startLocalLoad(resourceLoader);
186         return;
187     }
188
189 #if USE(QUICK_LOOK)
190     if (isQuickLookPreviewURL(resourceLoader.request().url())) {
191         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be handled as a QuickLook resource.", resourceLoader.url().string().utf8().data());
192         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled as a QuickLook resource (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
193         startLocalLoad(resourceLoader);
194         return;
195     }
196 #endif
197
198 #if USE(SOUP)
199     // For apps that call g_resource_load in a web extension.
200     // https://blogs.gnome.org/alexl/2012/01/26/resources-in-glib/
201     if (resourceLoader.request().url().protocolIs("resource")) {
202         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be handled as a GResource.", resourceLoader.url().string().utf8().data());
203         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled as a GResource (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
204         startLocalLoad(resourceLoader);
205         return;
206     }
207 #endif
208
209 #if ENABLE(SERVICE_WORKER)
210     WebServiceWorkerProvider::singleton().handleFetch(resourceLoader, resource, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, [trackingParameters, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, maximumBufferingTime = maximumBufferingTime(resource), resourceLoader = makeRef(resourceLoader)] (ServiceWorkerClientFetch::Result result) mutable {
211         if (result != ServiceWorkerClientFetch::Result::Unhandled) {
212             LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be scheduled through ServiceWorker handle fetch algorithm", resourceLoader->url().string().latin1().data());
213             return;
214         }
215         if (resourceLoader->options().serviceWorkersMode == ServiceWorkersMode::Only) {
216             callOnMainThread([resourceLoader = WTFMove(resourceLoader)] {
217                 auto error = internalError(resourceLoader->request().url());
218                 error.setType(ResourceError::Type::Cancellation);
219                 resourceLoader->didFail(error);
220             });
221             return;
222         }
223
224         if (!WebProcess::singleton().webLoaderStrategy().tryLoadingUsingURLSchemeHandler(resourceLoader))
225             WebProcess::singleton().webLoaderStrategy().scheduleLoadFromNetworkProcess(resourceLoader.get(), resourceLoader->request(), trackingParameters, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, maximumBufferingTime);
226     });
227 #else
228     if (!tryLoadingUsingURLSchemeHandler(resourceLoader))
229         scheduleLoadFromNetworkProcess(resourceLoader, resourceLoader.request(), trackingParameters, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, maximumBufferingTime(resource));
230 #endif
231 }
232
233 bool WebLoaderStrategy::tryLoadingUsingURLSchemeHandler(ResourceLoader& resourceLoader)
234 {
235     auto* webFrameLoaderClient = toWebFrameLoaderClient(resourceLoader.frameLoader()->client());
236     auto* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
237     auto* webPage = webFrame ? webFrame->page() : nullptr;
238     if (webPage) {
239         if (auto* handler = webPage->urlSchemeHandlerForScheme(resourceLoader.request().url().protocol().toStringWithoutCopying())) {
240             LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, URL '%s' will be handled by a UIProcess URL scheme handler.", resourceLoader.url().string().utf8().data());
241             RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled by a UIProcess URL scheme handler (frame = %p, resourceID = %lu)", resourceLoader.frame(), resourceLoader.identifier());
242
243             handler->startNewTask(resourceLoader);
244             return true;
245         }
246     }
247     return false;
248 }
249
250 void WebLoaderStrategy::scheduleLoadFromNetworkProcess(ResourceLoader& resourceLoader, const ResourceRequest& request, const WebResourceLoader::TrackingParameters& trackingParameters, PAL::SessionID sessionID, bool shouldClearReferrerOnHTTPSToHTTPRedirect, Seconds maximumBufferingTime)
251 {
252     ResourceLoadIdentifier identifier = resourceLoader.identifier();
253     ASSERT(identifier);
254
255     LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be scheduled with the NetworkProcess with priority %d", resourceLoader.url().string().latin1().data(), static_cast<int>(resourceLoader.request().priority()));
256
257     ContentSniffingPolicy contentSniffingPolicy = resourceLoader.shouldSniffContent() ? SniffContent : DoNotSniffContent;
258     ContentEncodingSniffingPolicy contentEncodingSniffingPolicy = resourceLoader.shouldSniffContentEncoding() ? ContentEncodingSniffingPolicy::Sniff : ContentEncodingSniffingPolicy::DoNotSniff;
259     StoredCredentialsPolicy storedCredentialsPolicy = resourceLoader.shouldUseCredentialStorage() ? StoredCredentialsPolicy::Use : StoredCredentialsPolicy::DoNotUse;
260
261     NetworkResourceLoadParameters loadParameters;
262     loadParameters.identifier = identifier;
263     loadParameters.webPageID = trackingParameters.pageID;
264     loadParameters.webFrameID = trackingParameters.frameID;
265     loadParameters.sessionID = sessionID;
266     loadParameters.request = request;
267     loadParameters.contentSniffingPolicy = contentSniffingPolicy;
268     loadParameters.contentEncodingSniffingPolicy = contentEncodingSniffingPolicy;
269     loadParameters.storedCredentialsPolicy = storedCredentialsPolicy;
270     // If there is no WebFrame then this resource cannot be authenticated with the client.
271     loadParameters.clientCredentialPolicy = (loadParameters.webFrameID && loadParameters.webPageID && resourceLoader.isAllowedToAskUserForCredentials()) ? ClientCredentialPolicy::MayAskClientForCredentials : ClientCredentialPolicy::CannotAskClientForCredentials;
272     loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect = shouldClearReferrerOnHTTPSToHTTPRedirect;
273     loadParameters.defersLoading = resourceLoader.defersLoading();
274     loadParameters.needsCertificateInfo = resourceLoader.shouldIncludeCertificateInfo();
275     loadParameters.maximumBufferingTime = maximumBufferingTime;
276     loadParameters.derivedCachedDataTypesToRetrieve = resourceLoader.options().derivedCachedDataTypesToRetrieve;
277     loadParameters.options = resourceLoader.options();
278     loadParameters.preflightPolicy = resourceLoader.options().preflightPolicy;
279
280     auto* document = resourceLoader.frame() ? resourceLoader.frame()->document() : nullptr;
281     if (resourceLoader.options().cspResponseHeaders)
282         loadParameters.cspResponseHeaders = resourceLoader.options().cspResponseHeaders;
283     else if (document && !document->shouldBypassMainWorldContentSecurityPolicy()) {
284         if (auto* contentSecurityPolicy = document->contentSecurityPolicy())
285             loadParameters.cspResponseHeaders = contentSecurityPolicy->responseHeaders();
286     }
287
288     if (resourceLoader.isSubresourceLoader()) {
289         if (auto* headers = static_cast<SubresourceLoader&>(resourceLoader).originalHeaders())
290             loadParameters.originalRequestHeaders = *headers;
291     }
292
293 #if ENABLE(CONTENT_EXTENSIONS)
294     if (document) {
295         loadParameters.mainDocumentURL = document->topDocument().url();
296         // FIXME: Instead of passing userContentControllerIdentifier, the NetworkProcess should be able to get it using webPageId.
297         auto* webFrameLoaderClient = toWebFrameLoaderClient(resourceLoader.frame()->loader().client());
298         auto* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
299         auto* webPage = webFrame ? webFrame->page() : nullptr;
300         if (webPage)
301             loadParameters.userContentControllerIdentifier = webPage->userContentControllerIdentifier();
302     }
303 #endif
304
305     // FIXME: All loaders should provide their origin if navigation mode is cors/no-cors/same-origin.
306     // As a temporary approach, we use the document origin if available or the HTTP Origin header otherwise.
307     if (resourceLoader.isSubresourceLoader())
308         loadParameters.sourceOrigin = static_cast<SubresourceLoader&>(resourceLoader).origin();
309
310     if (!loadParameters.sourceOrigin && document)
311         loadParameters.sourceOrigin = &document->securityOrigin();
312     if (!loadParameters.sourceOrigin) {
313         auto origin = request.httpOrigin();
314         if (!origin.isNull())
315             loadParameters.sourceOrigin = SecurityOrigin::createFromString(origin);
316     }
317
318     if (loadParameters.options.mode != FetchOptions::Mode::Navigate) {
319         ASSERT(loadParameters.sourceOrigin);
320         if (!loadParameters.sourceOrigin) {
321             scheduleInternallyFailedLoad(resourceLoader);
322             return;
323         }
324     }
325
326     loadParameters.shouldRestrictHTTPResponseAccess = shouldPerformSecurityChecks();
327
328     loadParameters.isMainFrameNavigation = resourceLoader.frame() && resourceLoader.frame()->isMainFrame() && resourceLoader.options().mode == FetchOptions::Mode::Navigate;
329
330     loadParameters.shouldEnableCrossOriginResourcePolicy = RuntimeEnabledFeatures::sharedFeatures().crossOriginResourcePolicyEnabled() && !loadParameters.isMainFrameNavigation;
331
332     if (resourceLoader.options().mode == FetchOptions::Mode::Navigate) {
333         Vector<RefPtr<SecurityOrigin>> frameAncestorOrigins;
334         for (auto* frame = resourceLoader.frame(); frame; frame = frame->tree().parent())
335             frameAncestorOrigins.append(makeRefPtr(frame->document()->securityOrigin()));
336         loadParameters.frameAncestorOrigins = WTFMove(frameAncestorOrigins);
337     }
338
339     ASSERT((loadParameters.webPageID && loadParameters.webFrameID) || loadParameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials);
340
341     RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: Resource is being scheduled with the NetworkProcess (frame = %p, priority = %d, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), static_cast<int>(resourceLoader.request().priority()), loadParameters.webPageID, loadParameters.webFrameID, loadParameters.identifier);
342     if (!WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::ScheduleResourceLoad(loadParameters), 0)) {
343         RELEASE_LOG_ERROR_IF_ALLOWED(resourceLoader, "scheduleLoad: Unable to schedule resource with the NetworkProcess (frame = %p, priority = %d, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), static_cast<int>(resourceLoader.request().priority()), loadParameters.webPageID, loadParameters.webFrameID, loadParameters.identifier);
344         // We probably failed to schedule this load with the NetworkProcess because it had crashed.
345         // This load will never succeed so we will schedule it to fail asynchronously.
346         scheduleInternallyFailedLoad(resourceLoader);
347         return;
348     }
349
350     m_webResourceLoaders.set(identifier, WebResourceLoader::create(resourceLoader, trackingParameters));
351 }
352
353 void WebLoaderStrategy::scheduleInternallyFailedLoad(WebCore::ResourceLoader& resourceLoader)
354 {
355     m_internallyFailedResourceLoaders.add(&resourceLoader);
356     m_internallyFailedLoadTimer.startOneShot(0_s);
357 }
358
359 void WebLoaderStrategy::internallyFailedLoadTimerFired()
360 {
361     for (auto& resourceLoader : copyToVector(m_internallyFailedResourceLoaders))
362         resourceLoader->didFail(internalError(resourceLoader->url()));
363 }
364
365 void WebLoaderStrategy::startLocalLoad(WebCore::ResourceLoader& resourceLoader)
366 {
367     resourceLoader.start();
368     m_webResourceLoaders.set(resourceLoader.identifier(), WebResourceLoader::create(resourceLoader, { }));
369 }
370
371 void WebLoaderStrategy::addURLSchemeTaskProxy(WebURLSchemeTaskProxy& task)
372 {
373     auto result = m_urlSchemeTasks.add(task.identifier(), &task);
374     ASSERT_UNUSED(result, result.isNewEntry);
375 }
376
377 void WebLoaderStrategy::removeURLSchemeTaskProxy(WebURLSchemeTaskProxy& task)
378 {
379     m_urlSchemeTasks.remove(task.identifier());
380 }
381
382 void WebLoaderStrategy::remove(ResourceLoader* resourceLoader)
383 {
384     ASSERT(resourceLoader);
385     LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::remove, url '%s'", resourceLoader->url().string().utf8().data());
386
387     if (auto task = m_urlSchemeTasks.take(resourceLoader->identifier())) {
388         ASSERT(!m_internallyFailedResourceLoaders.contains(resourceLoader));
389         task->stopLoading();
390         return;
391     }
392
393     if (m_internallyFailedResourceLoaders.contains(resourceLoader)) {
394         m_internallyFailedResourceLoaders.remove(resourceLoader);
395         return;
396     }
397     
398     ResourceLoadIdentifier identifier = resourceLoader->identifier();
399     if (!identifier) {
400         LOG_ERROR("WebLoaderStrategy removing a ResourceLoader that has no identifier.");
401         return;
402     }
403
404 #if ENABLE(SERVICE_WORKER)
405     if (WebServiceWorkerProvider::singleton().cancelFetch(makeObjectIdentifier<FetchIdentifierType>(identifier)))
406         return;
407 #endif
408
409     RefPtr<WebResourceLoader> loader = m_webResourceLoaders.take(identifier);
410     // Loader may not be registered if we created it, but haven't scheduled yet (a bundle client can decide to cancel such request via willSendRequest).
411     if (!loader)
412         return;
413
414     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::RemoveLoadIdentifier(identifier), 0);
415
416     // It's possible that this WebResourceLoader might be just about to message back to the NetworkProcess (e.g. ContinueWillSendRequest)
417     // but there's no point in doing so anymore.
418     loader->detachFromCoreLoader();
419 }
420
421 void WebLoaderStrategy::setDefersLoading(ResourceLoader* resourceLoader, bool defers)
422 {
423     ResourceLoadIdentifier identifier = resourceLoader->identifier();
424     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::SetDefersLoading(identifier, defers), 0);
425 }
426
427 void WebLoaderStrategy::crossOriginRedirectReceived(ResourceLoader*, const URL&)
428 {
429     // We handle cross origin redirects entirely within the NetworkProcess.
430     // We override this call in the WebProcess to make it a no-op.
431 }
432
433 void WebLoaderStrategy::servePendingRequests(ResourceLoadPriority)
434 {
435     // This overrides the base class version.
436     // We don't need to do anything as this is handled by the network process.
437 }
438
439 void WebLoaderStrategy::suspendPendingRequests()
440 {
441     // Network process does keep requests in pending state.
442 }
443
444 void WebLoaderStrategy::resumePendingRequests()
445 {
446     // Network process does keep requests in pending state.
447 }
448
449 void WebLoaderStrategy::networkProcessCrashed()
450 {
451     RELEASE_LOG_ERROR(Network, "WebLoaderStrategy::networkProcessCrashed: failing all pending resource loaders");
452
453     for (auto& loader : m_webResourceLoaders)
454         scheduleInternallyFailedLoad(*loader.value->resourceLoader());
455
456     m_webResourceLoaders.clear();
457
458     auto pingLoadCompletionHandlers = WTFMove(m_pingLoadCompletionHandlers);
459     for (auto& pingLoadCompletionHandler : pingLoadCompletionHandlers.values())
460         pingLoadCompletionHandler(internalError(URL()), { });
461
462     auto preconnectCompletionHandlers = WTFMove(m_preconnectCompletionHandlers);
463     for (auto& preconnectCompletionHandler : preconnectCompletionHandlers.values())
464         preconnectCompletionHandler(internalError(URL()));
465 }
466
467 static bool shouldClearReferrerOnHTTPSToHTTPRedirect(Frame* frame)
468 {
469     if (frame) {
470         if (auto* document = frame->document())
471             return document->referrerPolicy() == ReferrerPolicy::NoReferrerWhenDowngrade;
472     }
473     return true;
474 }
475
476 std::optional<WebLoaderStrategy::SyncLoadResult> WebLoaderStrategy::tryLoadingSynchronouslyUsingURLSchemeHandler(FrameLoader& frameLoader, ResourceLoadIdentifier identifier, const ResourceRequest& request)
477 {
478     auto* webFrameLoaderClient = toWebFrameLoaderClient(frameLoader.client());
479     auto* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
480     auto* webPage = webFrame ? webFrame->page() : nullptr;
481     if (!webPage)
482         return std::nullopt;
483
484     auto* handler = webPage->urlSchemeHandlerForScheme(request.url().protocol().toStringWithoutCopying());
485     if (!handler)
486         return std::nullopt;
487
488     LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, sync load to URL '%s' will be handled by a UIProcess URL scheme handler.", request.url().string().utf8().data());
489
490     SyncLoadResult result;
491     handler->loadSynchronously(identifier, request, result.response, result.error, result.data);
492
493     return WTFMove(result);
494 }
495
496 void WebLoaderStrategy::loadResourceSynchronously(FrameLoader& frameLoader, unsigned long resourceLoadIdentifier, const ResourceRequest& request, ClientCredentialPolicy clientCredentialPolicy,  const FetchOptions& options, const HTTPHeaderMap& originalRequestHeaders, ResourceError& error, ResourceResponse& response, Vector<char>& data)
497 {
498     auto* document = frameLoader.frame().document();
499     if (!document) {
500         error = internalError(request.url());
501         return;
502     }
503
504     if (auto syncLoadResult = tryLoadingSynchronouslyUsingURLSchemeHandler(frameLoader, resourceLoadIdentifier, request)) {
505         error = WTFMove(syncLoadResult->error);
506         response = WTFMove(syncLoadResult->response);
507         data = WTFMove(syncLoadResult->data);
508         return;
509     }
510
511     WebFrameLoaderClient* webFrameLoaderClient = toWebFrameLoaderClient(frameLoader.client());
512     WebFrame* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
513     WebPage* webPage = webFrame ? webFrame->page() : nullptr;
514
515     NetworkResourceLoadParameters loadParameters;
516     loadParameters.identifier = resourceLoadIdentifier;
517     loadParameters.webPageID = webPage ? webPage->pageID() : 0;
518     loadParameters.webFrameID = webFrame ? webFrame->frameID() : 0;
519     loadParameters.sessionID = webPage ? webPage->sessionID() : PAL::SessionID::defaultSessionID();
520     loadParameters.request = request;
521     loadParameters.contentSniffingPolicy = SniffContent;
522     loadParameters.contentEncodingSniffingPolicy = ContentEncodingSniffingPolicy::Sniff;
523     loadParameters.storedCredentialsPolicy = options.credentials == FetchOptions::Credentials::Omit ? StoredCredentialsPolicy::DoNotUse : StoredCredentialsPolicy::Use;
524     loadParameters.clientCredentialPolicy = clientCredentialPolicy;
525     loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect = shouldClearReferrerOnHTTPSToHTTPRedirect(webFrame ? webFrame->coreFrame() : nullptr);
526     loadParameters.shouldRestrictHTTPResponseAccess = shouldPerformSecurityChecks();
527
528     loadParameters.options = options;
529     loadParameters.sourceOrigin = &document->securityOrigin();
530     if (!document->shouldBypassMainWorldContentSecurityPolicy()) {
531         if (auto* contentSecurityPolicy = document->contentSecurityPolicy())
532             loadParameters.cspResponseHeaders = contentSecurityPolicy->responseHeaders();
533     }
534     loadParameters.originalRequestHeaders = originalRequestHeaders;
535
536     data.shrink(0);
537
538     HangDetectionDisabler hangDetectionDisabler;
539
540     if (!WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad(loadParameters), Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::Reply(error, response, data), 0, Seconds::infinity(), IPC::SendSyncOption::DoNotProcessIncomingMessagesWhenWaitingForSyncReply)) {
541         RELEASE_LOG_ERROR_IF_ALLOWED(loadParameters.sessionID, "loadResourceSynchronously: failed sending synchronous network process message (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", loadParameters.webPageID, loadParameters.webFrameID, loadParameters.identifier);
542         if (auto* page = webPage->corePage())
543             page->diagnosticLoggingClient().logDiagnosticMessage(WebCore::DiagnosticLoggingKeys::internalErrorKey(), WebCore::DiagnosticLoggingKeys::synchronousMessageFailedKey(), WebCore::ShouldSample::No);
544         response = ResourceResponse();
545         error = internalError(request.url());
546     }
547 }
548
549 void WebLoaderStrategy::pageLoadCompleted(uint64_t webPageID)
550 {
551     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::PageLoadCompleted(webPageID), 0);
552 }
553
554 static uint64_t generateLoadIdentifier()
555 {
556     static uint64_t identifier = 0;
557     return ++identifier;
558 }
559
560 void WebLoaderStrategy::startPingLoad(Frame& frame, ResourceRequest& request, const HTTPHeaderMap& originalRequestHeaders, const FetchOptions& options, PingLoadCompletionHandler&& completionHandler)
561 {
562     auto* document = frame.document();
563     if (!document) {
564         if (completionHandler)
565             completionHandler(internalError(request.url()), { });
566         return;
567     }
568
569     NetworkResourceLoadParameters loadParameters;
570     loadParameters.identifier = generateLoadIdentifier();
571     loadParameters.request = request;
572     loadParameters.sourceOrigin = &document->securityOrigin();
573     loadParameters.sessionID = frame.page() ? frame.page()->sessionID() : PAL::SessionID::defaultSessionID();
574     loadParameters.storedCredentialsPolicy = options.credentials == FetchOptions::Credentials::Omit ? StoredCredentialsPolicy::DoNotUse : StoredCredentialsPolicy::Use;
575     loadParameters.options = options;
576     loadParameters.originalRequestHeaders = originalRequestHeaders;
577     loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect = shouldClearReferrerOnHTTPSToHTTPRedirect(&frame);
578     loadParameters.shouldRestrictHTTPResponseAccess = shouldPerformSecurityChecks();
579     if (!document->shouldBypassMainWorldContentSecurityPolicy()) {
580         if (auto * contentSecurityPolicy = document->contentSecurityPolicy())
581             loadParameters.cspResponseHeaders = contentSecurityPolicy->responseHeaders();
582     }
583
584 #if ENABLE(CONTENT_EXTENSIONS)
585     loadParameters.mainDocumentURL = document->topDocument().url();
586     // FIXME: Instead of passing userContentControllerIdentifier, we should just pass webPageId to NetworkProcess.
587     WebFrameLoaderClient* webFrameLoaderClient = toWebFrameLoaderClient(frame.loader().client());
588     WebFrame* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
589     WebPage* webPage = webFrame ? webFrame->page() : nullptr;
590     if (webPage)
591         loadParameters.userContentControllerIdentifier = webPage->userContentControllerIdentifier();
592 #endif
593
594     if (completionHandler)
595         m_pingLoadCompletionHandlers.add(loadParameters.identifier, WTFMove(completionHandler));
596
597     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::LoadPing { loadParameters }, 0);
598 }
599
600 void WebLoaderStrategy::didFinishPingLoad(uint64_t pingLoadIdentifier, ResourceError&& error, ResourceResponse&& response)
601 {
602     if (auto completionHandler = m_pingLoadCompletionHandlers.take(pingLoadIdentifier))
603         completionHandler(WTFMove(error), WTFMove(response));
604 }
605
606 void WebLoaderStrategy::preconnectTo(FrameLoader& frameLoader, const WebCore::URL& url, StoredCredentialsPolicy storedCredentialsPolicy, PreconnectCompletionHandler&& completionHandler)
607 {
608     uint64_t preconnectionIdentifier = generateLoadIdentifier();
609     auto addResult = m_preconnectCompletionHandlers.add(preconnectionIdentifier, WTFMove(completionHandler));
610     ASSERT_UNUSED(addResult, addResult.isNewEntry);
611
612     auto* webFrameLoaderClient = toWebFrameLoaderClient(frameLoader.client());
613     if (!webFrameLoaderClient) {
614         completionHandler(internalError(url));
615         return;
616     }
617     auto* webFrame = webFrameLoaderClient->webFrame();
618     if (!webFrame) {
619         completionHandler(internalError(url));
620         return;
621     }
622     auto* webPage = webFrame->page();
623     if (!webPage) {
624         completionHandler(internalError(url));
625         return;
626     }
627
628     NetworkResourceLoadParameters parameters;
629     parameters.request = ResourceRequest { url };
630     parameters.webPageID = webPage ? webPage->pageID() : 0;
631     parameters.webFrameID = webFrame ? webFrame->frameID() : 0;
632     parameters.sessionID = webPage ? webPage->sessionID() : PAL::SessionID::defaultSessionID();
633     parameters.storedCredentialsPolicy = storedCredentialsPolicy;
634     parameters.shouldPreconnectOnly = PreconnectOnly::Yes;
635     parameters.shouldRestrictHTTPResponseAccess = shouldPerformSecurityChecks();
636     // FIXME: Use the proper destination once all fetch options are passed.
637     parameters.options.destination = FetchOptions::Destination::EmptyString;
638
639     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::PreconnectTo(preconnectionIdentifier, WTFMove(parameters)), 0);
640 }
641
642 void WebLoaderStrategy::didFinishPreconnection(uint64_t preconnectionIdentifier, ResourceError&& error)
643 {
644     if (auto completionHandler = m_preconnectCompletionHandlers.take(preconnectionIdentifier))
645         completionHandler(WTFMove(error));
646 }
647
648 void WebLoaderStrategy::storeDerivedDataToCache(const SHA1::Digest& bodyHash, const String& type, const String& partition, WebCore::SharedBuffer& data)
649 {
650     NetworkCache::DataKey key { partition, type, bodyHash };
651     IPC::SharedBufferDataReference dataReference { &data };
652     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::StoreDerivedDataToCache(key, dataReference), 0);
653 }
654
655 bool WebLoaderStrategy::isOnLine() const
656 {
657     return m_isOnLine;
658 }
659
660 void WebLoaderStrategy::addOnlineStateChangeListener(Function<void(bool)>&& listener)
661 {
662     WebProcess::singleton().ensureNetworkProcessConnection();
663     m_onlineStateChangeListeners.append(WTFMove(listener));
664 }
665
666 void WebLoaderStrategy::setOnLineState(bool isOnLine)
667 {
668     if (m_isOnLine == isOnLine)
669         return;
670
671     m_isOnLine = isOnLine;
672     for (auto& listener : m_onlineStateChangeListeners)
673         listener(isOnLine);
674 }
675
676 void WebLoaderStrategy::setCaptureExtraNetworkLoadMetricsEnabled(bool enabled)
677 {
678     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::SetCaptureExtraNetworkLoadMetricsEnabled(enabled), 0);
679 }
680
681 ResourceResponse WebLoaderStrategy::responseFromResourceLoadIdentifier(uint64_t resourceLoadIdentifier)
682 {
683     ResourceResponse response;
684     WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::GetNetworkLoadInformationResponse { resourceLoadIdentifier }, Messages::NetworkConnectionToWebProcess::GetNetworkLoadInformationResponse::Reply { response }, 0, Seconds::infinity(), IPC::SendSyncOption::DoNotProcessIncomingMessagesWhenWaitingForSyncReply);
685     return response;
686 }
687
688 Vector<NetworkTransactionInformation> WebLoaderStrategy::intermediateLoadInformationFromResourceLoadIdentifier(uint64_t resourceLoadIdentifier)
689 {
690     Vector<NetworkTransactionInformation> information;
691     WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::GetNetworkLoadIntermediateInformation { resourceLoadIdentifier }, Messages::NetworkConnectionToWebProcess::GetNetworkLoadIntermediateInformation::Reply { information }, 0, Seconds::infinity(), IPC::SendSyncOption::DoNotProcessIncomingMessagesWhenWaitingForSyncReply);
692     return information;
693 }
694
695 NetworkLoadMetrics WebLoaderStrategy::networkMetricsFromResourceLoadIdentifier(uint64_t resourceLoadIdentifier)
696 {
697     NetworkLoadMetrics networkMetrics;
698     WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::TakeNetworkLoadInformationMetrics { resourceLoadIdentifier }, Messages::NetworkConnectionToWebProcess::TakeNetworkLoadInformationMetrics::Reply { networkMetrics }, 0, Seconds::infinity(), IPC::SendSyncOption::DoNotProcessIncomingMessagesWhenWaitingForSyncReply);
699     return networkMetrics;
700 }
701
702 bool WebLoaderStrategy::shouldPerformSecurityChecks() const
703 {
704     return RuntimeEnabledFeatures::sharedFeatures().restrictedHTTPResponseAccess();
705 }
706
707 bool WebLoaderStrategy::havePerformedSecurityChecks(const ResourceResponse& response) const
708 {
709     if (!shouldPerformSecurityChecks())
710         return false;
711     switch (response.source()) {
712     case ResourceResponse::Source::ApplicationCache:
713     case ResourceResponse::Source::MemoryCache:
714     case ResourceResponse::Source::MemoryCacheAfterValidation:
715     case ResourceResponse::Source::ServiceWorker:
716         return false;
717     case ResourceResponse::Source::DiskCache:
718     case ResourceResponse::Source::DiskCacheAfterValidation:
719     case ResourceResponse::Source::Network:
720     case ResourceResponse::Source::Unknown:
721         return true;
722     }
723     ASSERT_NOT_REACHED();
724     return false;
725 }
726
727 } // namespace WebKit