Use NetworkLoadChecker for navigation loads
[WebKit-https.git] / Source / WebKit / WebProcess / Network / WebLoaderStrategy.cpp
1 /*
2  * Copyright (C) 2012, 2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebLoaderStrategy.h"
28
29 #include "DataReference.h"
30 #include "HangDetectionDisabler.h"
31 #include "Logging.h"
32 #include "NetworkConnectionToWebProcessMessages.h"
33 #include "NetworkProcessConnection.h"
34 #include "NetworkResourceLoadParameters.h"
35 #include "SessionTracker.h"
36 #include "WebCompiledContentRuleList.h"
37 #include "WebCoreArgumentCoders.h"
38 #include "WebErrors.h"
39 #include "WebFrame.h"
40 #include "WebFrameLoaderClient.h"
41 #include "WebPage.h"
42 #include "WebPageProxyMessages.h"
43 #include "WebProcess.h"
44 #include "WebResourceLoader.h"
45 #include "WebServiceWorkerProvider.h"
46 #include "WebURLSchemeHandlerProxy.h"
47 #include "WebURLSchemeTaskProxy.h"
48 #include <WebCore/ApplicationCacheHost.h>
49 #include <WebCore/CachedResource.h>
50 #include <WebCore/ContentSecurityPolicy.h>
51 #include <WebCore/DiagnosticLoggingClient.h>
52 #include <WebCore/DiagnosticLoggingKeys.h>
53 #include <WebCore/Document.h>
54 #include <WebCore/DocumentLoader.h>
55 #include <WebCore/FetchOptions.h>
56 #include <WebCore/Frame.h>
57 #include <WebCore/FrameLoader.h>
58 #include <WebCore/NetscapePlugInStreamLoader.h>
59 #include <WebCore/PlatformStrategies.h>
60 #include <WebCore/ReferrerPolicy.h>
61 #include <WebCore/ResourceLoader.h>
62 #include <WebCore/RuntimeEnabledFeatures.h>
63 #include <WebCore/SecurityOrigin.h>
64 #include <WebCore/Settings.h>
65 #include <WebCore/SubresourceLoader.h>
66 #include <WebCore/UserContentProvider.h>
67 #include <pal/SessionID.h>
68 #include <wtf/CompletionHandler.h>
69 #include <wtf/text/CString.h>
70
71 #if USE(QUICK_LOOK)
72 #include <WebCore/QuickLook.h>
73 #endif
74
75 using namespace WebCore;
76
77 #define RELEASE_LOG_IF_ALLOWED(permissionChecker, fmt, ...) RELEASE_LOG_IF(permissionChecker.isAlwaysOnLoggingAllowed(), Network, "%p - WebLoaderStrategy::" fmt, this, ##__VA_ARGS__)
78 #define RELEASE_LOG_ERROR_IF_ALLOWED(permissionChecker, fmt, ...) RELEASE_LOG_ERROR_IF(permissionChecker.isAlwaysOnLoggingAllowed(), Network, "%p - WebLoaderStrategy::" fmt, this, ##__VA_ARGS__)
79
80 namespace WebKit {
81
82 WebLoaderStrategy::WebLoaderStrategy()
83     : m_internallyFailedLoadTimer(RunLoop::main(), this, &WebLoaderStrategy::internallyFailedLoadTimerFired)
84 {
85 }
86
87 WebLoaderStrategy::~WebLoaderStrategy()
88 {
89 }
90
91 void WebLoaderStrategy::loadResource(Frame& frame, CachedResource& resource, ResourceRequest&& request, const ResourceLoaderOptions& options, CompletionHandler<void(RefPtr<SubresourceLoader>&&)>&& completionHandler)
92 {
93     SubresourceLoader::create(frame, resource, WTFMove(request), options, [this, completionHandler = WTFMove(completionHandler), resource = CachedResourceHandle<CachedResource>(&resource), frame = makeRef(frame)] (RefPtr<SubresourceLoader>&& loader) mutable {
94         if (loader)
95             scheduleLoad(*loader, resource.get(), frame->document()->referrerPolicy() == ReferrerPolicy::NoReferrerWhenDowngrade);
96         else
97             RELEASE_LOG_IF_ALLOWED(frame.get(), "loadResource: Unable to create SubresourceLoader (frame = %p", &frame);
98         completionHandler(WTFMove(loader));
99     });
100 }
101
102 void WebLoaderStrategy::schedulePluginStreamLoad(Frame& frame, NetscapePlugInStreamLoaderClient& client, ResourceRequest&& request, CompletionHandler<void(RefPtr<NetscapePlugInStreamLoader>&&)>&& completionHandler)
103 {
104     NetscapePlugInStreamLoader::create(frame, client, WTFMove(request), [this, completionHandler = WTFMove(completionHandler), frame = makeRef(frame)] (RefPtr<NetscapePlugInStreamLoader>&& loader) mutable {
105         if (loader)
106             scheduleLoad(*loader, 0, frame->document()->referrerPolicy() == ReferrerPolicy::NoReferrerWhenDowngrade);
107         completionHandler(WTFMove(loader));
108     });
109 }
110
111 static Seconds maximumBufferingTime(CachedResource* resource)
112 {
113     if (!resource)
114         return 0_s;
115
116     switch (resource->type()) {
117     case CachedResource::Beacon:
118     case CachedResource::CSSStyleSheet:
119     case CachedResource::Script:
120 #if ENABLE(SVG_FONTS)
121     case CachedResource::SVGFontResource:
122 #endif
123     case CachedResource::FontResource:
124 #if ENABLE(APPLICATION_MANIFEST)
125     case CachedResource::ApplicationManifest:
126 #endif
127         return Seconds::infinity();
128     case CachedResource::ImageResource:
129         return 500_ms;
130     case CachedResource::MediaResource:
131         return 50_ms;
132     case CachedResource::MainResource:
133     case CachedResource::Icon:
134     case CachedResource::RawResource:
135     case CachedResource::SVGDocumentResource:
136     case CachedResource::LinkPrefetch:
137 #if ENABLE(VIDEO_TRACK)
138     case CachedResource::TextTrackResource:
139 #endif
140 #if ENABLE(XSLT)
141     case CachedResource::XSLStyleSheet:
142 #endif
143         return 0_s;
144     }
145
146     ASSERT_NOT_REACHED();
147     return 0_s;
148 }
149
150 void WebLoaderStrategy::scheduleLoad(ResourceLoader& resourceLoader, CachedResource* resource, bool shouldClearReferrerOnHTTPSToHTTPRedirect)
151 {
152     ResourceLoadIdentifier identifier = resourceLoader.identifier();
153     ASSERT(identifier);
154
155     auto& frameLoaderClient = resourceLoader.frameLoader()->client();
156
157     WebResourceLoader::TrackingParameters trackingParameters;
158     trackingParameters.pageID = frameLoaderClient.pageID().value();
159     trackingParameters.frameID = frameLoaderClient.frameID().value();
160     trackingParameters.resourceID = identifier;
161     auto sessionID = frameLoaderClient.sessionID();
162
163 #if ENABLE(WEB_ARCHIVE) || ENABLE(MHTML)
164     // If the DocumentLoader schedules this as an archive resource load,
165     // then we should remember the ResourceLoader in our records but not schedule it in the NetworkProcess.
166     if (resourceLoader.documentLoader()->scheduleArchiveLoad(resourceLoader, resourceLoader.request())) {
167         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be handled as an archive resource.", resourceLoader.url().string().utf8().data());
168         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled as an archive resource (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
169         m_webResourceLoaders.set(identifier, WebResourceLoader::create(resourceLoader, trackingParameters));
170         return;
171     }
172 #endif
173
174     if (resourceLoader.documentLoader()->applicationCacheHost().maybeLoadResource(resourceLoader, resourceLoader.request(), resourceLoader.request().url())) {
175         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be loaded from application cache.", resourceLoader.url().string().utf8().data());
176         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be loaded from application cache (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
177         m_webResourceLoaders.set(identifier, WebResourceLoader::create(resourceLoader, trackingParameters));
178         return;
179     }
180
181     if (resourceLoader.request().url().protocolIsData()) {
182         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be loaded as data.", resourceLoader.url().string().utf8().data());
183         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be loaded as data (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
184         startLocalLoad(resourceLoader);
185         return;
186     }
187
188 #if USE(QUICK_LOOK)
189     if (isQuickLookPreviewURL(resourceLoader.request().url())) {
190         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be handled as a QuickLook resource.", resourceLoader.url().string().utf8().data());
191         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled as a QuickLook resource (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
192         startLocalLoad(resourceLoader);
193         return;
194     }
195 #endif
196
197 #if USE(SOUP)
198     // For apps that call g_resource_load in a web extension.
199     // https://blogs.gnome.org/alexl/2012/01/26/resources-in-glib/
200     if (resourceLoader.request().url().protocolIs("resource")) {
201         LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be handled as a GResource.", resourceLoader.url().string().utf8().data());
202         RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled as a GResource (frame = %p, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), trackingParameters.pageID, trackingParameters.frameID, identifier);
203         startLocalLoad(resourceLoader);
204         return;
205     }
206 #endif
207
208 #if ENABLE(SERVICE_WORKER)
209     WebServiceWorkerProvider::singleton().handleFetch(resourceLoader, resource, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, [trackingParameters, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, maximumBufferingTime = maximumBufferingTime(resource), resourceLoader = makeRef(resourceLoader)] (ServiceWorkerClientFetch::Result result) mutable {
210         if (result != ServiceWorkerClientFetch::Result::Unhandled) {
211             LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be scheduled through ServiceWorker handle fetch algorithm", resourceLoader->url().string().latin1().data());
212             return;
213         }
214         if (resourceLoader->options().serviceWorkersMode == ServiceWorkersMode::Only) {
215             callOnMainThread([resourceLoader = WTFMove(resourceLoader)] {
216                 auto error = internalError(resourceLoader->request().url());
217                 error.setType(ResourceError::Type::Cancellation);
218                 resourceLoader->didFail(error);
219             });
220             return;
221         }
222
223         if (!WebProcess::singleton().webLoaderStrategy().tryLoadingUsingURLSchemeHandler(resourceLoader))
224             WebProcess::singleton().webLoaderStrategy().scheduleLoadFromNetworkProcess(resourceLoader.get(), resourceLoader->request(), trackingParameters, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, maximumBufferingTime);
225     });
226 #else
227     if (!tryLoadingUsingURLSchemeHandler(resourceLoader))
228         scheduleLoadFromNetworkProcess(resourceLoader, resourceLoader.request(), trackingParameters, sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect, maximumBufferingTime(resource));
229 #endif
230 }
231
232 bool WebLoaderStrategy::tryLoadingUsingURLSchemeHandler(ResourceLoader& resourceLoader)
233 {
234     auto* webFrameLoaderClient = toWebFrameLoaderClient(resourceLoader.frameLoader()->client());
235     auto* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
236     auto* webPage = webFrame ? webFrame->page() : nullptr;
237     if (webPage) {
238         if (auto* handler = webPage->urlSchemeHandlerForScheme(resourceLoader.request().url().protocol().toStringWithoutCopying())) {
239             LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, URL '%s' will be handled by a UIProcess URL scheme handler.", resourceLoader.url().string().utf8().data());
240             RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: URL will be handled by a UIProcess URL scheme handler (frame = %p, resourceID = %lu)", resourceLoader.frame(), resourceLoader.identifier());
241
242             handler->startNewTask(resourceLoader);
243             return true;
244         }
245     }
246     return false;
247 }
248
249 void WebLoaderStrategy::scheduleLoadFromNetworkProcess(ResourceLoader& resourceLoader, const ResourceRequest& request, const WebResourceLoader::TrackingParameters& trackingParameters, PAL::SessionID sessionID, bool shouldClearReferrerOnHTTPSToHTTPRedirect, Seconds maximumBufferingTime)
250 {
251     ResourceLoadIdentifier identifier = resourceLoader.identifier();
252     ASSERT(identifier);
253
254     LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::scheduleLoad, url '%s' will be scheduled with the NetworkProcess with priority %d", resourceLoader.url().string().latin1().data(), static_cast<int>(resourceLoader.request().priority()));
255
256     ContentSniffingPolicy contentSniffingPolicy = resourceLoader.shouldSniffContent() ? SniffContent : DoNotSniffContent;
257     ContentEncodingSniffingPolicy contentEncodingSniffingPolicy = resourceLoader.shouldSniffContentEncoding() ? ContentEncodingSniffingPolicy::Sniff : ContentEncodingSniffingPolicy::DoNotSniff;
258     StoredCredentialsPolicy storedCredentialsPolicy = resourceLoader.shouldUseCredentialStorage() ? StoredCredentialsPolicy::Use : StoredCredentialsPolicy::DoNotUse;
259
260     NetworkResourceLoadParameters loadParameters;
261     loadParameters.identifier = identifier;
262     loadParameters.webPageID = trackingParameters.pageID;
263     loadParameters.webFrameID = trackingParameters.frameID;
264     loadParameters.sessionID = sessionID;
265     loadParameters.request = request;
266     loadParameters.contentSniffingPolicy = contentSniffingPolicy;
267     loadParameters.contentEncodingSniffingPolicy = contentEncodingSniffingPolicy;
268     loadParameters.storedCredentialsPolicy = storedCredentialsPolicy;
269     // If there is no WebFrame then this resource cannot be authenticated with the client.
270     loadParameters.clientCredentialPolicy = (loadParameters.webFrameID && loadParameters.webPageID && resourceLoader.isAllowedToAskUserForCredentials()) ? ClientCredentialPolicy::MayAskClientForCredentials : ClientCredentialPolicy::CannotAskClientForCredentials;
271     loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect = shouldClearReferrerOnHTTPSToHTTPRedirect;
272     loadParameters.defersLoading = resourceLoader.defersLoading();
273     loadParameters.needsCertificateInfo = resourceLoader.shouldIncludeCertificateInfo();
274     loadParameters.maximumBufferingTime = maximumBufferingTime;
275     loadParameters.derivedCachedDataTypesToRetrieve = resourceLoader.options().derivedCachedDataTypesToRetrieve;
276     loadParameters.options = resourceLoader.options();
277     loadParameters.preflightPolicy = resourceLoader.options().preflightPolicy;
278
279     auto* document = resourceLoader.frame() ? resourceLoader.frame()->document() : nullptr;
280     if (resourceLoader.options().cspResponseHeaders)
281         loadParameters.cspResponseHeaders = resourceLoader.options().cspResponseHeaders;
282     else if (document && !document->shouldBypassMainWorldContentSecurityPolicy()) {
283         if (auto* contentSecurityPolicy = document->contentSecurityPolicy())
284             loadParameters.cspResponseHeaders = contentSecurityPolicy->responseHeaders();
285     }
286
287     if (resourceLoader.isSubresourceLoader()) {
288         if (auto* headers = static_cast<SubresourceLoader&>(resourceLoader).originalHeaders())
289             loadParameters.originalRequestHeaders = *headers;
290     }
291
292 #if ENABLE(CONTENT_EXTENSIONS)
293     if (document) {
294         loadParameters.mainDocumentURL = document->topDocument().url();
295         // FIXME: Instead of passing userContentControllerIdentifier, the NetworkProcess should be able to get it using webPageId.
296         auto* webFrameLoaderClient = toWebFrameLoaderClient(resourceLoader.frame()->loader().client());
297         auto* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
298         auto* webPage = webFrame ? webFrame->page() : nullptr;
299         if (webPage)
300             loadParameters.userContentControllerIdentifier = webPage->userContentControllerIdentifier();
301     }
302 #endif
303
304     // FIXME: All loaders should provide their origin if navigation mode is cors/no-cors/same-origin.
305     // As a temporary approach, we use the document origin if available or the HTTP Origin header otherwise.
306     if (resourceLoader.isSubresourceLoader())
307         loadParameters.sourceOrigin = static_cast<SubresourceLoader&>(resourceLoader).origin();
308
309     if (!loadParameters.sourceOrigin && document)
310         loadParameters.sourceOrigin = &document->securityOrigin();
311     if (!loadParameters.sourceOrigin) {
312         auto origin = request.httpOrigin();
313         if (!origin.isNull())
314             loadParameters.sourceOrigin = SecurityOrigin::createFromString(origin);
315     }
316
317     if (loadParameters.options.mode != FetchOptions::Mode::Navigate) {
318         ASSERT(loadParameters.sourceOrigin);
319         if (!loadParameters.sourceOrigin) {
320             scheduleInternallyFailedLoad(resourceLoader);
321             return;
322         }
323     }
324
325     loadParameters.shouldRestrictHTTPResponseAccess = RuntimeEnabledFeatures::sharedFeatures().restrictedHTTPResponseAccess();
326
327     loadParameters.isMainFrameNavigation = resourceLoader.frame() && resourceLoader.frame()->isMainFrame() && resourceLoader.options().mode == FetchOptions::Mode::Navigate;
328
329     loadParameters.shouldEnableFromOriginResponseHeader = RuntimeEnabledFeatures::sharedFeatures().fromOriginResponseHeaderEnabled() && !loadParameters.isMainFrameNavigation;
330     if (loadParameters.shouldEnableFromOriginResponseHeader) {
331         Vector<RefPtr<WebCore::SecurityOrigin>> frameAncestorOrigins;
332         for (auto* frame = resourceLoader.frame(); frame; frame = frame->tree().parent()) {
333             if (frame->document())
334                 frameAncestorOrigins.append(makeRefPtr(frame->document()->securityOrigin()));
335             if (frame->isMainFrame())
336                 break;
337         }
338         loadParameters.frameAncestorOrigins = WTFMove(frameAncestorOrigins);
339     }
340
341     ASSERT((loadParameters.webPageID && loadParameters.webFrameID) || loadParameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials);
342
343     RELEASE_LOG_IF_ALLOWED(resourceLoader, "scheduleLoad: Resource is being scheduled with the NetworkProcess (frame = %p, priority = %d, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), static_cast<int>(resourceLoader.request().priority()), loadParameters.webPageID, loadParameters.webFrameID, loadParameters.identifier);
344     if (!WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::ScheduleResourceLoad(loadParameters), 0)) {
345         RELEASE_LOG_ERROR_IF_ALLOWED(resourceLoader, "scheduleLoad: Unable to schedule resource with the NetworkProcess (frame = %p, priority = %d, pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", resourceLoader.frame(), static_cast<int>(resourceLoader.request().priority()), loadParameters.webPageID, loadParameters.webFrameID, loadParameters.identifier);
346         // We probably failed to schedule this load with the NetworkProcess because it had crashed.
347         // This load will never succeed so we will schedule it to fail asynchronously.
348         scheduleInternallyFailedLoad(resourceLoader);
349         return;
350     }
351
352     m_webResourceLoaders.set(identifier, WebResourceLoader::create(resourceLoader, trackingParameters));
353 }
354
355 void WebLoaderStrategy::scheduleInternallyFailedLoad(WebCore::ResourceLoader& resourceLoader)
356 {
357     m_internallyFailedResourceLoaders.add(&resourceLoader);
358     m_internallyFailedLoadTimer.startOneShot(0_s);
359 }
360
361 void WebLoaderStrategy::internallyFailedLoadTimerFired()
362 {
363     for (auto& resourceLoader : copyToVector(m_internallyFailedResourceLoaders))
364         resourceLoader->didFail(internalError(resourceLoader->url()));
365 }
366
367 void WebLoaderStrategy::startLocalLoad(WebCore::ResourceLoader& resourceLoader)
368 {
369     resourceLoader.start();
370     m_webResourceLoaders.set(resourceLoader.identifier(), WebResourceLoader::create(resourceLoader, { }));
371 }
372
373 void WebLoaderStrategy::addURLSchemeTaskProxy(WebURLSchemeTaskProxy& task)
374 {
375     auto result = m_urlSchemeTasks.add(task.identifier(), &task);
376     ASSERT_UNUSED(result, result.isNewEntry);
377 }
378
379 void WebLoaderStrategy::removeURLSchemeTaskProxy(WebURLSchemeTaskProxy& task)
380 {
381     m_urlSchemeTasks.remove(task.identifier());
382 }
383
384 void WebLoaderStrategy::remove(ResourceLoader* resourceLoader)
385 {
386     ASSERT(resourceLoader);
387     LOG(NetworkScheduling, "(WebProcess) WebLoaderStrategy::remove, url '%s'", resourceLoader->url().string().utf8().data());
388
389     if (auto task = m_urlSchemeTasks.take(resourceLoader->identifier())) {
390         ASSERT(!m_internallyFailedResourceLoaders.contains(resourceLoader));
391         task->stopLoading();
392         return;
393     }
394
395     if (m_internallyFailedResourceLoaders.contains(resourceLoader)) {
396         m_internallyFailedResourceLoaders.remove(resourceLoader);
397         return;
398     }
399     
400     ResourceLoadIdentifier identifier = resourceLoader->identifier();
401     if (!identifier) {
402         LOG_ERROR("WebLoaderStrategy removing a ResourceLoader that has no identifier.");
403         return;
404     }
405
406 #if ENABLE(SERVICE_WORKER)
407     if (WebServiceWorkerProvider::singleton().cancelFetch(identifier))
408         return;
409 #endif
410
411     RefPtr<WebResourceLoader> loader = m_webResourceLoaders.take(identifier);
412     // Loader may not be registered if we created it, but haven't scheduled yet (a bundle client can decide to cancel such request via willSendRequest).
413     if (!loader)
414         return;
415
416     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::RemoveLoadIdentifier(identifier), 0);
417
418     // It's possible that this WebResourceLoader might be just about to message back to the NetworkProcess (e.g. ContinueWillSendRequest)
419     // but there's no point in doing so anymore.
420     loader->detachFromCoreLoader();
421 }
422
423 void WebLoaderStrategy::setDefersLoading(ResourceLoader* resourceLoader, bool defers)
424 {
425     ResourceLoadIdentifier identifier = resourceLoader->identifier();
426     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::SetDefersLoading(identifier, defers), 0);
427 }
428
429 void WebLoaderStrategy::crossOriginRedirectReceived(ResourceLoader*, const URL&)
430 {
431     // We handle cross origin redirects entirely within the NetworkProcess.
432     // We override this call in the WebProcess to make it a no-op.
433 }
434
435 void WebLoaderStrategy::servePendingRequests(ResourceLoadPriority)
436 {
437     // This overrides the base class version.
438     // We don't need to do anything as this is handled by the network process.
439 }
440
441 void WebLoaderStrategy::suspendPendingRequests()
442 {
443     // Network process does keep requests in pending state.
444 }
445
446 void WebLoaderStrategy::resumePendingRequests()
447 {
448     // Network process does keep requests in pending state.
449 }
450
451 void WebLoaderStrategy::networkProcessCrashed()
452 {
453     RELEASE_LOG_ERROR(Network, "WebLoaderStrategy::networkProcessCrashed: failing all pending resource loaders");
454
455     for (auto& loader : m_webResourceLoaders)
456         scheduleInternallyFailedLoad(*loader.value->resourceLoader());
457
458     m_webResourceLoaders.clear();
459
460     auto pingLoadCompletionHandlers = WTFMove(m_pingLoadCompletionHandlers);
461     for (auto& pingLoadCompletionHandler : pingLoadCompletionHandlers.values())
462         pingLoadCompletionHandler(internalError(URL()), { });
463
464     auto preconnectCompletionHandlers = WTFMove(m_preconnectCompletionHandlers);
465     for (auto& preconnectCompletionHandler : preconnectCompletionHandlers.values())
466         preconnectCompletionHandler(internalError(URL()));
467 }
468
469 static bool shouldClearReferrerOnHTTPSToHTTPRedirect(Frame* frame)
470 {
471     if (frame) {
472         if (auto* document = frame->document())
473             return document->referrerPolicy() == ReferrerPolicy::NoReferrerWhenDowngrade;
474     }
475     return true;
476 }
477
478 void WebLoaderStrategy::loadResourceSynchronously(FrameLoader& frameLoader, unsigned long resourceLoadIdentifier, const ResourceRequest& request, ClientCredentialPolicy clientCredentialPolicy,  const FetchOptions& options, const HTTPHeaderMap& originalRequestHeaders, ResourceError& error, ResourceResponse& response, Vector<char>& data)
479 {
480     auto* document = frameLoader.frame().document();
481     if (!document) {
482         error = internalError(request.url());
483         return;
484     }
485
486     WebFrameLoaderClient* webFrameLoaderClient = toWebFrameLoaderClient(frameLoader.client());
487     WebFrame* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
488     WebPage* webPage = webFrame ? webFrame->page() : nullptr;
489
490     NetworkResourceLoadParameters loadParameters;
491     loadParameters.identifier = resourceLoadIdentifier;
492     loadParameters.webPageID = webPage ? webPage->pageID() : 0;
493     loadParameters.webFrameID = webFrame ? webFrame->frameID() : 0;
494     loadParameters.sessionID = webPage ? webPage->sessionID() : PAL::SessionID::defaultSessionID();
495     loadParameters.request = request;
496     loadParameters.contentSniffingPolicy = SniffContent;
497     loadParameters.contentEncodingSniffingPolicy = ContentEncodingSniffingPolicy::Sniff;
498     loadParameters.storedCredentialsPolicy = options.credentials == FetchOptions::Credentials::Omit ? StoredCredentialsPolicy::DoNotUse : StoredCredentialsPolicy::Use;
499     loadParameters.clientCredentialPolicy = clientCredentialPolicy;
500     loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect = shouldClearReferrerOnHTTPSToHTTPRedirect(webFrame ? webFrame->coreFrame() : nullptr);
501     loadParameters.shouldRestrictHTTPResponseAccess = RuntimeEnabledFeatures::sharedFeatures().restrictedHTTPResponseAccess();
502
503     loadParameters.options = options;
504     loadParameters.sourceOrigin = &document->securityOrigin();
505     if (!document->shouldBypassMainWorldContentSecurityPolicy()) {
506         if (auto* contentSecurityPolicy = document->contentSecurityPolicy())
507             loadParameters.cspResponseHeaders = contentSecurityPolicy->responseHeaders();
508     }
509     loadParameters.originalRequestHeaders = originalRequestHeaders;
510
511     data.shrink(0);
512
513     HangDetectionDisabler hangDetectionDisabler;
514
515     if (!WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad(loadParameters), Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::Reply(error, response, data), 0, Seconds::infinity(), IPC::SendSyncOption::DoNotProcessIncomingMessagesWhenWaitingForSyncReply)) {
516         RELEASE_LOG_ERROR_IF_ALLOWED(loadParameters.sessionID, "loadResourceSynchronously: failed sending synchronous network process message (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", loadParameters.webPageID, loadParameters.webFrameID, loadParameters.identifier);
517         if (auto* page = webPage->corePage())
518             page->diagnosticLoggingClient().logDiagnosticMessage(WebCore::DiagnosticLoggingKeys::internalErrorKey(), WebCore::DiagnosticLoggingKeys::synchronousMessageFailedKey(), WebCore::ShouldSample::No);
519         response = ResourceResponse();
520         error = internalError(request.url());
521     }
522 }
523
524 static uint64_t generateLoadIdentifier()
525 {
526     static uint64_t identifier = 0;
527     return ++identifier;
528 }
529
530 void WebLoaderStrategy::startPingLoad(Frame& frame, ResourceRequest& request, const HTTPHeaderMap& originalRequestHeaders, const FetchOptions& options, PingLoadCompletionHandler&& completionHandler)
531 {
532     auto* document = frame.document();
533     if (!document) {
534         if (completionHandler)
535             completionHandler(internalError(request.url()), { });
536         return;
537     }
538
539     NetworkResourceLoadParameters loadParameters;
540     loadParameters.identifier = generateLoadIdentifier();
541     loadParameters.request = request;
542     loadParameters.sourceOrigin = &document->securityOrigin();
543     loadParameters.sessionID = frame.page() ? frame.page()->sessionID() : PAL::SessionID::defaultSessionID();
544     loadParameters.storedCredentialsPolicy = options.credentials == FetchOptions::Credentials::Omit ? StoredCredentialsPolicy::DoNotUse : StoredCredentialsPolicy::Use;
545     loadParameters.options = options;
546     loadParameters.originalRequestHeaders = originalRequestHeaders;
547     loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect = shouldClearReferrerOnHTTPSToHTTPRedirect(&frame);
548     loadParameters.shouldRestrictHTTPResponseAccess = RuntimeEnabledFeatures::sharedFeatures().restrictedHTTPResponseAccess();
549     if (!document->shouldBypassMainWorldContentSecurityPolicy()) {
550         if (auto * contentSecurityPolicy = document->contentSecurityPolicy())
551             loadParameters.cspResponseHeaders = contentSecurityPolicy->responseHeaders();
552     }
553
554 #if ENABLE(CONTENT_EXTENSIONS)
555     loadParameters.mainDocumentURL = document->topDocument().url();
556     // FIXME: Instead of passing userContentControllerIdentifier, we should just pass webPageId to NetworkProcess.
557     WebFrameLoaderClient* webFrameLoaderClient = toWebFrameLoaderClient(frame.loader().client());
558     WebFrame* webFrame = webFrameLoaderClient ? webFrameLoaderClient->webFrame() : nullptr;
559     WebPage* webPage = webFrame ? webFrame->page() : nullptr;
560     if (webPage)
561         loadParameters.userContentControllerIdentifier = webPage->userContentControllerIdentifier();
562 #endif
563
564     if (completionHandler)
565         m_pingLoadCompletionHandlers.add(loadParameters.identifier, WTFMove(completionHandler));
566
567     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::LoadPing { loadParameters }, 0);
568 }
569
570 void WebLoaderStrategy::didFinishPingLoad(uint64_t pingLoadIdentifier, ResourceError&& error, ResourceResponse&& response)
571 {
572     if (auto completionHandler = m_pingLoadCompletionHandlers.take(pingLoadIdentifier))
573         completionHandler(WTFMove(error), WTFMove(response));
574 }
575
576 void WebLoaderStrategy::preconnectTo(FrameLoader& frameLoader, const WebCore::URL& url, StoredCredentialsPolicy storedCredentialsPolicy, PreconnectCompletionHandler&& completionHandler)
577 {
578     uint64_t preconnectionIdentifier = generateLoadIdentifier();
579     auto addResult = m_preconnectCompletionHandlers.add(preconnectionIdentifier, WTFMove(completionHandler));
580     ASSERT_UNUSED(addResult, addResult.isNewEntry);
581
582     auto* webFrameLoaderClient = toWebFrameLoaderClient(frameLoader.client());
583     if (!webFrameLoaderClient) {
584         completionHandler(internalError(url));
585         return;
586     }
587     auto* webFrame = webFrameLoaderClient->webFrame();
588     if (!webFrame) {
589         completionHandler(internalError(url));
590         return;
591     }
592     auto* webPage = webFrame->page();
593     if (!webPage) {
594         completionHandler(internalError(url));
595         return;
596     }
597
598     NetworkResourceLoadParameters parameters;
599     parameters.request = ResourceRequest { url };
600     parameters.webPageID = webPage ? webPage->pageID() : 0;
601     parameters.webFrameID = webFrame ? webFrame->frameID() : 0;
602     parameters.sessionID = webPage ? webPage->sessionID() : PAL::SessionID::defaultSessionID();
603     parameters.storedCredentialsPolicy = storedCredentialsPolicy;
604     parameters.shouldPreconnectOnly = PreconnectOnly::Yes;
605     parameters.shouldRestrictHTTPResponseAccess = RuntimeEnabledFeatures::sharedFeatures().restrictedHTTPResponseAccess();
606     // FIXME: Use the proper destination once all fetch options are passed.
607     parameters.options.destination = FetchOptions::Destination::EmptyString;
608
609     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::PreconnectTo(preconnectionIdentifier, WTFMove(parameters)), 0);
610 }
611
612 void WebLoaderStrategy::didFinishPreconnection(uint64_t preconnectionIdentifier, ResourceError&& error)
613 {
614     if (auto completionHandler = m_preconnectCompletionHandlers.take(preconnectionIdentifier))
615         completionHandler(WTFMove(error));
616 }
617
618 void WebLoaderStrategy::storeDerivedDataToCache(const SHA1::Digest& bodyHash, const String& type, const String& partition, WebCore::SharedBuffer& data)
619 {
620     NetworkCache::DataKey key { partition, type, bodyHash };
621     IPC::SharedBufferDataReference dataReference { &data };
622     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::StoreDerivedDataToCache(key, dataReference), 0);
623 }
624
625 bool WebLoaderStrategy::isOnLine() const
626 {
627     return m_isOnLine;
628 }
629
630 void WebLoaderStrategy::addOnlineStateChangeListener(Function<void(bool)>&& listener)
631 {
632     WebProcess::singleton().ensureNetworkProcessConnection();
633     m_onlineStateChangeListeners.append(WTFMove(listener));
634 }
635
636 void WebLoaderStrategy::setOnLineState(bool isOnLine)
637 {
638     if (m_isOnLine == isOnLine)
639         return;
640
641     m_isOnLine = isOnLine;
642     for (auto& listener : m_onlineStateChangeListeners)
643         listener(isOnLine);
644 }
645
646 void WebLoaderStrategy::setCaptureExtraNetworkLoadMetricsEnabled(bool enabled)
647 {
648     WebProcess::singleton().ensureNetworkProcessConnection().connection().send(Messages::NetworkConnectionToWebProcess::SetCaptureExtraNetworkLoadMetricsEnabled(enabled), 0);
649 }
650
651 ResourceResponse WebLoaderStrategy::responseFromResourceLoadIdentifier(uint64_t resourceLoadIdentifier)
652 {
653     ResourceResponse response;
654     WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::TakeNetworkLoadInformationResponse { resourceLoadIdentifier }, Messages::NetworkConnectionToWebProcess::TakeNetworkLoadInformationResponse::Reply { response }, 0, Seconds::infinity(), IPC::SendSyncOption::DoNotProcessIncomingMessagesWhenWaitingForSyncReply);
655     return response;
656 }
657
658 NetworkLoadMetrics WebLoaderStrategy::networkMetricsFromResourceLoadIdentifier(uint64_t resourceLoadIdentifier)
659 {
660     NetworkLoadMetrics networkMetrics;
661     WebProcess::singleton().ensureNetworkProcessConnection().connection().sendSync(Messages::NetworkConnectionToWebProcess::TakeNetworkLoadInformationMetrics { resourceLoadIdentifier }, Messages::NetworkConnectionToWebProcess::TakeNetworkLoadInformationMetrics::Reply { networkMetrics }, 0, Seconds::infinity(), IPC::SendSyncOption::DoNotProcessIncomingMessagesWhenWaitingForSyncReply);
662     return networkMetrics;
663 }
664
665 bool WebLoaderStrategy::isDoingLoadingSecurityChecks() const
666 {
667     return RuntimeEnabledFeatures::sharedFeatures().restrictedHTTPResponseAccess();
668 }
669
670 } // namespace WebKit