Use CompletionHandlers for ResourceHandleClient::didReceiveResponseAsync
[WebKit-https.git] / Source / WebKitLegacy / WebCoreSupport / WebResourceLoadScheduler.cpp
1 /*
2     Copyright (C) 1998 Lars Knoll (knoll@mpi-hd.mpg.de)
3     Copyright (C) 2001 Dirk Mueller (mueller@kde.org)
4     Copyright (C) 2002 Waldo Bastian (bastian@kde.org)
5     Copyright (C) 2006 Samuel Weinig (sam.weinig@gmail.com)
6     Copyright (C) 2004-2008, 2015 Apple Inc. All rights reserved.
7     Copyright (C) 2010 Google Inc. All rights reserved.
8
9     This library is free software; you can redistribute it and/or
10     modify it under the terms of the GNU Library General Public
11     License as published by the Free Software Foundation; either
12     version 2 of the License, or (at your option) any later version.
13
14     This library is distributed in the hope that it will be useful,
15     but WITHOUT ANY WARRANTY; without even the implied warranty of
16     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
17     Library General Public License for more details.
18
19     You should have received a copy of the GNU Library General Public License
20     along with this library; see the file COPYING.LIB.  If not, write to
21     the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
22     Boston, MA 02110-1301, USA.
23  */
24
25 #include "WebResourceLoadScheduler.h"
26
27 #include <WebCore/Document.h>
28 #include <WebCore/DocumentLoader.h>
29 #include <WebCore/FetchOptions.h>
30 #include <WebCore/Frame.h>
31 #include <WebCore/FrameLoader.h>
32 #include <WebCore/NetscapePlugInStreamLoader.h>
33 #include <WebCore/PingHandle.h>
34 #include <WebCore/PlatformStrategies.h>
35 #include <WebCore/ResourceRequest.h>
36 #include <WebCore/SubresourceLoader.h>
37 #include <WebCore/URL.h>
38 #include <wtf/MainThread.h>
39 #include <wtf/SetForScope.h>
40 #include <wtf/text/CString.h>
41
42 #if PLATFORM(IOS)
43 #include <WebCore/RuntimeApplicationChecks.h>
44 #endif
45
46 // Match the parallel connection count used by the networking layer.
47 static unsigned maxRequestsInFlightPerHost;
48 #if !PLATFORM(IOS)
49 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 20;
50 #else
51 // Limiting this seems to regress performance in some local cases so let's just make it large.
52 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 10000;
53 #endif
54
55 using namespace WebCore;
56
57 WebResourceLoadScheduler& webResourceLoadScheduler()
58 {
59     return static_cast<WebResourceLoadScheduler&>(*platformStrategies()->loaderStrategy());
60 }
61
62 WebResourceLoadScheduler::HostInformation* WebResourceLoadScheduler::hostForURL(const URL& url, CreateHostPolicy createHostPolicy)
63 {
64     if (!url.protocolIsInHTTPFamily())
65         return m_nonHTTPProtocolHost;
66
67     m_hosts.checkConsistency();
68     String hostName = url.host();
69     HostInformation* host = m_hosts.get(hostName);
70     if (!host && createHostPolicy == CreateIfNotFound) {
71         host = new HostInformation(hostName, maxRequestsInFlightPerHost);
72         m_hosts.add(hostName, host);
73     }
74     return host;
75 }
76
77 WebResourceLoadScheduler::WebResourceLoadScheduler()
78     : m_nonHTTPProtocolHost(new HostInformation(String(), maxRequestsInFlightForNonHTTPProtocols))
79     , m_requestTimer(*this, &WebResourceLoadScheduler::requestTimerFired)
80     , m_suspendPendingRequestsCount(0)
81     , m_isSerialLoadingEnabled(false)
82 {
83     maxRequestsInFlightPerHost = initializeMaximumHTTPConnectionCountPerHost();
84 }
85
86 WebResourceLoadScheduler::~WebResourceLoadScheduler()
87 {
88 }
89
90 void WebResourceLoadScheduler::loadResource(Frame& frame, CachedResource& resource, ResourceRequest&& request, const ResourceLoaderOptions& options, CompletionHandler<void(RefPtr<WebCore::SubresourceLoader>&&)>&& completionHandler)
91 {
92     SubresourceLoader::create(frame, resource, WTFMove(request), options, [this, completionHandler = WTFMove(completionHandler)] (RefPtr<WebCore::SubresourceLoader>&& loader) mutable {
93         if (loader)
94             scheduleLoad(loader.get());
95 #if PLATFORM(IOS)
96         // Since we defer loader initialization until scheduling on iOS, the frame
97         // load delegate that would be called in SubresourceLoader::create() on
98         // other ports might be called in scheduleLoad() instead. Our contract to
99         // callers of this method is that a null loader is returned if the load was
100         // cancelled by a frame load delegate.
101         if (!loader || loader->reachedTerminalState())
102             return completionHandler(nullptr);
103 #endif
104         completionHandler(WTFMove(loader));
105     });
106 }
107
108 void WebResourceLoadScheduler::loadResourceSynchronously(NetworkingContext* context, unsigned long, const ResourceRequest& request, StoredCredentialsPolicy storedCredentialsPolicy, ClientCredentialPolicy, ResourceError& error, ResourceResponse& response, Vector<char>& data)
109 {
110     ResourceHandle::loadResourceSynchronously(context, request, storedCredentialsPolicy, error, response, data);
111 }
112
113 void WebResourceLoadScheduler::schedulePluginStreamLoad(Frame& frame, NetscapePlugInStreamLoaderClient& client, ResourceRequest&& request, CompletionHandler<void(RefPtr<WebCore::NetscapePlugInStreamLoader>&&)>&& completionHandler)
114 {
115     NetscapePlugInStreamLoader::create(frame, client, WTFMove(request), [this, completionHandler = WTFMove(completionHandler)] (RefPtr<WebCore::NetscapePlugInStreamLoader>&& loader) mutable {
116         if (loader)
117             scheduleLoad(loader.get());
118         completionHandler(WTFMove(loader));
119     });
120 }
121
122 void WebResourceLoadScheduler::scheduleLoad(ResourceLoader* resourceLoader)
123 {
124     ASSERT(resourceLoader);
125
126 #if PLATFORM(IOS)
127     // If there's a web archive resource for this URL, we don't need to schedule the load since it will never touch the network.
128     if (!isSuspendingPendingRequests() && resourceLoader->documentLoader()->archiveResourceForURL(resourceLoader->iOSOriginalRequest().url())) {
129         resourceLoader->startLoading();
130         return;
131     }
132 #else
133     if (resourceLoader->documentLoader()->archiveResourceForURL(resourceLoader->request().url())) {
134         resourceLoader->start();
135         return;
136     }
137 #endif
138
139 #if PLATFORM(IOS)
140     HostInformation* host = hostForURL(resourceLoader->iOSOriginalRequest().url(), CreateIfNotFound);
141 #else
142     HostInformation* host = hostForURL(resourceLoader->url(), CreateIfNotFound);
143 #endif
144
145     ResourceLoadPriority priority = resourceLoader->request().priority();
146
147     bool hadRequests = host->hasRequests();
148     host->schedule(resourceLoader, priority);
149
150 #if PLATFORM(COCOA)
151     if (ResourceRequest::resourcePrioritiesEnabled() && !isSuspendingPendingRequests()) {
152         // Serve all requests at once to keep the pipeline full at the network layer.
153         // FIXME: Does this code do anything useful, given that we also set maxRequestsInFlightPerHost to effectively unlimited on these platforms?
154         servePendingRequests(host, ResourceLoadPriority::VeryLow);
155         return;
156     }
157 #endif
158
159 #if PLATFORM(IOS)
160     if ((priority > ResourceLoadPriority::Low || !resourceLoader->iOSOriginalRequest().url().protocolIsInHTTPFamily() || (priority == ResourceLoadPriority::Low && !hadRequests)) && !isSuspendingPendingRequests()) {
161         // Try to request important resources immediately.
162         servePendingRequests(host, priority);
163         return;
164     }
165 #else
166     if (priority > ResourceLoadPriority::Low || !resourceLoader->url().protocolIsInHTTPFamily() || (priority == ResourceLoadPriority::Low && !hadRequests)) {
167         // Try to request important resources immediately.
168         servePendingRequests(host, priority);
169         return;
170     }
171 #endif
172
173     // Handle asynchronously so early low priority requests don't
174     // get scheduled before later high priority ones.
175     scheduleServePendingRequests();
176 }
177
178 void WebResourceLoadScheduler::remove(ResourceLoader* resourceLoader)
179 {
180     ASSERT(resourceLoader);
181
182     HostInformation* host = hostForURL(resourceLoader->url());
183     if (host)
184         host->remove(resourceLoader);
185 #if PLATFORM(IOS)
186     // ResourceLoader::url() doesn't start returning the correct value until the load starts. If we get canceled before that, we need to look for originalRequest url instead.
187     // FIXME: ResourceLoader::url() should be made to return a sensible value at all times.
188     if (!resourceLoader->iOSOriginalRequest().isNull()) {
189         HostInformation* originalHost = hostForURL(resourceLoader->iOSOriginalRequest().url());
190         if (originalHost && originalHost != host)
191             originalHost->remove(resourceLoader);
192     }
193 #endif
194     scheduleServePendingRequests();
195 }
196
197 void WebResourceLoadScheduler::setDefersLoading(ResourceLoader*, bool)
198 {
199 }
200
201 void WebResourceLoadScheduler::crossOriginRedirectReceived(ResourceLoader* resourceLoader, const URL& redirectURL)
202 {
203     HostInformation* oldHost = hostForURL(resourceLoader->url());
204     ASSERT(oldHost);
205     if (!oldHost)
206         return;
207
208     HostInformation* newHost = hostForURL(redirectURL, CreateIfNotFound);
209
210     if (oldHost->name() == newHost->name())
211         return;
212
213     newHost->addLoadInProgress(resourceLoader);
214     oldHost->remove(resourceLoader);
215 }
216
217 void WebResourceLoadScheduler::servePendingRequests(ResourceLoadPriority minimumPriority)
218 {
219     if (isSuspendingPendingRequests())
220         return;
221
222     m_requestTimer.stop();
223     
224     servePendingRequests(m_nonHTTPProtocolHost, minimumPriority);
225
226     for (auto* host : copyToVector(m_hosts.values())) {
227         if (host->hasRequests())
228             servePendingRequests(host, minimumPriority);
229         else
230             delete m_hosts.take(host->name());
231     }
232 }
233
234 void WebResourceLoadScheduler::servePendingRequests(HostInformation* host, ResourceLoadPriority minimumPriority)
235 {
236     auto priority = ResourceLoadPriority::Highest;
237     while (true) {
238         auto& requestsPending = host->requestsPending(priority);
239         while (!requestsPending.isEmpty()) {
240             RefPtr<ResourceLoader> resourceLoader = requestsPending.first();
241
242             // For named hosts - which are only http(s) hosts - we should always enforce the connection limit.
243             // For non-named hosts - everything but http(s) - we should only enforce the limit if the document isn't done parsing 
244             // and we don't know all stylesheets yet.
245             Document* document = resourceLoader->frameLoader() ? resourceLoader->frameLoader()->frame().document() : 0;
246             bool shouldLimitRequests = !host->name().isNull() || (document && (document->parsing() || !document->haveStylesheetsLoaded()));
247             if (shouldLimitRequests && host->limitRequests(priority))
248                 return;
249
250             requestsPending.removeFirst();
251             host->addLoadInProgress(resourceLoader.get());
252 #if PLATFORM(IOS)
253             if (!IOSApplication::isWebProcess()) {
254                 resourceLoader->startLoading();
255                 return;
256             }
257 #endif
258             resourceLoader->start();
259         }
260         if (priority == minimumPriority)
261             return;
262         --priority;
263     }
264 }
265
266 void WebResourceLoadScheduler::suspendPendingRequests()
267 {
268     ++m_suspendPendingRequestsCount;
269 }
270
271 void WebResourceLoadScheduler::resumePendingRequests()
272 {
273     ASSERT(m_suspendPendingRequestsCount);
274     --m_suspendPendingRequestsCount;
275     if (m_suspendPendingRequestsCount)
276         return;
277     if (!m_hosts.isEmpty() || m_nonHTTPProtocolHost->hasRequests())
278         scheduleServePendingRequests();
279 }
280     
281 void WebResourceLoadScheduler::scheduleServePendingRequests()
282 {
283     if (!m_requestTimer.isActive())
284         m_requestTimer.startOneShot(0_s);
285 }
286
287 void WebResourceLoadScheduler::requestTimerFired()
288 {
289     servePendingRequests();
290 }
291
292 WebResourceLoadScheduler::HostInformation::HostInformation(const String& name, unsigned maxRequestsInFlight)
293     : m_name(name)
294     , m_maxRequestsInFlight(maxRequestsInFlight)
295 {
296 }
297
298 WebResourceLoadScheduler::HostInformation::~HostInformation()
299 {
300     ASSERT(!hasRequests());
301 }
302
303 unsigned WebResourceLoadScheduler::HostInformation::priorityToIndex(ResourceLoadPriority priority)
304 {
305     switch (priority) {
306     case ResourceLoadPriority::VeryLow:
307         return 0;
308     case ResourceLoadPriority::Low:
309         return 1;
310     case ResourceLoadPriority::Medium:
311         return 2;
312     case ResourceLoadPriority::High:
313         return 3;
314     case ResourceLoadPriority::VeryHigh:
315         return 4;
316     }
317     ASSERT_NOT_REACHED();
318     return 0;
319 }
320
321 void WebResourceLoadScheduler::HostInformation::schedule(ResourceLoader* resourceLoader, ResourceLoadPriority priority)
322 {
323     m_requestsPending[priorityToIndex(priority)].append(resourceLoader);
324 }
325     
326 void WebResourceLoadScheduler::HostInformation::addLoadInProgress(ResourceLoader* resourceLoader)
327 {
328     m_requestsLoading.add(resourceLoader);
329 }
330     
331 void WebResourceLoadScheduler::HostInformation::remove(ResourceLoader* resourceLoader)
332 {
333     if (m_requestsLoading.remove(resourceLoader))
334         return;
335     
336     for (auto& requestQueue : m_requestsPending) {
337         for (auto it = requestQueue.begin(), end = requestQueue.end(); it != end; ++it) {
338             if (*it == resourceLoader) {
339                 requestQueue.remove(it);
340                 return;
341             }
342         }
343     }
344 }
345
346 bool WebResourceLoadScheduler::HostInformation::hasRequests() const
347 {
348     if (!m_requestsLoading.isEmpty())
349         return true;
350     for (auto& requestQueue : m_requestsPending) {
351         if (!requestQueue.isEmpty())
352             return true;
353     }
354     return false;
355 }
356
357 bool WebResourceLoadScheduler::HostInformation::limitRequests(ResourceLoadPriority priority) const 
358 {
359     if (priority == ResourceLoadPriority::VeryLow && !m_requestsLoading.isEmpty())
360         return true;
361     return m_requestsLoading.size() >= (webResourceLoadScheduler().isSerialLoadingEnabled() ? 1 : m_maxRequestsInFlight);
362 }
363
364 void WebResourceLoadScheduler::startPingLoad(Frame& frame, ResourceRequest& request, const HTTPHeaderMap&, const FetchOptions& options, PingLoadCompletionHandler&& completionHandler)
365 {
366     // PingHandle manages its own lifetime, deleting itself when its purpose has been fulfilled.
367     new PingHandle(frame.loader().networkingContext(), request, options.credentials != FetchOptions::Credentials::Omit, options.redirect == FetchOptions::Redirect::Follow, WTFMove(completionHandler));
368 }
369
370 void WebResourceLoadScheduler::preconnectTo(NetworkingContext&, const URL&, StoredCredentialsPolicy, PreconnectCompletionHandler&&)
371 {
372 }
373