Move URL from WebCore to WTF
[WebKit-https.git] / Source / WebKitLegacy / WebCoreSupport / WebResourceLoadScheduler.cpp
1 /*
2     Copyright (C) 1998 Lars Knoll (knoll@mpi-hd.mpg.de)
3     Copyright (C) 2001 Dirk Mueller (mueller@kde.org)
4     Copyright (C) 2002 Waldo Bastian (bastian@kde.org)
5     Copyright (C) 2006 Samuel Weinig (sam.weinig@gmail.com)
6     Copyright (C) 2004-2008, 2015 Apple Inc. All rights reserved.
7     Copyright (C) 2010 Google Inc. All rights reserved.
8
9     This library is free software; you can redistribute it and/or
10     modify it under the terms of the GNU Library General Public
11     License as published by the Free Software Foundation; either
12     version 2 of the License, or (at your option) any later version.
13
14     This library is distributed in the hope that it will be useful,
15     but WITHOUT ANY WARRANTY; without even the implied warranty of
16     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
17     Library General Public License for more details.
18
19     You should have received a copy of the GNU Library General Public License
20     along with this library; see the file COPYING.LIB.  If not, write to
21     the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
22     Boston, MA 02110-1301, USA.
23  */
24
25 #include "WebResourceLoadScheduler.h"
26
27 #include "PingHandle.h"
28 #include <WebCore/Document.h>
29 #include <WebCore/DocumentLoader.h>
30 #include <WebCore/FetchOptions.h>
31 #include <WebCore/Frame.h>
32 #include <WebCore/FrameLoader.h>
33 #include <WebCore/NetscapePlugInStreamLoader.h>
34 #include <WebCore/NetworkStateNotifier.h>
35 #include <WebCore/PlatformStrategies.h>
36 #include <WebCore/ResourceRequest.h>
37 #include <WebCore/SubresourceLoader.h>
38 #include <wtf/MainThread.h>
39 #include <wtf/SetForScope.h>
40 #include <wtf/URL.h>
41 #include <wtf/text/CString.h>
42
43 #if PLATFORM(IOS_FAMILY)
44 #include <WebCore/RuntimeApplicationChecks.h>
45 #endif
46
47 // Match the parallel connection count used by the networking layer.
48 static unsigned maxRequestsInFlightPerHost;
49 #if !PLATFORM(IOS_FAMILY)
50 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 20;
51 #else
52 // Limiting this seems to regress performance in some local cases so let's just make it large.
53 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 10000;
54 #endif
55
56 using namespace WebCore;
57
58 WebResourceLoadScheduler& webResourceLoadScheduler()
59 {
60     return static_cast<WebResourceLoadScheduler&>(*platformStrategies()->loaderStrategy());
61 }
62
63 WebResourceLoadScheduler::HostInformation* WebResourceLoadScheduler::hostForURL(const URL& url, CreateHostPolicy createHostPolicy)
64 {
65     if (!url.protocolIsInHTTPFamily())
66         return m_nonHTTPProtocolHost;
67
68     m_hosts.checkConsistency();
69     String hostName = url.host().toString();
70     HostInformation* host = m_hosts.get(hostName);
71     if (!host && createHostPolicy == CreateIfNotFound) {
72         host = new HostInformation(hostName, maxRequestsInFlightPerHost);
73         m_hosts.add(hostName, host);
74     }
75     return host;
76 }
77
78 WebResourceLoadScheduler::WebResourceLoadScheduler()
79     : m_nonHTTPProtocolHost(new HostInformation(String(), maxRequestsInFlightForNonHTTPProtocols))
80     , m_requestTimer(*this, &WebResourceLoadScheduler::requestTimerFired)
81     , m_suspendPendingRequestsCount(0)
82     , m_isSerialLoadingEnabled(false)
83 {
84     maxRequestsInFlightPerHost = initializeMaximumHTTPConnectionCountPerHost();
85 }
86
87 WebResourceLoadScheduler::~WebResourceLoadScheduler()
88 {
89 }
90
91 void WebResourceLoadScheduler::loadResource(Frame& frame, CachedResource& resource, ResourceRequest&& request, const ResourceLoaderOptions& options, CompletionHandler<void(RefPtr<WebCore::SubresourceLoader>&&)>&& completionHandler)
92 {
93     SubresourceLoader::create(frame, resource, WTFMove(request), options, [this, completionHandler = WTFMove(completionHandler)] (RefPtr<WebCore::SubresourceLoader>&& loader) mutable {
94         if (loader)
95             scheduleLoad(loader.get());
96 #if PLATFORM(IOS_FAMILY)
97         // Since we defer loader initialization until scheduling on iOS, the frame
98         // load delegate that would be called in SubresourceLoader::create() on
99         // other ports might be called in scheduleLoad() instead. Our contract to
100         // callers of this method is that a null loader is returned if the load was
101         // cancelled by a frame load delegate.
102         if (!loader || loader->reachedTerminalState())
103             return completionHandler(nullptr);
104 #endif
105         completionHandler(WTFMove(loader));
106     });
107 }
108
109 void WebResourceLoadScheduler::loadResourceSynchronously(FrameLoader& frameLoader, unsigned long, const ResourceRequest& request, ClientCredentialPolicy, const FetchOptions& options, const HTTPHeaderMap&, ResourceError& error, ResourceResponse& response, Vector<char>& data)
110 {
111     ResourceHandle::loadResourceSynchronously(frameLoader.networkingContext(), request, options.credentials == FetchOptions::Credentials::Omit ? StoredCredentialsPolicy::DoNotUse : StoredCredentialsPolicy::Use, error, response, data);
112 }
113
114 void WebResourceLoadScheduler::pageLoadCompleted(uint64_t /*webPageID*/)
115 {
116 }
117
118 void WebResourceLoadScheduler::schedulePluginStreamLoad(Frame& frame, NetscapePlugInStreamLoaderClient& client, ResourceRequest&& request, CompletionHandler<void(RefPtr<WebCore::NetscapePlugInStreamLoader>&&)>&& completionHandler)
119 {
120     NetscapePlugInStreamLoader::create(frame, client, WTFMove(request), [this, completionHandler = WTFMove(completionHandler)] (RefPtr<WebCore::NetscapePlugInStreamLoader>&& loader) mutable {
121         if (loader)
122             scheduleLoad(loader.get());
123         completionHandler(WTFMove(loader));
124     });
125 }
126
127 void WebResourceLoadScheduler::scheduleLoad(ResourceLoader* resourceLoader)
128 {
129     ASSERT(resourceLoader);
130
131 #if PLATFORM(IOS_FAMILY)
132     // If there's a web archive resource for this URL, we don't need to schedule the load since it will never touch the network.
133     if (!isSuspendingPendingRequests() && resourceLoader->documentLoader()->archiveResourceForURL(resourceLoader->iOSOriginalRequest().url())) {
134         resourceLoader->startLoading();
135         return;
136     }
137 #else
138     if (resourceLoader->documentLoader()->archiveResourceForURL(resourceLoader->request().url())) {
139         resourceLoader->start();
140         return;
141     }
142 #endif
143
144 #if PLATFORM(IOS_FAMILY)
145     HostInformation* host = hostForURL(resourceLoader->iOSOriginalRequest().url(), CreateIfNotFound);
146 #else
147     HostInformation* host = hostForURL(resourceLoader->url(), CreateIfNotFound);
148 #endif
149
150     ResourceLoadPriority priority = resourceLoader->request().priority();
151
152     bool hadRequests = host->hasRequests();
153     host->schedule(resourceLoader, priority);
154
155 #if PLATFORM(COCOA)
156     if (ResourceRequest::resourcePrioritiesEnabled() && !isSuspendingPendingRequests()) {
157         // Serve all requests at once to keep the pipeline full at the network layer.
158         // FIXME: Does this code do anything useful, given that we also set maxRequestsInFlightPerHost to effectively unlimited on these platforms?
159         servePendingRequests(host, ResourceLoadPriority::VeryLow);
160         return;
161     }
162 #endif
163
164 #if PLATFORM(IOS_FAMILY)
165     if ((priority > ResourceLoadPriority::Low || !resourceLoader->iOSOriginalRequest().url().protocolIsInHTTPFamily() || (priority == ResourceLoadPriority::Low && !hadRequests)) && !isSuspendingPendingRequests()) {
166         // Try to request important resources immediately.
167         servePendingRequests(host, priority);
168         return;
169     }
170 #else
171     if (priority > ResourceLoadPriority::Low || !resourceLoader->url().protocolIsInHTTPFamily() || (priority == ResourceLoadPriority::Low && !hadRequests)) {
172         // Try to request important resources immediately.
173         servePendingRequests(host, priority);
174         return;
175     }
176 #endif
177
178     // Handle asynchronously so early low priority requests don't
179     // get scheduled before later high priority ones.
180     scheduleServePendingRequests();
181 }
182
183 void WebResourceLoadScheduler::remove(ResourceLoader* resourceLoader)
184 {
185     ASSERT(resourceLoader);
186
187     HostInformation* host = hostForURL(resourceLoader->url());
188     if (host)
189         host->remove(resourceLoader);
190 #if PLATFORM(IOS_FAMILY)
191     // ResourceLoader::url() doesn't start returning the correct value until the load starts. If we get canceled before that, we need to look for originalRequest url instead.
192     // FIXME: ResourceLoader::url() should be made to return a sensible value at all times.
193     if (!resourceLoader->iOSOriginalRequest().isNull()) {
194         HostInformation* originalHost = hostForURL(resourceLoader->iOSOriginalRequest().url());
195         if (originalHost && originalHost != host)
196             originalHost->remove(resourceLoader);
197     }
198 #endif
199     scheduleServePendingRequests();
200 }
201
202 void WebResourceLoadScheduler::setDefersLoading(ResourceLoader& loader, bool defers)
203 {
204     if (!defers && !loader.deferredRequest().isNull()) {
205         loader.setRequest(loader.takeDeferredRequest());
206         loader.start();
207     }
208 }
209
210 void WebResourceLoadScheduler::crossOriginRedirectReceived(ResourceLoader* resourceLoader, const URL& redirectURL)
211 {
212     HostInformation* oldHost = hostForURL(resourceLoader->url());
213     ASSERT(oldHost);
214     if (!oldHost)
215         return;
216
217     HostInformation* newHost = hostForURL(redirectURL, CreateIfNotFound);
218
219     if (oldHost->name() == newHost->name())
220         return;
221
222     newHost->addLoadInProgress(resourceLoader);
223     oldHost->remove(resourceLoader);
224 }
225
226 void WebResourceLoadScheduler::servePendingRequests(ResourceLoadPriority minimumPriority)
227 {
228     if (isSuspendingPendingRequests())
229         return;
230
231     m_requestTimer.stop();
232     
233     servePendingRequests(m_nonHTTPProtocolHost, minimumPriority);
234
235     for (auto* host : copyToVector(m_hosts.values())) {
236         if (host->hasRequests())
237             servePendingRequests(host, minimumPriority);
238         else
239             delete m_hosts.take(host->name());
240     }
241 }
242
243 void WebResourceLoadScheduler::servePendingRequests(HostInformation* host, ResourceLoadPriority minimumPriority)
244 {
245     auto priority = ResourceLoadPriority::Highest;
246     while (true) {
247         auto& requestsPending = host->requestsPending(priority);
248         while (!requestsPending.isEmpty()) {
249             RefPtr<ResourceLoader> resourceLoader = requestsPending.first();
250
251             // For named hosts - which are only http(s) hosts - we should always enforce the connection limit.
252             // For non-named hosts - everything but http(s) - we should only enforce the limit if the document isn't done parsing 
253             // and we don't know all stylesheets yet.
254             Document* document = resourceLoader->frameLoader() ? resourceLoader->frameLoader()->frame().document() : 0;
255             bool shouldLimitRequests = !host->name().isNull() || (document && (document->parsing() || !document->haveStylesheetsLoaded()));
256             if (shouldLimitRequests && host->limitRequests(priority))
257                 return;
258
259             requestsPending.removeFirst();
260             host->addLoadInProgress(resourceLoader.get());
261 #if PLATFORM(IOS_FAMILY)
262             if (!IOSApplication::isWebProcess()) {
263                 resourceLoader->startLoading();
264                 return;
265             }
266 #endif
267             resourceLoader->start();
268         }
269         if (priority == minimumPriority)
270             return;
271         --priority;
272     }
273 }
274
275 void WebResourceLoadScheduler::suspendPendingRequests()
276 {
277     ++m_suspendPendingRequestsCount;
278 }
279
280 void WebResourceLoadScheduler::resumePendingRequests()
281 {
282     ASSERT(m_suspendPendingRequestsCount);
283     --m_suspendPendingRequestsCount;
284     if (m_suspendPendingRequestsCount)
285         return;
286     if (!m_hosts.isEmpty() || m_nonHTTPProtocolHost->hasRequests())
287         scheduleServePendingRequests();
288 }
289     
290 void WebResourceLoadScheduler::scheduleServePendingRequests()
291 {
292     if (!m_requestTimer.isActive())
293         m_requestTimer.startOneShot(0_s);
294 }
295
296 void WebResourceLoadScheduler::requestTimerFired()
297 {
298     servePendingRequests();
299 }
300
301 WebResourceLoadScheduler::HostInformation::HostInformation(const String& name, unsigned maxRequestsInFlight)
302     : m_name(name)
303     , m_maxRequestsInFlight(maxRequestsInFlight)
304 {
305 }
306
307 WebResourceLoadScheduler::HostInformation::~HostInformation()
308 {
309     ASSERT(!hasRequests());
310 }
311
312 unsigned WebResourceLoadScheduler::HostInformation::priorityToIndex(ResourceLoadPriority priority)
313 {
314     switch (priority) {
315     case ResourceLoadPriority::VeryLow:
316         return 0;
317     case ResourceLoadPriority::Low:
318         return 1;
319     case ResourceLoadPriority::Medium:
320         return 2;
321     case ResourceLoadPriority::High:
322         return 3;
323     case ResourceLoadPriority::VeryHigh:
324         return 4;
325     }
326     ASSERT_NOT_REACHED();
327     return 0;
328 }
329
330 void WebResourceLoadScheduler::HostInformation::schedule(ResourceLoader* resourceLoader, ResourceLoadPriority priority)
331 {
332     m_requestsPending[priorityToIndex(priority)].append(resourceLoader);
333 }
334     
335 void WebResourceLoadScheduler::HostInformation::addLoadInProgress(ResourceLoader* resourceLoader)
336 {
337     m_requestsLoading.add(resourceLoader);
338 }
339     
340 void WebResourceLoadScheduler::HostInformation::remove(ResourceLoader* resourceLoader)
341 {
342     if (m_requestsLoading.remove(resourceLoader))
343         return;
344     
345     for (auto& requestQueue : m_requestsPending) {
346         for (auto it = requestQueue.begin(), end = requestQueue.end(); it != end; ++it) {
347             if (*it == resourceLoader) {
348                 requestQueue.remove(it);
349                 return;
350             }
351         }
352     }
353 }
354
355 bool WebResourceLoadScheduler::HostInformation::hasRequests() const
356 {
357     if (!m_requestsLoading.isEmpty())
358         return true;
359     for (auto& requestQueue : m_requestsPending) {
360         if (!requestQueue.isEmpty())
361             return true;
362     }
363     return false;
364 }
365
366 bool WebResourceLoadScheduler::HostInformation::limitRequests(ResourceLoadPriority priority) const 
367 {
368     if (priority == ResourceLoadPriority::VeryLow && !m_requestsLoading.isEmpty())
369         return true;
370     return m_requestsLoading.size() >= (webResourceLoadScheduler().isSerialLoadingEnabled() ? 1 : m_maxRequestsInFlight);
371 }
372
373 void WebResourceLoadScheduler::startPingLoad(Frame& frame, ResourceRequest& request, const HTTPHeaderMap&, const FetchOptions& options, PingLoadCompletionHandler&& completionHandler)
374 {
375     // PingHandle manages its own lifetime, deleting itself when its purpose has been fulfilled.
376     new PingHandle(frame.loader().networkingContext(), request, options.credentials != FetchOptions::Credentials::Omit, options.redirect == FetchOptions::Redirect::Follow, WTFMove(completionHandler));
377 }
378
379 bool WebResourceLoadScheduler::isOnLine() const
380 {
381     return NetworkStateNotifier::singleton().onLine();
382 }
383
384 void WebResourceLoadScheduler::addOnlineStateChangeListener(WTF::Function<void(bool)>&& listener)
385 {
386     NetworkStateNotifier::singleton().addListener(WTFMove(listener));
387 }
388
389 void WebResourceLoadScheduler::preconnectTo(FrameLoader&, const URL&, StoredCredentialsPolicy, PreconnectCompletionHandler&&)
390 {
391 }
392