Multiple Layout tests from web-platform-tests/beacon/ are timing out.
[WebKit-https.git] / Source / WebKit / NetworkProcess / NetworkLoad.cpp
1 /*
2  * Copyright (C) 2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkLoad.h"
28
29 #include "AuthenticationManager.h"
30 #include "DownloadProxyMessages.h"
31 #include "NetworkProcess.h"
32 #include "SessionTracker.h"
33 #include "WebCoreArgumentCoders.h"
34 #include "WebErrors.h"
35 #include <WebCore/NotImplemented.h>
36 #include <WebCore/ResourceHandle.h>
37 #include <WebCore/ResourceRequest.h>
38 #include <WebCore/SessionID.h>
39 #include <WebCore/SharedBuffer.h>
40 #include <wtf/MainThread.h>
41 #include <wtf/Seconds.h>
42
43 #if PLATFORM(COCOA)
44 #include "NetworkDataTaskCocoa.h"
45 #endif
46
47 #if ENABLE(NETWORK_CAPTURE)
48 #include "NetworkCaptureManager.h"
49 #endif
50
51 namespace WebKit {
52
53 using namespace WebCore;
54
55 #if USE(NETWORK_SESSION)
56
57 struct NetworkLoad::Throttle {
58     Throttle(NetworkLoad& load, Seconds delay, ResourceResponse&& response, ResponseCompletionHandler&& handler)
59         : timer(load, &NetworkLoad::throttleDelayCompleted)
60         , response(WTFMove(response))
61         , responseCompletionHandler(WTFMove(handler))
62     {
63         timer.startOneShot(delay);
64     }
65     Timer timer;
66     ResourceResponse response;
67     ResponseCompletionHandler responseCompletionHandler;
68 };
69
70 NetworkLoad::NetworkLoad(NetworkLoadClient& client, NetworkLoadParameters&& parameters, NetworkSession& networkSession)
71     : m_client(client)
72     , m_parameters(WTFMove(parameters))
73     , m_currentRequest(m_parameters.request)
74 {
75 #if ENABLE(NETWORK_CAPTURE)
76     switch (NetworkCapture::Manager::singleton().mode()) {
77     case NetworkCapture::Manager::RecordReplayMode::Record:
78         initializeForRecord(networkSession);
79         break;
80     case NetworkCapture::Manager::RecordReplayMode::Replay:
81         initializeForReplay(networkSession);
82         break;
83     case NetworkCapture::Manager::RecordReplayMode::Disabled:
84         initialize(networkSession);
85         break;
86     }
87 #else
88     initialize(networkSession);
89 #endif
90 }
91
92 #if ENABLE(NETWORK_CAPTURE)
93 void NetworkLoad::initializeForRecord(NetworkSession& networkSession)
94 {
95     m_recorder = std::make_unique<NetworkCapture::Recorder>();
96     m_task = NetworkDataTask::create(networkSession, *this, m_parameters);
97     if (!m_parameters.defersLoading) {
98         m_task->resume();
99         m_recorder->recordRequestSent(m_parameters.request);
100     }
101 }
102
103 void NetworkLoad::initializeForReplay(NetworkSession& networkSession)
104 {
105     m_replayer = std::make_unique<NetworkCapture::Replayer>();
106     m_task = m_replayer->replayResource(networkSession, *this, m_parameters);
107     if (!m_parameters.defersLoading)
108         m_task->resume();
109 }
110 #endif
111
112 void NetworkLoad::initialize(NetworkSession& networkSession)
113 {
114     m_task = NetworkDataTask::create(networkSession, *this, m_parameters);
115     if (!m_parameters.defersLoading)
116         m_task->resume();
117 }
118
119 #else
120
121 NetworkLoad::NetworkLoad(NetworkLoadClient& client, NetworkLoadParameters&& parameters)
122     : m_client(client)
123     , m_parameters(WTFMove(parameters))
124     , m_networkingContext(RemoteNetworkingContext::create(m_parameters.sessionID, m_parameters.shouldClearReferrerOnHTTPSToHTTPRedirect))
125     , m_currentRequest(m_parameters.request)
126 {
127     m_handle = ResourceHandle::create(m_networkingContext.get(), m_parameters.request, this, m_parameters.defersLoading, m_parameters.contentSniffingPolicy == SniffContent);
128 }
129
130 #endif
131
132 NetworkLoad::~NetworkLoad()
133 {
134     ASSERT(RunLoop::isMain());
135 #if USE(NETWORK_SESSION)
136     if (m_responseCompletionHandler)
137         m_responseCompletionHandler(PolicyIgnore);
138 #if USE(PROTECTION_SPACE_AUTH_CALLBACK)
139     if (m_challengeCompletionHandler)
140         m_challengeCompletionHandler(AuthenticationChallengeDisposition::Cancel, { });
141 #endif
142     if (m_task)
143         m_task->clearClient();
144 #else
145 #if USE(PROTECTION_SPACE_AUTH_CALLBACK)
146     if (m_handle && m_waitingForContinueCanAuthenticateAgainstProtectionSpace)
147         m_handle->continueCanAuthenticateAgainstProtectionSpace(false);
148 #endif
149     if (m_handle)
150         m_handle->clearClient();
151 #endif
152 }
153
154 void NetworkLoad::setDefersLoading(bool defers)
155 {
156 #if USE(NETWORK_SESSION)
157     if (m_task) {
158         if (defers)
159             m_task->suspend();
160         else {
161             m_task->resume();
162 #if ENABLE(NETWORK_CAPTURE)
163             if (m_recorder)
164                 m_recorder->recordRequestSent(m_parameters.request);
165 #endif
166         }
167     }
168 #else
169     if (m_handle)
170         m_handle->setDefersLoading(defers);
171 #endif
172 }
173
174 void NetworkLoad::cancel()
175 {
176 #if USE(NETWORK_SESSION)
177     if (m_task)
178         m_task->cancel();
179 #else
180     if (m_handle)
181         m_handle->cancel();
182 #endif
183 }
184
185 void NetworkLoad::continueWillSendRequest(WebCore::ResourceRequest&& newRequest)
186 {
187 #if PLATFORM(COCOA)
188     m_currentRequest.updateFromDelegatePreservingOldProperties(newRequest.nsURLRequest(DoNotUpdateHTTPBody));
189 #elif USE(SOUP)
190     // FIXME: Implement ResourceRequest::updateFromDelegatePreservingOldProperties. See https://bugs.webkit.org/show_bug.cgi?id=126127.
191     m_currentRequest.updateFromDelegatePreservingOldProperties(newRequest);
192 #endif
193
194 #if ENABLE(NETWORK_CAPTURE)
195     if (m_recorder)
196         m_recorder->recordRedirectSent(newRequest);
197 #endif
198
199 #if USE(NETWORK_SESSION)
200     auto redirectCompletionHandler = std::exchange(m_redirectCompletionHandler, nullptr);
201     ASSERT(redirectCompletionHandler);
202     if (m_currentRequest.isNull()) {
203         NetworkLoadMetrics emptyMetrics;
204         didCompleteWithError(cancelledError(m_currentRequest), emptyMetrics);
205         if (redirectCompletionHandler)
206             redirectCompletionHandler({ });
207         return;
208     }
209
210     if (redirectCompletionHandler)
211         redirectCompletionHandler(m_currentRequest);
212 #else
213     if (m_currentRequest.isNull()) {
214         if (m_handle)
215             m_handle->cancel();
216         didFail(m_handle.get(), cancelledError(m_currentRequest));
217     } else if (m_handle) {
218         auto currentRequestCopy = m_currentRequest;
219         m_handle->continueWillSendRequest(WTFMove(currentRequestCopy));
220     }
221 #endif
222 }
223
224 void NetworkLoad::continueDidReceiveResponse()
225 {
226 #if USE(NETWORK_SESSION)
227     if (m_responseCompletionHandler) {
228         auto responseCompletionHandler = std::exchange(m_responseCompletionHandler, nullptr);
229         responseCompletionHandler(PolicyUse);
230     }
231 #else
232     if (m_handle)
233         m_handle->continueDidReceiveResponse();
234 #endif
235 }
236
237 #if USE(NETWORK_SESSION)
238 bool NetworkLoad::shouldCaptureExtraNetworkLoadMetrics() const
239 {
240     return m_client.get().shouldCaptureExtraNetworkLoadMetrics();
241 }
242 #endif
243
244 NetworkLoadClient::ShouldContinueDidReceiveResponse NetworkLoad::sharedDidReceiveResponse(ResourceResponse&& response)
245 {
246     response.setSource(ResourceResponse::Source::Network);
247     if (m_parameters.needsCertificateInfo)
248         response.includeCertificateInfo();
249
250     return m_client.get().didReceiveResponse(WTFMove(response));
251 }
252
253 void NetworkLoad::sharedWillSendRedirectedRequest(ResourceRequest&& request, ResourceResponse&& redirectResponse)
254 {
255     // We only expect to get the willSendRequest callback from ResourceHandle as the result of a redirect.
256     ASSERT(!redirectResponse.isNull());
257     ASSERT(RunLoop::isMain());
258
259 #if ENABLE(NETWORK_CAPTURE)
260     if (m_recorder)
261         m_recorder->recordRedirectReceived(request, redirectResponse);
262 #endif
263
264     auto oldRequest = WTFMove(m_currentRequest);
265     m_currentRequest = request;
266     m_client.get().willSendRedirectedRequest(WTFMove(oldRequest), WTFMove(request), WTFMove(redirectResponse));
267 }
268
269 #if USE(NETWORK_SESSION)
270
271 void NetworkLoad::convertTaskToDownload(PendingDownload& pendingDownload, const ResourceRequest& updatedRequest, const ResourceResponse& response)
272 {
273     if (!m_task)
274         return;
275
276     m_client = pendingDownload;
277     m_currentRequest = updatedRequest;
278     m_task->setPendingDownload(pendingDownload);
279
280     if (m_responseCompletionHandler)
281         NetworkProcess::singleton().findPendingDownloadLocation(*m_task.get(), std::exchange(m_responseCompletionHandler, nullptr), response);
282 }
283
284 void NetworkLoad::setPendingDownloadID(DownloadID downloadID)
285 {
286     if (!m_task)
287         return;
288
289     m_task->setPendingDownloadID(downloadID);
290 }
291
292 void NetworkLoad::setSuggestedFilename(const String& suggestedName)
293 {
294     if (!m_task)
295         return;
296
297     m_task->setSuggestedFilename(suggestedName);
298 }
299
300 void NetworkLoad::setPendingDownload(PendingDownload& pendingDownload)
301 {
302     if (!m_task)
303         return;
304
305     m_task->setPendingDownload(pendingDownload);
306 }
307
308 void NetworkLoad::willPerformHTTPRedirection(ResourceResponse&& response, ResourceRequest&& request, RedirectCompletionHandler&& completionHandler)
309 {
310     ASSERT(!m_redirectCompletionHandler);
311     m_redirectCompletionHandler = WTFMove(completionHandler);
312     sharedWillSendRedirectedRequest(WTFMove(request), WTFMove(response));
313 }
314
315 void NetworkLoad::didReceiveChallenge(const AuthenticationChallenge& challenge, ChallengeCompletionHandler&& completionHandler)
316 {
317     // Handle server trust evaluation at platform-level if requested, for performance reasons.
318 #if PLATFORM(COCOA)
319     if (challenge.protectionSpace().authenticationScheme() == ProtectionSpaceAuthenticationSchemeServerTrustEvaluationRequested
320         && !NetworkProcess::singleton().canHandleHTTPSServerTrustEvaluation()) {
321         if (m_task && m_task->allowsSpecificHTTPSCertificateForHost(challenge))
322             completionHandler(AuthenticationChallengeDisposition::UseCredential, serverTrustCredential(challenge));
323         else
324             completionHandler(AuthenticationChallengeDisposition::RejectProtectionSpace, { });
325         return;
326     }
327 #endif
328
329     m_challenge = challenge;
330 #if USE(PROTECTION_SPACE_AUTH_CALLBACK)
331     m_challengeCompletionHandler = WTFMove(completionHandler);
332     m_client.get().canAuthenticateAgainstProtectionSpaceAsync(challenge.protectionSpace());
333 #else
334     completeAuthenticationChallenge(WTFMove(completionHandler));
335 #endif
336 }
337
338 void NetworkLoad::completeAuthenticationChallenge(ChallengeCompletionHandler&& completionHandler)
339 {
340     bool isServerTrustEvaluation = m_challenge->protectionSpace().authenticationScheme() == ProtectionSpaceAuthenticationSchemeServerTrustEvaluationRequested;
341     if (m_parameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials && !isServerTrustEvaluation) {
342         completionHandler(AuthenticationChallengeDisposition::UseCredential, { });
343         return;
344     }
345
346     if (!m_task)
347         return;
348
349     if (auto* pendingDownload = m_task->pendingDownload())
350         NetworkProcess::singleton().authenticationManager().didReceiveAuthenticationChallenge(*pendingDownload, *m_challenge, WTFMove(completionHandler));
351     else
352         NetworkProcess::singleton().authenticationManager().didReceiveAuthenticationChallenge(m_parameters.webPageID, m_parameters.webFrameID, *m_challenge, WTFMove(completionHandler));
353 }
354
355 #if USE(PROTECTION_SPACE_AUTH_CALLBACK)
356 void NetworkLoad::continueCanAuthenticateAgainstProtectionSpace(bool result)
357 {
358     ASSERT(m_challengeCompletionHandler);
359     auto completionHandler = std::exchange(m_challengeCompletionHandler, nullptr);
360     if (!result) {
361         if (m_task && m_task->allowsSpecificHTTPSCertificateForHost(*m_challenge))
362             completionHandler(AuthenticationChallengeDisposition::UseCredential, serverTrustCredential(*m_challenge));
363         else
364             completionHandler(AuthenticationChallengeDisposition::RejectProtectionSpace, { });
365         return;
366     }
367
368     completeAuthenticationChallenge(WTFMove(completionHandler));
369 }
370 #endif
371
372 void NetworkLoad::didReceiveResponseNetworkSession(ResourceResponse&& response, ResponseCompletionHandler&& completionHandler)
373 {
374     ASSERT(RunLoop::isMain());
375     ASSERT(!m_throttle);
376
377     if (m_task && m_task->isDownload()) {
378         NetworkProcess::singleton().findPendingDownloadLocation(*m_task.get(), WTFMove(completionHandler), response);
379         return;
380     }
381
382     auto delay = NetworkProcess::singleton().loadThrottleLatency();
383     if (delay > 0_s) {
384         m_throttle = std::make_unique<Throttle>(*this, delay, WTFMove(response), WTFMove(completionHandler));
385         return;
386     }
387
388     notifyDidReceiveResponse(WTFMove(response), WTFMove(completionHandler));
389 }
390
391 void NetworkLoad::notifyDidReceiveResponse(ResourceResponse&& response, ResponseCompletionHandler&& completionHandler)
392 {
393     ASSERT(RunLoop::isMain());
394
395 #if ENABLE(NETWORK_CAPTURE)
396     if (m_recorder)
397         m_recorder->recordResponseReceived(response);
398 #endif
399
400     if (sharedDidReceiveResponse(WTFMove(response)) == NetworkLoadClient::ShouldContinueDidReceiveResponse::No) {
401         m_responseCompletionHandler = WTFMove(completionHandler);
402         return;
403     }
404     completionHandler(PolicyUse);
405 }
406
407 void NetworkLoad::didReceiveData(Ref<SharedBuffer>&& buffer)
408 {
409     ASSERT(!m_throttle);
410
411 #if ENABLE(NETWORK_CAPTURE)
412     if (m_recorder)
413         m_recorder->recordDataReceived(buffer.get());
414 #endif
415
416     // FIXME: This should be the encoded data length, not the decoded data length.
417     auto size = buffer->size();
418     m_client.get().didReceiveBuffer(WTFMove(buffer), size);
419 }
420
421 void NetworkLoad::didCompleteWithError(const ResourceError& error, const WebCore::NetworkLoadMetrics& networkLoadMetrics)
422 {
423     ASSERT(!m_throttle);
424
425 #if ENABLE(NETWORK_CAPTURE)
426     if (m_recorder)
427         m_recorder->recordFinish(error);
428 #endif
429
430     if (error.isNull())
431         m_client.get().didFinishLoading(networkLoadMetrics);
432     else
433         m_client.get().didFailLoading(error);
434 }
435
436 void NetworkLoad::throttleDelayCompleted()
437 {
438     ASSERT(m_throttle);
439
440     auto throttle = WTFMove(m_throttle);
441
442     notifyDidReceiveResponse(WTFMove(throttle->response), WTFMove(throttle->responseCompletionHandler));
443 }
444
445 void NetworkLoad::didSendData(uint64_t totalBytesSent, uint64_t totalBytesExpectedToSend)
446 {
447     m_client.get().didSendData(totalBytesSent, totalBytesExpectedToSend);
448 }
449
450 void NetworkLoad::wasBlocked()
451 {
452     m_client.get().didFailLoading(blockedError(m_currentRequest));
453 }
454
455 void NetworkLoad::cannotShowURL()
456 {
457     m_client.get().didFailLoading(cannotShowURLError(m_currentRequest));
458 }
459
460 #else
461
462 void NetworkLoad::didReceiveResponseAsync(ResourceHandle* handle, ResourceResponse&& receivedResponse)
463 {
464     ASSERT_UNUSED(handle, handle == m_handle);
465     if (sharedDidReceiveResponse(WTFMove(receivedResponse)) == NetworkLoadClient::ShouldContinueDidReceiveResponse::Yes)
466         m_handle->continueDidReceiveResponse();
467 }
468
469 void NetworkLoad::didReceiveData(ResourceHandle*, const char* /* data */, unsigned /* length */, int /* encodedDataLength */)
470 {
471     // The NetworkProcess should never get a didReceiveData callback.
472     // We should always be using didReceiveBuffer.
473     ASSERT_NOT_REACHED();
474 }
475
476 void NetworkLoad::didReceiveBuffer(ResourceHandle* handle, Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength)
477 {
478     ASSERT_UNUSED(handle, handle == m_handle);
479     m_client.get().didReceiveBuffer(WTFMove(buffer), reportedEncodedDataLength);
480 }
481
482 void NetworkLoad::didFinishLoading(ResourceHandle* handle)
483 {
484     ASSERT_UNUSED(handle, handle == m_handle);
485     NetworkLoadMetrics emptyMetrics;
486     m_client.get().didFinishLoading(emptyMetrics);
487 }
488
489 void NetworkLoad::didFail(ResourceHandle* handle, const ResourceError& error)
490 {
491     ASSERT_UNUSED(handle, !handle || handle == m_handle);
492     ASSERT(!error.isNull());
493
494     m_client.get().didFailLoading(error);
495 }
496
497 void NetworkLoad::willSendRequestAsync(ResourceHandle* handle, ResourceRequest&& request, ResourceResponse&& redirectResponse)
498 {
499     ASSERT_UNUSED(handle, handle == m_handle);
500     sharedWillSendRedirectedRequest(WTFMove(request), WTFMove(redirectResponse));
501 }
502
503 #if USE(PROTECTION_SPACE_AUTH_CALLBACK)
504 void NetworkLoad::canAuthenticateAgainstProtectionSpaceAsync(ResourceHandle* handle, const ProtectionSpace& protectionSpace)
505 {
506     ASSERT(RunLoop::isMain());
507     ASSERT_UNUSED(handle, handle == m_handle);
508
509     // Handle server trust evaluation at platform-level if requested, for performance reasons.
510     if (protectionSpace.authenticationScheme() == ProtectionSpaceAuthenticationSchemeServerTrustEvaluationRequested
511         && !NetworkProcess::singleton().canHandleHTTPSServerTrustEvaluation()) {
512         continueCanAuthenticateAgainstProtectionSpace(false);
513         return;
514     }
515
516     m_waitingForContinueCanAuthenticateAgainstProtectionSpace = true;
517     m_client.get().canAuthenticateAgainstProtectionSpaceAsync(protectionSpace);
518 }
519
520 void NetworkLoad::continueCanAuthenticateAgainstProtectionSpace(bool result)
521 {
522     m_waitingForContinueCanAuthenticateAgainstProtectionSpace = false;
523     if (m_handle)
524         m_handle->continueCanAuthenticateAgainstProtectionSpace(result);
525 }
526 #endif
527
528 void NetworkLoad::didSendData(ResourceHandle* handle, unsigned long long bytesSent, unsigned long long totalBytesToBeSent)
529 {
530     ASSERT_UNUSED(handle, handle == m_handle);
531
532     m_client.get().didSendData(bytesSent, totalBytesToBeSent);
533 }
534
535 void NetworkLoad::wasBlocked(ResourceHandle* handle)
536 {
537     ASSERT_UNUSED(handle, handle == m_handle);
538
539     didFail(handle, WebKit::blockedError(m_currentRequest));
540 }
541
542 void NetworkLoad::cannotShowURL(ResourceHandle* handle)
543 {
544     ASSERT_UNUSED(handle, handle == m_handle);
545
546     didFail(handle, WebKit::cannotShowURLError(m_currentRequest));
547 }
548
549 bool NetworkLoad::shouldUseCredentialStorage(ResourceHandle* handle)
550 {
551     ASSERT_UNUSED(handle, handle == m_handle || !m_handle); // m_handle will be 0 if called from ResourceHandle::start().
552
553     // When the WebProcess is handling loading a client is consulted each time this shouldUseCredentialStorage question is asked.
554     // In NetworkProcess mode we ask the WebProcess client up front once and then reuse the cached answer.
555
556     // We still need this sync version, because ResourceHandle itself uses it internally, even when the delegate uses an async one.
557
558     return m_parameters.allowStoredCredentials == AllowStoredCredentials;
559 }
560
561 void NetworkLoad::didReceiveAuthenticationChallenge(ResourceHandle* handle, const AuthenticationChallenge& challenge)
562 {
563     ASSERT_UNUSED(handle, handle == m_handle);
564
565     if (m_parameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials) {
566         challenge.authenticationClient()->receivedRequestToContinueWithoutCredential(challenge);
567         return;
568     }
569
570     NetworkProcess::singleton().authenticationManager().didReceiveAuthenticationChallenge(m_parameters.webPageID, m_parameters.webFrameID, challenge);
571 }
572
573 void NetworkLoad::receivedCancellation(ResourceHandle* handle, const AuthenticationChallenge&)
574 {
575     ASSERT_UNUSED(handle, handle == m_handle);
576
577     m_handle->cancel();
578     didFail(m_handle.get(), cancelledError(m_currentRequest));
579 }
580 #endif // USE(NETWORK_SESSION)
581
582 } // namespace WebKit