[SOUP] Network process assertion in NetworkDataTask::continueHTTPRedirection
[WebKit-https.git] / Source / WebKit2 / NetworkProcess / soup / NetworkDataTaskSoup.cpp
1 /*
2  * Copyright (C) 2016 Igalia S.L.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkDataTaskSoup.h"
28
29 #include "AuthenticationManager.h"
30 #include "DataReference.h"
31 #include "Download.h"
32 #include "DownloadSoupErrors.h"
33 #include "NetworkLoad.h"
34 #include "NetworkProcess.h"
35 #include "NetworkSessionSoup.h"
36 #include "WebErrors.h"
37 #include <WebCore/AuthenticationChallenge.h>
38 #include <WebCore/HTTPParsers.h>
39 #include <WebCore/NetworkStorageSession.h>
40 #include <WebCore/SharedBuffer.h>
41 #include <WebCore/SoupNetworkSession.h>
42 #include <wtf/MainThread.h>
43
44 using namespace WebCore;
45
46 namespace WebKit {
47
48 static const size_t gDefaultReadBufferSize = 8192;
49
50 NetworkDataTaskSoup::NetworkDataTaskSoup(NetworkSession& session, NetworkDataTaskClient& client, const ResourceRequest& requestWithCredentials, StoredCredentials storedCredentials, ContentSniffingPolicy shouldContentSniff, bool shouldClearReferrerOnHTTPSToHTTPRedirect)
51     : NetworkDataTask(session, client, requestWithCredentials, storedCredentials, shouldClearReferrerOnHTTPSToHTTPRedirect)
52     , m_shouldContentSniff(shouldContentSniff)
53     , m_timeoutSource(RunLoop::main(), this, &NetworkDataTaskSoup::timeoutFired)
54 {
55     static_cast<NetworkSessionSoup&>(m_session.get()).registerNetworkDataTask(*this);
56     if (m_scheduledFailureType != NoFailure)
57         return;
58
59     auto request = requestWithCredentials;
60     if (request.url().protocolIsInHTTPFamily()) {
61 #if ENABLE(WEB_TIMING)
62         m_startTime = monotonicallyIncreasingTimeMS();
63 #endif
64         auto url = request.url();
65         if (m_storedCredentials == AllowStoredCredentials) {
66             m_user = url.user();
67             m_password = url.pass();
68             request.removeCredentials();
69
70             if (m_user.isEmpty() && m_password.isEmpty())
71                 m_initialCredential = m_session->networkStorageSession().credentialStorage().get(request.url());
72             else
73                 m_session->networkStorageSession().credentialStorage().set(Credential(m_user, m_password, CredentialPersistenceNone), request.url());
74         }
75         applyAuthenticationToRequest(request);
76     }
77     createRequest(request);
78 }
79
80 NetworkDataTaskSoup::~NetworkDataTaskSoup()
81 {
82     clearRequest();
83     static_cast<NetworkSessionSoup&>(m_session.get()).unregisterNetworkDataTask(*this);
84 }
85
86 String NetworkDataTaskSoup::suggestedFilename() const
87 {
88     if (!m_suggestedFilename.isEmpty())
89         return m_suggestedFilename;
90
91     String suggestedFilename = m_response.suggestedFilename();
92     if (!suggestedFilename.isEmpty())
93         return suggestedFilename;
94
95     return decodeURLEscapeSequences(m_response.url().lastPathComponent());
96 }
97
98 void NetworkDataTaskSoup::setPendingDownloadLocation(const String& filename, const SandboxExtension::Handle& sandboxExtensionHandle, bool allowOverwrite)
99 {
100     NetworkDataTask::setPendingDownloadLocation(filename, sandboxExtensionHandle, allowOverwrite);
101     m_allowOverwriteDownload = allowOverwrite;
102 }
103
104 void NetworkDataTaskSoup::createRequest(const ResourceRequest& request)
105 {
106     GUniquePtr<SoupURI> soupURI = request.createSoupURI();
107     if (!soupURI) {
108         scheduleFailure(InvalidURLFailure);
109         return;
110     }
111
112     GRefPtr<SoupRequest> soupRequest = adoptGRef(soup_session_request_uri(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), soupURI.get(), nullptr));
113     if (!soupRequest) {
114         scheduleFailure(InvalidURLFailure);
115         return;
116     }
117
118     request.updateSoupRequest(soupRequest.get());
119
120     if (!request.url().protocolIsInHTTPFamily()) {
121         m_soupRequest = WTFMove(soupRequest);
122         return;
123     }
124
125     // HTTP request.
126     GRefPtr<SoupMessage> soupMessage = adoptGRef(soup_request_http_get_message(SOUP_REQUEST_HTTP(soupRequest.get())));
127     if (!soupMessage) {
128         scheduleFailure(InvalidURLFailure);
129         return;
130     }
131
132     request.updateSoupMessage(soupMessage.get());
133     if (m_shouldContentSniff == DoNotSniffContent)
134         soup_message_disable_feature(soupMessage.get(), SOUP_TYPE_CONTENT_SNIFFER);
135     if (m_user.isEmpty() && m_password.isEmpty() && m_storedCredentials == DoNotAllowStoredCredentials) {
136         // In case credential is not available and credential storage should not to be used,
137         // disable authentication manager so that credentials stored in libsoup are not used.
138         soup_message_disable_feature(soupMessage.get(), SOUP_TYPE_AUTH_MANAGER);
139     }
140
141     // Make sure we have an Accept header for subresources; some sites want this to serve some of their subresources.
142     if (!soup_message_headers_get_one(soupMessage->request_headers, "Accept"))
143         soup_message_headers_append(soupMessage->request_headers, "Accept", "*/*");
144
145     // In the case of XHR .send() and .send("") explicitly tell libsoup to send a zero content-lenght header
146     // for consistency with other UA implementations like Firefox. It's done in the backend here instead of
147     // in XHR code since in XHR CORS checking prevents us from this kind of late header manipulation.
148     if ((soupMessage->method == SOUP_METHOD_POST || soupMessage->method == SOUP_METHOD_PUT) && !soupMessage->request_body->length)
149         soup_message_headers_set_content_length(soupMessage->request_headers, 0);
150
151     unsigned flags = SOUP_MESSAGE_NO_REDIRECT;
152     soup_message_set_flags(soupMessage.get(), static_cast<SoupMessageFlags>(soup_message_get_flags(soupMessage.get()) | flags));
153
154 #if SOUP_CHECK_VERSION(2, 43, 1)
155     soup_message_set_priority(soupMessage.get(), toSoupMessagePriority(request.priority()));
156 #endif
157
158     m_soupRequest = WTFMove(soupRequest);
159     m_soupMessage = WTFMove(soupMessage);
160
161     g_signal_connect(m_soupMessage.get(), "notify::tls-errors", G_CALLBACK(tlsErrorsChangedCallback), this);
162     g_signal_connect(m_soupMessage.get(), "got-headers", G_CALLBACK(gotHeadersCallback), this);
163     g_signal_connect(m_soupMessage.get(), "wrote-body-data", G_CALLBACK(wroteBodyDataCallback), this);
164     g_signal_connect(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), "authenticate",  G_CALLBACK(authenticateCallback), this);
165 #if ENABLE(WEB_TIMING)
166     g_signal_connect(m_soupMessage.get(), "network-event", G_CALLBACK(networkEventCallback), this);
167     g_signal_connect(m_soupMessage.get(), "restarted", G_CALLBACK(restartedCallback), this);
168 #if SOUP_CHECK_VERSION(2, 49, 91)
169     g_signal_connect(m_soupMessage.get(), "starting", G_CALLBACK(startingCallback), this);
170 #else
171     g_signal_connect(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), "request-started", G_CALLBACK(requestStartedCallback), this);
172 #endif
173 #endif
174 }
175
176 void NetworkDataTaskSoup::clearRequest()
177 {
178     if (m_state == State::Completed)
179         return;
180
181     m_state = State::Completed;
182
183     stopTimeout();
184     m_pendingResult = nullptr;
185     m_soupRequest = nullptr;
186     m_inputStream = nullptr;
187     m_multipartInputStream = nullptr;
188     m_downloadOutputStream = nullptr;
189     g_cancellable_cancel(m_cancellable.get());
190     m_cancellable = nullptr;
191     if (m_soupMessage) {
192         g_signal_handlers_disconnect_matched(m_soupMessage.get(), G_SIGNAL_MATCH_DATA, 0, 0, nullptr, nullptr, this);
193         soup_session_cancel_message(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), m_soupMessage.get(), SOUP_STATUS_CANCELLED);
194         m_soupMessage = nullptr;
195     }
196     g_signal_handlers_disconnect_matched(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), G_SIGNAL_MATCH_DATA, 0, 0, nullptr, nullptr, this);
197 }
198
199 void NetworkDataTaskSoup::resume()
200 {
201     ASSERT(m_state != State::Running);
202     if (m_state == State::Canceling || m_state == State::Completed)
203         return;
204
205     m_state = State::Running;
206
207     if (m_scheduledFailureType != NoFailure) {
208         ASSERT(m_failureTimer.isActive());
209         return;
210     }
211
212     startTimeout();
213
214     RefPtr<NetworkDataTaskSoup> protectedThis(this);
215     if (m_soupRequest && !m_cancellable) {
216         m_cancellable = adoptGRef(g_cancellable_new());
217         soup_request_send_async(m_soupRequest.get(), m_cancellable.get(), reinterpret_cast<GAsyncReadyCallback>(sendRequestCallback), protectedThis.leakRef());
218         return;
219     }
220
221     if (m_pendingResult) {
222         GRefPtr<GAsyncResult> pendingResult = WTFMove(m_pendingResult);
223         if (m_inputStream)
224             readCallback(m_inputStream.get(), pendingResult.get(), protectedThis.leakRef());
225         else if (m_multipartInputStream)
226             requestNextPartCallback(m_multipartInputStream.get(), pendingResult.get(), protectedThis.leakRef());
227         else if (m_soupRequest)
228             sendRequestCallback(m_soupRequest.get(), pendingResult.get(), protectedThis.leakRef());
229         else
230             ASSERT_NOT_REACHED();
231     }
232 }
233
234 void NetworkDataTaskSoup::suspend()
235 {
236     ASSERT(m_state != State::Suspended);
237     if (m_state == State::Canceling || m_state == State::Completed)
238         return;
239     m_state = State::Suspended;
240
241     stopTimeout();
242 }
243
244 void NetworkDataTaskSoup::cancel()
245 {
246     if (m_state == State::Canceling || m_state == State::Completed)
247         return;
248
249     m_state = State::Canceling;
250
251     if (m_soupMessage)
252         soup_session_cancel_message(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), m_soupMessage.get(), SOUP_STATUS_CANCELLED);
253
254     g_cancellable_cancel(m_cancellable.get());
255
256     if (isDownload())
257         cleanDownloadFiles();
258 }
259
260 void NetworkDataTaskSoup::invalidateAndCancel()
261 {
262     cancel();
263     clearRequest();
264 }
265
266 NetworkDataTask::State NetworkDataTaskSoup::state() const
267 {
268     return m_state;
269 }
270
271 void NetworkDataTaskSoup::timeoutFired()
272 {
273     if (m_state == State::Canceling || m_state == State::Completed || !m_client) {
274         clearRequest();
275         return;
276     }
277
278     RefPtr<NetworkDataTaskSoup> protectedThis(this);
279     invalidateAndCancel();
280     m_client->didCompleteWithError(ResourceError::timeoutError(m_firstRequest.url()));
281 }
282
283 void NetworkDataTaskSoup::startTimeout()
284 {
285     if (m_firstRequest.timeoutInterval() > 0)
286         m_timeoutSource.startOneShot(m_firstRequest.timeoutInterval());
287 }
288
289 void NetworkDataTaskSoup::stopTimeout()
290 {
291     m_timeoutSource.stop();
292 }
293
294 void NetworkDataTaskSoup::sendRequestCallback(SoupRequest* soupRequest, GAsyncResult* result, NetworkDataTaskSoup* task)
295 {
296     RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task);
297     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) {
298         task->clearRequest();
299         return;
300     }
301     ASSERT(soupRequest == task->m_soupRequest.get());
302
303     if (task->state() == State::Suspended) {
304         ASSERT(!task->m_pendingResult);
305         task->m_pendingResult = result;
306         return;
307     }
308
309     GUniqueOutPtr<GError> error;
310     GRefPtr<GInputStream> inputStream = adoptGRef(soup_request_send_finish(soupRequest, result, &error.outPtr()));
311     if (error)
312         task->didFail(ResourceError::httpError(task->m_soupMessage.get(), error.get(), soupRequest));
313     else
314         task->didSendRequest(WTFMove(inputStream));
315 }
316
317 void NetworkDataTaskSoup::didSendRequest(GRefPtr<GInputStream>&& inputStream)
318 {
319     if (m_soupMessage) {
320         if (m_shouldContentSniff == SniffContent && m_soupMessage->status_code != SOUP_STATUS_NOT_MODIFIED)
321             m_response.setSniffedContentType(soup_request_get_content_type(m_soupRequest.get()));
322         m_response.updateFromSoupMessage(m_soupMessage.get());
323
324         if (shouldStartHTTPRedirection()) {
325             m_inputStream = WTFMove(inputStream);
326             skipInputStreamForRedirection();
327             return;
328         }
329
330         if (m_response.isMultipart())
331             m_multipartInputStream = adoptGRef(soup_multipart_input_stream_new(m_soupMessage.get(), inputStream.get()));
332         else
333             m_inputStream = WTFMove(inputStream);
334
335 #if ENABLE(WEB_TIMING)
336         m_response.networkLoadTiming().responseStart = monotonicallyIncreasingTimeMS() - m_startTime;
337 #endif
338     } else {
339         m_response.setURL(m_firstRequest.url());
340         const gchar* contentType = soup_request_get_content_type(m_soupRequest.get());
341         m_response.setMimeType(extractMIMETypeFromMediaType(contentType));
342         m_response.setTextEncodingName(extractCharsetFromMediaType(contentType));
343         m_response.setExpectedContentLength(soup_request_get_content_length(m_soupRequest.get()));
344
345         m_inputStream = WTFMove(inputStream);
346     }
347
348     didReceiveResponse();
349 }
350
351 void NetworkDataTaskSoup::didReceiveResponse()
352 {
353     ASSERT(!m_response.isNull());
354
355     auto response = ResourceResponse(m_response);
356     m_client->didReceiveResponseNetworkSession(WTFMove(response), [this, protectedThis = makeRef(*this)](PolicyAction policyAction) {
357         if (m_state == State::Canceling || m_state == State::Completed) {
358             clearRequest();
359             return;
360         }
361
362         switch (policyAction) {
363         case PolicyAction::PolicyUse:
364             if (m_inputStream)
365                 read();
366             else if (m_multipartInputStream)
367                 requestNextPart();
368             else
369                 ASSERT_NOT_REACHED();
370
371             break;
372         case PolicyAction::PolicyIgnore:
373             clearRequest();
374             break;
375         case PolicyAction::PolicyDownload:
376             download();
377             break;
378         }
379     });
380 }
381
382 void NetworkDataTaskSoup::tlsErrorsChangedCallback(SoupMessage* soupMessage, GParamSpec*, NetworkDataTaskSoup* task)
383 {
384     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) {
385         task->clearRequest();
386         return;
387     }
388
389     ASSERT(soupMessage == task->m_soupMessage.get());
390     task->tlsErrorsChanged();
391 }
392
393 void NetworkDataTaskSoup::tlsErrorsChanged()
394 {
395     ASSERT(m_soupRequest);
396     SoupNetworkSession::checkTLSErrors(m_soupRequest.get(), m_soupMessage.get(), [this] (const ResourceError& error) {
397         if (error.isNull())
398             return;
399
400         RefPtr<NetworkDataTaskSoup> protectedThis(this);
401         invalidateAndCancel();
402         m_client->didCompleteWithError(error);
403     });
404 }
405
406 void NetworkDataTaskSoup::applyAuthenticationToRequest(ResourceRequest& request)
407 {
408     // We always put the credentials into the URL. In the Coca port HTTP family credentials are applied in
409     // the didReceiveChallenge callback, but libsoup requires us to use this method to override
410     // any previously remembered credentials. It has its own per-session credential storage.
411     auto url = request.url();
412     if (!m_initialCredential.isEmpty()) {
413         url.setUser(m_initialCredential.user());
414         url.setPass(m_initialCredential.password());
415     } else {
416         url.setUser(m_user);
417         url.setPass(m_password);
418     }
419
420     m_user = String();
421     m_password = String();
422
423     request.setURL(url);
424 }
425
426 void NetworkDataTaskSoup::authenticateCallback(SoupSession* session, SoupMessage* soupMessage, SoupAuth* soupAuth, gboolean retrying, NetworkDataTaskSoup* task)
427 {
428     ASSERT(session == static_cast<NetworkSessionSoup&>(task->m_session.get()).soupSession());
429     if (soupMessage != task->m_soupMessage.get())
430         return;
431
432     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) {
433         task->clearRequest();
434         return;
435     }
436
437     task->authenticate(AuthenticationChallenge(soupMessage, soupAuth, retrying));
438 }
439
440 static inline bool isAuthenticationFailureStatusCode(int httpStatusCode)
441 {
442     return httpStatusCode == SOUP_STATUS_PROXY_AUTHENTICATION_REQUIRED || httpStatusCode == SOUP_STATUS_UNAUTHORIZED;
443 }
444
445 void NetworkDataTaskSoup::authenticate(AuthenticationChallenge&& challenge)
446 {
447     ASSERT(m_soupMessage);
448     if (m_storedCredentials == AllowStoredCredentials) {
449         if (!m_initialCredential.isEmpty() || challenge.previousFailureCount()) {
450             // The stored credential wasn't accepted, stop using it. There is a race condition
451             // here, since a different credential might have already been stored by another
452             // NetworkDataTask, but the observable effect should be very minor, if any.
453             m_session->networkStorageSession().credentialStorage().remove(challenge.protectionSpace());
454         }
455
456         if (!challenge.previousFailureCount()) {
457             auto credential = m_session->networkStorageSession().credentialStorage().get(challenge.protectionSpace());
458             if (!credential.isEmpty() && credential != m_initialCredential) {
459                 ASSERT(credential.persistence() == CredentialPersistenceNone);
460
461                 if (isAuthenticationFailureStatusCode(challenge.failureResponse().httpStatusCode())) {
462                     // Store the credential back, possibly adding it as a default for this directory.
463                     m_session->networkStorageSession().credentialStorage().set(credential, challenge.protectionSpace(), challenge.failureResponse().url());
464                 }
465                 soup_auth_authenticate(challenge.soupAuth(), credential.user().utf8().data(), credential.password().utf8().data());
466                 return;
467             }
468         }
469     }
470
471     soup_session_pause_message(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), m_soupMessage.get());
472
473     // We could also do this before we even start the request, but that would be at the expense
474     // of all request latency, versus a one-time latency for the small subset of requests that
475     // use HTTP authentication. In the end, this doesn't matter much, because persistent credentials
476     // will become session credentials after the first use.
477     if (m_storedCredentials == AllowStoredCredentials) {
478         auto protectionSpace = challenge.protectionSpace();
479         m_session->networkStorageSession().getCredentialFromPersistentStorage(protectionSpace,
480             [this, protectedThis = makeRef(*this), authChallenge = WTFMove(challenge)] (Credential&& credential) mutable {
481                 if (m_state == State::Canceling || m_state == State::Completed || !m_client) {
482                     clearRequest();
483                     return;
484                 }
485
486                 authChallenge.setProposedCredential(WTFMove(credential));
487                 continueAuthenticate(WTFMove(authChallenge));
488         });
489     } else
490         continueAuthenticate(WTFMove(challenge));
491 }
492
493 void NetworkDataTaskSoup::continueAuthenticate(AuthenticationChallenge&& challenge)
494 {
495     m_client->didReceiveChallenge(challenge, [this, protectedThis = makeRef(*this), challenge](AuthenticationChallengeDisposition disposition, const Credential& credential) {
496         if (m_state == State::Canceling || m_state == State::Completed) {
497             clearRequest();
498             return;
499         }
500
501         if (disposition == AuthenticationChallengeDisposition::Cancel) {
502             cancel();
503             didFail(cancelledError(m_soupRequest.get()));
504             return;
505         }
506
507         if (disposition == AuthenticationChallengeDisposition::UseCredential && !credential.isEmpty()) {
508             if (m_storedCredentials == AllowStoredCredentials) {
509                 // Eventually we will manage per-session credentials only internally or use some newly-exposed API from libsoup,
510                 // because once we authenticate via libsoup, there is no way to ignore it for a particular request. Right now,
511                 // we place the credentials in the store even though libsoup will never fire the authenticate signal again for
512                 // this protection space.
513                 if (credential.persistence() == CredentialPersistenceForSession || credential.persistence() == CredentialPersistencePermanent)
514                     m_session->networkStorageSession().credentialStorage().set(credential, challenge.protectionSpace(), challenge.failureResponse().url());
515
516                 if (credential.persistence() == CredentialPersistencePermanent) {
517                     m_protectionSpaceForPersistentStorage = challenge.protectionSpace();
518                     m_credentialForPersistentStorage = credential;
519                 }
520             }
521
522             soup_auth_authenticate(challenge.soupAuth(), credential.user().utf8().data(), credential.password().utf8().data());
523         }
524
525         soup_session_unpause_message(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), m_soupMessage.get());
526     });
527 }
528
529 void NetworkDataTaskSoup::skipInputStreamForRedirectionCallback(GInputStream* inputStream, GAsyncResult* result, NetworkDataTaskSoup* task)
530 {
531     RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task);
532     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) {
533         task->clearRequest();
534         return;
535     }
536     ASSERT(inputStream == task->m_inputStream.get());
537
538     GUniqueOutPtr<GError> error;
539     gssize bytesSkipped = g_input_stream_skip_finish(inputStream, result, &error.outPtr());
540     if (error)
541         task->didFail(ResourceError::genericGError(error.get(), task->m_soupRequest.get()));
542     else if (bytesSkipped > 0)
543         task->skipInputStreamForRedirection();
544     else
545         task->didFinishSkipInputStreamForRedirection();
546 }
547
548 void NetworkDataTaskSoup::skipInputStreamForRedirection()
549 {
550     ASSERT(m_inputStream);
551     RefPtr<NetworkDataTaskSoup> protectedThis(this);
552     g_input_stream_skip_async(m_inputStream.get(), gDefaultReadBufferSize, G_PRIORITY_DEFAULT, m_cancellable.get(),
553         reinterpret_cast<GAsyncReadyCallback>(skipInputStreamForRedirectionCallback), protectedThis.leakRef());
554 }
555
556 void NetworkDataTaskSoup::didFinishSkipInputStreamForRedirection()
557 {
558     g_input_stream_close(m_inputStream.get(), nullptr, nullptr);
559     continueHTTPRedirection();
560 }
561
562 static bool shouldRedirectAsGET(SoupMessage* message, bool crossOrigin)
563 {
564     if (message->method == SOUP_METHOD_GET || message->method == SOUP_METHOD_HEAD)
565         return false;
566
567     switch (message->status_code) {
568     case SOUP_STATUS_SEE_OTHER:
569         return true;
570     case SOUP_STATUS_FOUND:
571     case SOUP_STATUS_MOVED_PERMANENTLY:
572         if (message->method == SOUP_METHOD_POST)
573             return true;
574         break;
575     }
576
577     if (crossOrigin && message->method == SOUP_METHOD_DELETE)
578         return true;
579
580     return false;
581 }
582
583 bool NetworkDataTaskSoup::shouldStartHTTPRedirection()
584 {
585     ASSERT(m_soupMessage);
586     ASSERT(!m_response.isNull());
587
588     auto status = m_response.httpStatusCode();
589     if (!SOUP_STATUS_IS_REDIRECTION(status))
590         return false;
591
592     // Some 3xx status codes aren't actually redirects.
593     if (status == 300 || status == 304 || status == 305 || status == 306)
594         return false;
595
596     if (m_response.httpHeaderField(HTTPHeaderName::Location).isEmpty())
597         return false;
598
599     return true;
600 }
601
602 void NetworkDataTaskSoup::continueHTTPRedirection()
603 {
604     ASSERT(m_soupMessage);
605     ASSERT(!m_response.isNull());
606
607     static const unsigned maxRedirects = 20;
608     if (m_redirectCount++ > maxRedirects) {
609         didFail(ResourceError::transportError(m_soupRequest.get(), SOUP_STATUS_TOO_MANY_REDIRECTS, "Too many redirects"));
610         return;
611     }
612
613     ResourceRequest request = m_firstRequest;
614     request.setURL(URL(m_response.url(), m_response.httpHeaderField(HTTPHeaderName::Location)));
615     request.setFirstPartyForCookies(request.url());
616
617     // Should not set Referer after a redirect from a secure resource to non-secure one.
618     if (m_shouldClearReferrerOnHTTPSToHTTPRedirect && !request.url().protocolIs("https") && protocolIs(request.httpReferrer(), "https"))
619         request.clearHTTPReferrer();
620
621     bool isCrossOrigin = !protocolHostAndPortAreEqual(m_firstRequest.url(), request.url());
622     if (!equalLettersIgnoringASCIICase(request.httpMethod(), "get")) {
623         // Change newRequest method to GET if change was made during a previous redirection or if current redirection says so.
624         if (m_soupMessage->method == SOUP_METHOD_GET || !request.url().protocolIsInHTTPFamily() || shouldRedirectAsGET(m_soupMessage.get(), isCrossOrigin)) {
625             request.setHTTPMethod("GET");
626             request.setHTTPBody(nullptr);
627             request.clearHTTPContentType();
628         }
629     }
630
631     const auto& url = request.url();
632     m_user = url.user();
633     m_password = url.pass();
634     m_lastHTTPMethod = request.httpMethod();
635     request.removeCredentials();
636
637     if (isCrossOrigin) {
638         // The network layer might carry over some headers from the original request that
639         // we want to strip here because the redirect is cross-origin.
640         request.clearHTTPAuthorization();
641         request.clearHTTPOrigin();
642     } else if (url.protocolIsInHTTPFamily() && m_storedCredentials == AllowStoredCredentials) {
643         if (m_user.isEmpty() && m_password.isEmpty()) {
644             auto credential = m_session->networkStorageSession().credentialStorage().get(request.url());
645             if (!credential.isEmpty())
646                 m_initialCredential = credential;
647         }
648     }
649
650     clearRequest();
651
652     auto response = ResourceResponse(m_response);
653     m_client->willPerformHTTPRedirection(WTFMove(response), WTFMove(request), [this, protectedThis = makeRef(*this), isCrossOrigin](const ResourceRequest& newRequest) {
654         if (newRequest.isNull() || m_state == State::Canceling)
655             return;
656
657         auto request = newRequest;
658         if (request.url().protocolIsInHTTPFamily()) {
659 #if ENABLE(WEB_TIMING)
660             if (isCrossOrigin)
661                 m_startTime = monotonicallyIncreasingTimeMS();
662 #endif
663             applyAuthenticationToRequest(request);
664         }
665         createRequest(request);
666         if (m_soupRequest && m_state != State::Suspended) {
667             m_state = State::Suspended;
668             resume();
669         }
670     });
671 }
672
673 void NetworkDataTaskSoup::readCallback(GInputStream* inputStream, GAsyncResult* result, NetworkDataTaskSoup* task)
674 {
675     RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task);
676     if (task->state() == State::Canceling || task->state() == State::Completed || (!task->m_client && !task->isDownload())) {
677         task->clearRequest();
678         return;
679     }
680     ASSERT(inputStream == task->m_inputStream.get());
681
682     if (task->state() == State::Suspended) {
683         ASSERT(!task->m_pendingResult);
684         task->m_pendingResult = result;
685         return;
686     }
687
688     GUniqueOutPtr<GError> error;
689     gssize bytesRead = g_input_stream_read_finish(inputStream, result, &error.outPtr());
690     if (error)
691         task->didFail(ResourceError::genericGError(error.get(), task->m_soupRequest.get()));
692     else if (bytesRead > 0)
693         task->didRead(bytesRead);
694     else
695         task->didFinishRead();
696 }
697
698 void NetworkDataTaskSoup::read()
699 {
700     RefPtr<NetworkDataTaskSoup> protectedThis(this);
701     ASSERT(m_inputStream);
702     m_readBuffer.grow(gDefaultReadBufferSize);
703     g_input_stream_read_async(m_inputStream.get(), m_readBuffer.data(), m_readBuffer.size(), G_PRIORITY_DEFAULT, m_cancellable.get(),
704         reinterpret_cast<GAsyncReadyCallback>(readCallback), protectedThis.leakRef());
705 }
706
707 void NetworkDataTaskSoup::didRead(gssize bytesRead)
708 {
709     m_readBuffer.shrink(bytesRead);
710     if (m_downloadOutputStream) {
711         ASSERT(isDownload());
712         writeDownload();
713     } else {
714         ASSERT(m_client);
715         m_client->didReceiveData(SharedBuffer::adoptVector(m_readBuffer));
716         read();
717     }
718 }
719
720 void NetworkDataTaskSoup::didFinishRead()
721 {
722     ASSERT(m_inputStream);
723     g_input_stream_close(m_inputStream.get(), nullptr, nullptr);
724     m_inputStream = nullptr;
725     if (m_multipartInputStream) {
726         requestNextPart();
727         return;
728     }
729
730     if (m_downloadOutputStream) {
731         didFinishDownload();
732         return;
733     }
734
735     clearRequest();
736     ASSERT(m_client);
737     m_client->didCompleteWithError({ });
738 }
739
740 void NetworkDataTaskSoup::requestNextPartCallback(SoupMultipartInputStream* multipartInputStream, GAsyncResult* result, NetworkDataTaskSoup* task)
741 {
742     RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task);
743     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) {
744         task->clearRequest();
745         return;
746     }
747     ASSERT(multipartInputStream == task->m_multipartInputStream.get());
748
749     if (task->state() == State::Suspended) {
750         ASSERT(!task->m_pendingResult);
751         task->m_pendingResult = result;
752         return;
753     }
754
755     GUniqueOutPtr<GError> error;
756     GRefPtr<GInputStream> inputStream = adoptGRef(soup_multipart_input_stream_next_part_finish(multipartInputStream, result, &error.outPtr()));
757     if (error)
758         task->didFail(ResourceError::httpError(task->m_soupMessage.get(), error.get(), task->m_soupRequest.get()));
759     else if (inputStream)
760         task->didRequestNextPart(WTFMove(inputStream));
761     else
762         task->didFinishRequestNextPart();
763 }
764
765 void NetworkDataTaskSoup::requestNextPart()
766 {
767     RefPtr<NetworkDataTaskSoup> protectedThis(this);
768     ASSERT(m_multipartInputStream);
769     ASSERT(!m_inputStream);
770     soup_multipart_input_stream_next_part_async(m_multipartInputStream.get(), G_PRIORITY_DEFAULT, m_cancellable.get(),
771         reinterpret_cast<GAsyncReadyCallback>(requestNextPartCallback), protectedThis.leakRef());
772 }
773
774 void NetworkDataTaskSoup::didRequestNextPart(GRefPtr<GInputStream>&& inputStream)
775 {
776     ASSERT(!m_inputStream);
777     m_inputStream = WTFMove(inputStream);
778     m_response = ResourceResponse();
779     m_response.setURL(m_firstRequest.url());
780     m_response.updateFromSoupMessageHeaders(soup_multipart_input_stream_get_headers(m_multipartInputStream.get()));
781     didReceiveResponse();
782 }
783
784 void NetworkDataTaskSoup::didFinishRequestNextPart()
785 {
786     ASSERT(!m_inputStream);
787     ASSERT(m_multipartInputStream);
788     g_input_stream_close(G_INPUT_STREAM(m_multipartInputStream.get()), nullptr, nullptr);
789     clearRequest();
790     m_client->didCompleteWithError({ });
791 }
792
793 void NetworkDataTaskSoup::gotHeadersCallback(SoupMessage* soupMessage, NetworkDataTaskSoup* task)
794 {
795     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) {
796         task->clearRequest();
797         return;
798     }
799     ASSERT(task->m_soupMessage.get() == soupMessage);
800     task->didGetHeaders();
801 }
802
803 void NetworkDataTaskSoup::didGetHeaders()
804 {
805     // We are a bit more conservative with the persistent credential storage than the session store,
806     // since we are waiting until we know that this authentication succeeded before actually storing.
807     // This is because we want to avoid hitting the disk twice (once to add and once to remove) for
808     // incorrect credentials or polluting the keychain with invalid credentials.
809     if (!isAuthenticationFailureStatusCode(m_soupMessage->status_code) && m_soupMessage->status_code < 500) {
810         m_session->networkStorageSession().saveCredentialToPersistentStorage(m_protectionSpaceForPersistentStorage, m_credentialForPersistentStorage);
811         m_protectionSpaceForPersistentStorage = ProtectionSpace();
812         m_credentialForPersistentStorage = Credential();
813     }
814 }
815
816 void NetworkDataTaskSoup::wroteBodyDataCallback(SoupMessage* soupMessage, SoupBuffer* buffer, NetworkDataTaskSoup* task)
817 {
818     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) {
819         task->clearRequest();
820         return;
821     }
822     ASSERT(task->m_soupMessage.get() == soupMessage);
823     task->didWriteBodyData(buffer->length);
824 }
825
826 void NetworkDataTaskSoup::didWriteBodyData(uint64_t bytesSent)
827 {
828     RefPtr<NetworkDataTaskSoup> protectedThis(this);
829     m_bodyDataTotalBytesSent += bytesSent;
830     m_client->didSendData(m_bodyDataTotalBytesSent, m_soupMessage->request_body->length);
831 }
832
833 void NetworkDataTaskSoup::download()
834 {
835     ASSERT(isDownload());
836     ASSERT(m_pendingDownloadLocation);
837     ASSERT(!m_response.isNull());
838
839     if (m_response.httpStatusCode() >= 400) {
840         didFailDownload(platformDownloadNetworkError(m_response.httpStatusCode(), m_response.url(), m_response.httpStatusText()));
841         return;
842     }
843
844     m_downloadDestinationFile = adoptGRef(g_file_new_for_uri(m_pendingDownloadLocation.utf8().data()));
845     GRefPtr<GFileOutputStream> outputStream;
846     GUniqueOutPtr<GError> error;
847     if (m_allowOverwriteDownload)
848         outputStream = adoptGRef(g_file_replace(m_downloadDestinationFile.get(), nullptr, FALSE, G_FILE_CREATE_NONE, nullptr, &error.outPtr()));
849     else
850         outputStream = adoptGRef(g_file_create(m_downloadDestinationFile.get(), G_FILE_CREATE_NONE, nullptr, &error.outPtr()));
851     if (!outputStream) {
852         didFailDownload(platformDownloadDestinationError(m_response, error->message));
853         return;
854     }
855
856     String intermediateURI = m_pendingDownloadLocation + ".wkdownload";
857     m_downloadIntermediateFile = adoptGRef(g_file_new_for_uri(intermediateURI.utf8().data()));
858     outputStream = adoptGRef(g_file_replace(m_downloadIntermediateFile.get(), 0, TRUE, G_FILE_CREATE_NONE, 0, &error.outPtr()));
859     if (!outputStream) {
860         didFailDownload(platformDownloadDestinationError(m_response, error->message));
861         return;
862     }
863     m_downloadOutputStream = adoptGRef(G_OUTPUT_STREAM(outputStream.leakRef()));
864
865     auto& downloadManager = NetworkProcess::singleton().downloadManager();
866     auto download = std::make_unique<Download>(downloadManager, m_pendingDownloadID, this, m_session->sessionID(), suggestedFilename());
867     auto* downloadPtr = download.get();
868     downloadManager.dataTaskBecameDownloadTask(m_pendingDownloadID, WTFMove(download));
869     downloadPtr->didCreateDestination(m_pendingDownloadLocation);
870
871     ASSERT(m_client);
872     m_client->didBecomeDownload();
873
874     read();
875 }
876
877 void NetworkDataTaskSoup::writeDownloadCallback(GOutputStream* outputStream, GAsyncResult* result, NetworkDataTaskSoup* task)
878 {
879     RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task);
880     if (task->state() == State::Canceling || task->state() == State::Completed || !task->isDownload()) {
881         task->clearRequest();
882         return;
883     }
884     ASSERT(outputStream == task->m_downloadOutputStream.get());
885
886     GUniqueOutPtr<GError> error;
887     gsize bytesWritten;
888 #if GLIB_CHECK_VERSION(2, 44, 0)
889     g_output_stream_write_all_finish(outputStream, result, &bytesWritten, &error.outPtr());
890 #else
891     gssize writeTaskResult = g_task_propagate_int(G_TASK(result), &error.outPtr());
892     if (writeTaskResult != -1)
893         bytesWritten = writeTaskResult;
894 #endif
895     if (error)
896         task->didFailDownload(platformDownloadDestinationError(task->m_response, error->message));
897     else
898         task->didWriteDownload(bytesWritten);
899 }
900
901 void NetworkDataTaskSoup::writeDownload()
902 {
903     RefPtr<NetworkDataTaskSoup> protectedThis(this);
904 #if GLIB_CHECK_VERSION(2, 44, 0)
905     g_output_stream_write_all_async(m_downloadOutputStream.get(), m_readBuffer.data(), m_readBuffer.size(), G_PRIORITY_DEFAULT, m_cancellable.get(),
906         reinterpret_cast<GAsyncReadyCallback>(writeDownloadCallback), protectedThis.leakRef());
907 #else
908     GRefPtr<GTask> writeTask = adoptGRef(g_task_new(m_downloadOutputStream.get(), m_cancellable.get(),
909         reinterpret_cast<GAsyncReadyCallback>(writeDownloadCallback), protectedThis.leakRef()));
910     g_task_set_task_data(writeTask.get(), this, nullptr);
911     g_task_run_in_thread(writeTask.get(), [](GTask* writeTask, gpointer source, gpointer userData, GCancellable* cancellable) {
912         auto* task = static_cast<NetworkDataTaskSoup*>(userData);
913         GOutputStream* outputStream = G_OUTPUT_STREAM(source);
914         RELEASE_ASSERT(task->m_downloadOutputStream.get() == outputStream);
915         RELEASE_ASSERT(task->m_cancellable.get() == cancellable);
916         GError* error = nullptr;
917         if (g_cancellable_set_error_if_cancelled(cancellable, &error)) {
918             g_task_return_error(writeTask, error);
919             return;
920         }
921
922         gsize bytesWritten;
923         if (g_output_stream_write_all(outputStream, task->m_readBuffer.data(), task->m_readBuffer.size(), &bytesWritten, cancellable, &error))
924             g_task_return_int(writeTask, bytesWritten);
925         else
926             g_task_return_error(writeTask, error);
927     });
928 #endif
929 }
930
931 void NetworkDataTaskSoup::didWriteDownload(gsize bytesWritten)
932 {
933     ASSERT(bytesWritten == m_readBuffer.size());
934     auto* download = NetworkProcess::singleton().downloadManager().download(m_pendingDownloadID);
935     ASSERT(download);
936     download->didReceiveData(bytesWritten);
937     read();
938 }
939
940 void NetworkDataTaskSoup::didFinishDownload()
941 {
942     ASSERT(!m_response.isNull());
943     ASSERT(m_downloadOutputStream);
944     g_output_stream_close(m_downloadOutputStream.get(), nullptr, nullptr);
945     m_downloadOutputStream = nullptr;
946
947     ASSERT(m_downloadDestinationFile);
948     ASSERT(m_downloadIntermediateFile);
949     GUniqueOutPtr<GError> error;
950     if (!g_file_move(m_downloadIntermediateFile.get(), m_downloadDestinationFile.get(), G_FILE_COPY_OVERWRITE, m_cancellable.get(), nullptr, nullptr, &error.outPtr())) {
951         didFailDownload(platformDownloadDestinationError(m_response, error->message));
952         return;
953     }
954
955     GRefPtr<GFileInfo> info = adoptGRef(g_file_info_new());
956     CString uri = m_response.url().string().utf8();
957     g_file_info_set_attribute_string(info.get(), "metadata::download-uri", uri.data());
958     g_file_info_set_attribute_string(info.get(), "xattr::xdg.origin.url", uri.data());
959     g_file_set_attributes_async(m_downloadDestinationFile.get(), info.get(), G_FILE_QUERY_INFO_NONE, G_PRIORITY_DEFAULT, nullptr, nullptr, nullptr);
960
961     clearRequest();
962     auto* download = NetworkProcess::singleton().downloadManager().download(m_pendingDownloadID);
963     ASSERT(download);
964     download->didFinish();
965 }
966
967 void NetworkDataTaskSoup::didFailDownload(const ResourceError& error)
968 {
969     clearRequest();
970     cleanDownloadFiles();
971     if (m_client)
972         m_client->didCompleteWithError(error);
973     else {
974         auto* download = NetworkProcess::singleton().downloadManager().download(m_pendingDownloadID);
975         ASSERT(download);
976         download->didFail(error, IPC::DataReference());
977     }
978 }
979
980 void NetworkDataTaskSoup::cleanDownloadFiles()
981 {
982     if (m_downloadDestinationFile) {
983         g_file_delete(m_downloadDestinationFile.get(), nullptr, nullptr);
984         m_downloadDestinationFile = nullptr;
985     }
986     if (m_downloadIntermediateFile) {
987         g_file_delete(m_downloadIntermediateFile.get(), nullptr, nullptr);
988         m_downloadIntermediateFile = nullptr;
989     }
990 }
991
992 void NetworkDataTaskSoup::didFail(const ResourceError& error)
993 {
994     if (isDownload()) {
995         didFailDownload(platformDownloadNetworkError(error.errorCode(), error.failingURL(), error.localizedDescription()));
996         return;
997     }
998
999     clearRequest();
1000     ASSERT(m_client);
1001     m_client->didCompleteWithError(error);
1002 }
1003
1004 #if ENABLE(WEB_TIMING)
1005 void NetworkDataTaskSoup::networkEventCallback(SoupMessage* soupMessage, GSocketClientEvent event, GIOStream*, NetworkDataTaskSoup* task)
1006 {
1007     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client)
1008         return;
1009
1010     ASSERT(task->m_soupMessage.get() == soupMessage);
1011     task->networkEvent(event);
1012 }
1013
1014 void NetworkDataTaskSoup::networkEvent(GSocketClientEvent event)
1015 {
1016     double deltaTime = monotonicallyIncreasingTimeMS() - m_startTime;
1017     auto& loadTiming = m_response.networkLoadTiming();
1018     switch (event) {
1019     case G_SOCKET_CLIENT_RESOLVING:
1020         loadTiming.domainLookupStart = deltaTime;
1021         break;
1022     case G_SOCKET_CLIENT_RESOLVED:
1023         loadTiming.domainLookupEnd = deltaTime;
1024         break;
1025     case G_SOCKET_CLIENT_CONNECTING:
1026         loadTiming.connectStart = deltaTime;
1027         break;
1028     case G_SOCKET_CLIENT_CONNECTED:
1029         // Web Timing considers that connection time involves dns, proxy & TLS negotiation...
1030         // so we better pick G_SOCKET_CLIENT_COMPLETE for connectEnd
1031         break;
1032     case G_SOCKET_CLIENT_PROXY_NEGOTIATING:
1033         break;
1034     case G_SOCKET_CLIENT_PROXY_NEGOTIATED:
1035         break;
1036     case G_SOCKET_CLIENT_TLS_HANDSHAKING:
1037         loadTiming.secureConnectionStart = deltaTime;
1038         break;
1039     case G_SOCKET_CLIENT_TLS_HANDSHAKED:
1040         break;
1041     case G_SOCKET_CLIENT_COMPLETE:
1042         loadTiming.connectEnd = deltaTime;
1043         break;
1044     default:
1045         ASSERT_NOT_REACHED();
1046         break;
1047     }
1048 }
1049
1050 #if SOUP_CHECK_VERSION(2, 49, 91)
1051 void NetworkDataTaskSoup::startingCallback(SoupMessage* soupMessage, NetworkDataTaskSoup* task)
1052 {
1053     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client)
1054         return;
1055
1056     ASSERT(task->m_soupMessage.get() == soupMessage);
1057     task->didStartRequest();
1058 }
1059 #else
1060 void NetworkDataTaskSoup::requestStartedCallback(SoupSession* session, SoupMessage* soupMessage, SoupSocket*, NetworkDataTaskSoup* task)
1061 {
1062     ASSERT(session == static_cast<NetworkSessionSoup&>(task->m_session.get()).soupSession());
1063     if (soupMessage != task->m_soupMessage.get())
1064         return;
1065
1066     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client)
1067         return;
1068
1069     task->didStartRequest();
1070 }
1071 #endif
1072
1073 void NetworkDataTaskSoup::didStartRequest()
1074 {
1075     m_response.networkLoadTiming().requestStart = monotonicallyIncreasingTimeMS() - m_startTime;
1076 }
1077
1078 void NetworkDataTaskSoup::restartedCallback(SoupMessage* soupMessage, NetworkDataTaskSoup* task)
1079 {
1080     // Called each time the message is going to be sent again except the first time.
1081     // This happens when libsoup handles HTTP authentication.
1082     if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client)
1083         return;
1084
1085     ASSERT(task->m_soupMessage.get() == soupMessage);
1086     task->didRestart();
1087 }
1088
1089 void NetworkDataTaskSoup::didRestart()
1090 {
1091     m_startTime = monotonicallyIncreasingTimeMS();
1092 }
1093 #endif // ENABLE(WEB_TIMING)
1094
1095 } // namespace WebKit
1096