[Curl] Surface additional NetworkLoadMetrics
[WebKit-https.git] / Source / WebKit / NetworkProcess / curl / NetworkDataTaskCurl.cpp
1 /*
2  * Copyright (C) 2018 Sony Interactive Entertainment Inc.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkDataTaskCurl.h"
28
29 #include "AuthenticationManager.h"
30 #include "NetworkSessionCurl.h"
31 #include <WebCore/AuthenticationChallenge.h>
32 #include <WebCore/CookiesStrategy.h>
33 #include <WebCore/CurlRequest.h>
34 #include <WebCore/NetworkStorageSession.h>
35 #include <WebCore/NotImplemented.h>
36 #include <WebCore/ResourceError.h>
37 #include <WebCore/SameSiteInfo.h>
38
39 using namespace WebCore;
40
41 namespace WebKit {
42
43 NetworkDataTaskCurl::NetworkDataTaskCurl(NetworkSession& session, NetworkDataTaskClient& client, const ResourceRequest& requestWithCredentials, StoredCredentialsPolicy storedCredentialsPolicy, ContentSniffingPolicy shouldContentSniff, ContentEncodingSniffingPolicy, bool shouldClearReferrerOnHTTPSToHTTPRedirect, bool dataTaskIsForMainFrameNavigation)
44     : NetworkDataTask(session, client, requestWithCredentials, storedCredentialsPolicy, shouldClearReferrerOnHTTPSToHTTPRedirect, dataTaskIsForMainFrameNavigation)
45 {
46     if (m_scheduledFailureType != NoFailure)
47         return;
48
49     auto request = requestWithCredentials;
50     if (request.url().protocolIsInHTTPFamily()) {
51         if (m_storedCredentialsPolicy == StoredCredentialsPolicy::Use) {
52             auto url = request.url();
53             m_user = url.user();
54             m_password = url.pass();
55             request.removeCredentials();
56
57             if (m_user.isEmpty() && m_password.isEmpty())
58                 m_initialCredential = m_session->networkStorageSession().credentialStorage().get(m_partition, request.url());
59             else
60                 m_session->networkStorageSession().credentialStorage().set(m_partition, Credential(m_user, m_password, CredentialPersistenceNone), request.url());
61         }
62     }
63
64     m_curlRequest = createCurlRequest(request, ShouldPreprocess::Yes);
65     if (!m_initialCredential.isEmpty())
66         m_curlRequest->setUserPass(m_initialCredential.user(), m_initialCredential.password());
67     m_curlRequest->start();
68 }
69
70 NetworkDataTaskCurl::~NetworkDataTaskCurl()
71 {
72     if (m_curlRequest)
73         m_curlRequest->invalidateClient();
74 }
75
76 void NetworkDataTaskCurl::resume()
77 {
78     ASSERT(m_state != State::Running);
79     if (m_state == State::Canceling || m_state == State::Completed)
80         return;
81
82     m_state = State::Running;
83
84     if (m_scheduledFailureType != NoFailure) {
85         ASSERT(m_failureTimer.isActive());
86         return;
87     }
88
89     if (m_curlRequest)
90         m_curlRequest->resume();
91 }
92
93 void NetworkDataTaskCurl::suspend()
94 {
95     ASSERT(m_state != State::Suspended);
96     if (m_state == State::Canceling || m_state == State::Completed)
97         return;
98
99     m_state = State::Suspended;
100
101     if (m_curlRequest)
102         m_curlRequest->suspend();
103 }
104
105 void NetworkDataTaskCurl::cancel()
106 {
107     if (m_state == State::Canceling || m_state == State::Completed)
108         return;
109
110     m_state = State::Canceling;
111
112     if (m_curlRequest)
113         m_curlRequest->cancel();
114 }
115
116 void NetworkDataTaskCurl::invalidateAndCancel()
117 {
118     cancel();
119
120     if (m_curlRequest)
121         m_curlRequest->invalidateClient();
122 }
123
124 NetworkDataTask::State NetworkDataTaskCurl::state() const
125 {
126     return m_state;
127 }
128
129 Ref<CurlRequest> NetworkDataTaskCurl::createCurlRequest(const ResourceRequest& request, ShouldPreprocess shouldPreprocess)
130 {
131     m_currentRequest = request;
132
133     if (shouldPreprocess == ShouldPreprocess::Yes)
134         appendCookieHeader(m_currentRequest);
135
136     // Creates a CurlRequest in suspended state.
137     // Then, NetworkDataTaskCurl::resume() will be called and communication resumes.
138     return CurlRequest::create(m_currentRequest, *this, CurlRequest::ShouldSuspend::Yes);
139 }
140
141 void NetworkDataTaskCurl::curlDidSendData(CurlRequest&, unsigned long long totalBytesSent, unsigned long long totalBytesExpectedToSend)
142 {
143     auto protectedThis = makeRef(*this);
144     if (state() == State::Canceling || state() == State::Completed || !m_client)
145         return;
146
147     m_client->didSendData(totalBytesSent, totalBytesExpectedToSend);
148 }
149
150 void NetworkDataTaskCurl::curlDidReceiveResponse(CurlRequest& request, const CurlResponse& receivedResponse)
151 {
152     auto protectedThis = makeRef(*this);
153     if (state() == State::Canceling || state() == State::Completed || !m_client)
154         return;
155
156     m_response = ResourceResponse(receivedResponse);
157     m_response.setDeprecatedNetworkLoadMetrics(request.networkLoadMetrics().isolatedCopy());
158
159     handleCookieHeaders(receivedResponse);
160
161     if (m_response.shouldRedirect()) {
162         willPerformHTTPRedirection();
163         return;
164     }
165
166     if (m_response.isUnauthorized()) {
167         tryHttpAuthentication(AuthenticationChallenge(receivedResponse, m_authFailureCount, m_response));
168         m_authFailureCount++;
169         return;
170     }
171
172     didReceiveResponse(ResourceResponse(m_response), [this, protectedThis = makeRef(*this)](PolicyAction policyAction) {
173         if (m_state == State::Canceling || m_state == State::Completed)
174             return;
175
176         switch (policyAction) {
177         case PolicyAction::Use:
178             if (m_curlRequest)
179                 m_curlRequest->completeDidReceiveResponse();
180             break;
181         case PolicyAction::Ignore:
182             break;
183         case PolicyAction::Download:
184             notImplemented();
185             break;
186         }
187     });
188 }
189
190 void NetworkDataTaskCurl::curlDidReceiveBuffer(CurlRequest&, Ref<SharedBuffer>&& buffer)
191 {
192     auto protectedThis = makeRef(*this);
193     if (state() == State::Canceling || state() == State::Completed || (!m_client && !isDownload()))
194         return;
195
196     m_client->didReceiveData(WTFMove(buffer));
197 }
198
199 void NetworkDataTaskCurl::curlDidComplete(CurlRequest& request)
200 {
201     if (state() == State::Canceling || state() == State::Completed || (!m_client && !isDownload()))
202         return;
203
204     m_response.setDeprecatedNetworkLoadMetrics(request.networkLoadMetrics().isolatedCopy());
205
206     m_client->didCompleteWithError({ }, m_response.deprecatedNetworkLoadMetrics());
207 }
208
209 void NetworkDataTaskCurl::curlDidFailWithError(CurlRequest&, const ResourceError& resourceError)
210 {
211     if (state() == State::Canceling || state() == State::Completed || (!m_client && !isDownload()))
212         return;
213
214     m_client->didCompleteWithError(resourceError);
215 }
216
217 bool NetworkDataTaskCurl::shouldRedirectAsGET(const ResourceRequest& request, bool crossOrigin)
218 {
219     if (request.httpMethod() == "GET" || request.httpMethod() == "HEAD")
220         return false;
221
222     if (!request.url().protocolIsInHTTPFamily())
223         return true;
224
225     if (m_response.isSeeOther())
226         return true;
227
228     if ((m_response.isMovedPermanently() || m_response.isFound()) && (request.httpMethod() == "POST"))
229         return true;
230
231     if (crossOrigin && (request.httpMethod() == "DELETE"))
232         return true;
233
234     return false;
235 }
236
237 void NetworkDataTaskCurl::willPerformHTTPRedirection()
238 {
239     static const int maxRedirects = 20;
240
241     if (m_redirectCount++ > maxRedirects) {
242         m_client->didCompleteWithError(ResourceError::httpError(CURLE_TOO_MANY_REDIRECTS, m_response.url()));
243         return;
244     }
245
246     ResourceRequest request = m_currentRequest;
247     URL redirectedURL = URL(m_response.url(), m_response.httpHeaderField(HTTPHeaderName::Location));
248     if (!redirectedURL.hasFragmentIdentifier() && request.url().hasFragmentIdentifier())
249         redirectedURL.setFragmentIdentifier(request.url().fragmentIdentifier());
250     request.setURL(redirectedURL);
251
252     // Should not set Referer after a redirect from a secure resource to non-secure one.
253     if (m_shouldClearReferrerOnHTTPSToHTTPRedirect && !request.url().protocolIs("https") && protocolIs(request.httpReferrer(), "https"))
254         request.clearHTTPReferrer();
255
256     bool isCrossOrigin = !protocolHostAndPortAreEqual(m_currentRequest.url(), request.url());
257     if (!equalLettersIgnoringASCIICase(request.httpMethod(), "get")) {
258         // Change request method to GET if change was made during a previous redirection or if current redirection says so.
259         if (!request.url().protocolIsInHTTPFamily() || shouldRedirectAsGET(request, isCrossOrigin)) {
260             request.setHTTPMethod("GET");
261             request.setHTTPBody(nullptr);
262             request.clearHTTPContentType();
263         }
264     }
265
266     bool didChangeCredential = false;
267     const auto& url = request.url();
268     m_user = url.user();
269     m_password = url.pass();
270     m_lastHTTPMethod = request.httpMethod();
271     request.removeCredentials();
272
273     if (isCrossOrigin) {
274         // The network layer might carry over some headers from the original request that
275         // we want to strip here because the redirect is cross-origin.
276         request.clearHTTPAuthorization();
277         request.clearHTTPOrigin();
278     } else if (m_storedCredentialsPolicy == StoredCredentialsPolicy::Use) {
279         // Only consider applying authentication credentials if this is actually a redirect and the redirect
280         // URL didn't include credentials of its own.
281         if (m_user.isEmpty() && m_password.isEmpty()) {
282             auto credential = m_session->networkStorageSession().credentialStorage().get(m_partition, request.url());
283             if (!credential.isEmpty()) {
284                 m_initialCredential = credential;
285                 didChangeCredential = true;
286             }
287         }
288     }
289
290     auto response = ResourceResponse(m_response);
291     m_client->willPerformHTTPRedirection(WTFMove(response), WTFMove(request), [this, protectedThis = makeRef(*this), didChangeCredential](const ResourceRequest& newRequest) {
292         if (newRequest.isNull() || m_state == State::Canceling)
293             return;
294
295         if (m_curlRequest)
296             m_curlRequest->cancel();
297
298         m_curlRequest = createCurlRequest(newRequest, ShouldPreprocess::Yes);
299         if (didChangeCredential && !m_initialCredential.isEmpty())
300             m_curlRequest->setUserPass(m_initialCredential.user(), m_initialCredential.password());
301         m_curlRequest->start();
302
303         if (m_state != State::Suspended) {
304             m_state = State::Suspended;
305             resume();
306         }
307     });
308 }
309
310 void NetworkDataTaskCurl::tryHttpAuthentication(const AuthenticationChallenge& challenge)
311 {
312     if (!m_user.isNull() && !m_password.isNull()) {
313         auto persistence = m_storedCredentialsPolicy == WebCore::StoredCredentialsPolicy::Use ? WebCore::CredentialPersistenceForSession : WebCore::CredentialPersistenceNone;
314         restartWithCredential(Credential(m_user, m_password, persistence));
315         m_user = String();
316         m_password = String();
317         return;
318     }
319
320     if (m_storedCredentialsPolicy == StoredCredentialsPolicy::Use) {
321         if (!m_initialCredential.isEmpty() || challenge.previousFailureCount()) {
322             // The stored credential wasn't accepted, stop using it. There is a race condition
323             // here, since a different credential might have already been stored by another
324             // NetworkDataTask, but the observable effect should be very minor, if any.
325             m_session->networkStorageSession().credentialStorage().remove(m_partition, challenge.protectionSpace());
326         }
327
328         if (!challenge.previousFailureCount()) {
329             auto credential = m_session->networkStorageSession().credentialStorage().get(m_partition, challenge.protectionSpace());
330             if (!credential.isEmpty() && credential != m_initialCredential) {
331                 ASSERT(credential.persistence() == CredentialPersistenceNone);
332                 if (challenge.failureResponse().isUnauthorized()) {
333                     // Store the credential back, possibly adding it as a default for this directory.
334                     m_session->networkStorageSession().credentialStorage().set(m_partition, credential, challenge.protectionSpace(), challenge.failureResponse().url());
335                 }
336                 restartWithCredential(credential);
337                 return;
338             }
339         }
340     }
341
342     m_client->didReceiveChallenge(challenge, [this, protectedThis = makeRef(*this), challenge](AuthenticationChallengeDisposition disposition, const Credential& credential) {
343         if (m_state == State::Canceling || m_state == State::Completed)
344             return;
345
346         if (disposition == AuthenticationChallengeDisposition::Cancel) {
347             cancel();
348             m_client->didCompleteWithError(ResourceError::httpError(CURLE_COULDNT_RESOLVE_HOST, m_response.url()));
349             return;
350         }
351
352         if (disposition == AuthenticationChallengeDisposition::UseCredential && !credential.isEmpty()) {
353             if (m_storedCredentialsPolicy == StoredCredentialsPolicy::Use) {
354                 if (credential.persistence() == CredentialPersistenceForSession || credential.persistence() == CredentialPersistencePermanent)
355                     m_session->networkStorageSession().credentialStorage().set(m_partition, credential, challenge.protectionSpace(), challenge.failureResponse().url());
356             }
357         }
358
359         restartWithCredential(credential);
360     });
361 }
362
363 void NetworkDataTaskCurl::restartWithCredential(const Credential& credential)
364 {
365     if (m_curlRequest)
366         m_curlRequest->cancel();
367
368     m_curlRequest = createCurlRequest(m_currentRequest, ShouldPreprocess::No);
369     if (!credential.isEmpty())
370         m_curlRequest->setUserPass(credential.user(), credential.password());
371     m_curlRequest->start();
372
373     if (m_state != State::Suspended) {
374         m_state = State::Suspended;
375         resume();
376     }
377 }
378
379 void NetworkDataTaskCurl::appendCookieHeader(WebCore::ResourceRequest& request)
380 {
381     const auto& storageSession = m_session->networkStorageSession();
382     const auto& cookieJar = storageSession.cookieStorage();
383     auto includeSecureCookies = request.url().protocolIs("https") ? IncludeSecureCookies::Yes : IncludeSecureCookies::No;
384     auto cookieHeaderField = cookieJar.cookieRequestHeaderFieldValue(storageSession, request.firstPartyForCookies(), WebCore::SameSiteInfo::create(request), request.url(), std::nullopt, std::nullopt, includeSecureCookies).first;
385     if (!cookieHeaderField.isEmpty())
386         request.addHTTPHeaderField(HTTPHeaderName::Cookie, cookieHeaderField);
387 }
388
389 void NetworkDataTaskCurl::handleCookieHeaders(const CurlResponse& response)
390 {
391     static const auto setCookieHeader = "set-cookie: ";
392
393     const auto& storageSession = m_session->networkStorageSession();
394     const auto& cookieJar = storageSession.cookieStorage();
395     for (auto header : response.headers) {
396         if (header.startsWithIgnoringASCIICase(setCookieHeader)) {
397             String setCookieString = header.right(header.length() - strlen(setCookieHeader));
398             cookieJar.setCookiesFromHTTPResponse(storageSession, response.url, setCookieString);
399         }
400     }
401 }
402
403 } // namespace WebKit