Fix some builds after r240292
[WebKit-https.git] / Source / WebCore / platform / network / curl / ResourceHandleCurl.cpp
1 /*
2  * Copyright (C) 2004, 2006 Apple Inc.  All rights reserved.
3  * Copyright (C) 2005, 2006 Michael Emmel mike.emmel@gmail.com
4  * Copyright (C) 2017 Sony Interactive Entertainment Inc.
5  * All rights reserved.
6  * Copyright (C) 2017 NAVER Corp.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
18  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
25  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  */
29
30 #include "config.h"
31 #include "ResourceHandle.h"
32
33 #if USE(CURL)
34
35 #include "CookieJar.h"
36 #include "CookieJarCurl.h"
37 #include "CredentialStorage.h"
38 #include "CurlCacheManager.h"
39 #include "CurlContext.h"
40 #include "CurlRequest.h"
41 #include "FileSystem.h"
42 #include "HTTPParsers.h"
43 #include "Logging.h"
44 #include "NetworkStorageSession.h"
45 #include "ResourceHandleInternal.h"
46 #include "SameSiteInfo.h"
47 #include "SharedBuffer.h"
48 #include "SynchronousLoaderClient.h"
49 #include "TextEncoding.h"
50 #include <wtf/CompletionHandler.h>
51 #include <wtf/text/Base64.h>
52
53 namespace WebCore {
54
55 ResourceHandleInternal::~ResourceHandleInternal()
56 {
57     if (m_curlRequest)
58         m_curlRequest->invalidateClient();
59 }
60
61 ResourceHandle::~ResourceHandle() = default;
62
63 bool ResourceHandle::start()
64 {
65     ASSERT(isMainThread());
66
67     CurlContext::singleton();
68
69     // The frame could be null if the ResourceHandle is not associated to any
70     // Frame, e.g. if we are downloading a file.
71     // If the frame is not null but the page is null this must be an attempted
72     // load from an unload handler, so let's just block it.
73     // If both the frame and the page are not null the context is valid.
74     if (d->m_context && !d->m_context->isValid())
75         return false;
76
77     // Only allow the POST and GET methods for non-HTTP requests.
78     auto request = firstRequest();
79     if (!request.url().protocolIsInHTTPFamily() && request.httpMethod() != "GET" && request.httpMethod() != "POST") {
80         scheduleFailure(InvalidURLFailure); // Error must not be reported immediately
81         return true;
82     }
83
84     d->m_startTime = MonotonicTime::now();
85
86     d->m_curlRequest = createCurlRequest(WTFMove(request));
87
88     if (auto credential = getCredential(d->m_firstRequest, false)) {
89         d->m_curlRequest->setUserPass(credential->user(), credential->password());
90         d->m_curlRequest->setAuthenticationScheme(ProtectionSpaceAuthenticationSchemeHTTPBasic);
91     }
92
93     d->m_curlRequest->setStartTime(d->m_startTime);
94     d->m_curlRequest->start();
95
96     return true;
97 }
98
99 void ResourceHandle::cancel()
100 {
101     ASSERT(isMainThread());
102
103     d->m_cancelled = true;
104
105     if (d->m_curlRequest)
106         d->m_curlRequest->cancel();
107 }
108
109 bool ResourceHandle::cancelledOrClientless()
110 {
111     if (d->m_cancelled)
112         return true;
113
114     return !client();
115 }
116
117 void ResourceHandle::addCacheValidationHeaders(ResourceRequest& request)
118 {
119     ASSERT(isMainThread());
120
121     d->m_addedCacheValidationHeaders = false;
122
123     auto hasCacheHeaders = request.httpHeaderFields().contains(HTTPHeaderName::IfModifiedSince) || request.httpHeaderFields().contains(HTTPHeaderName::IfNoneMatch);
124     if (hasCacheHeaders)
125         return;
126
127     auto& cache = CurlCacheManager::singleton();
128     URL cacheUrl = request.url();
129     cacheUrl.removeFragmentIdentifier();
130
131     if (cache.isCached(cacheUrl)) {
132         cache.addCacheEntryClient(cacheUrl, this);
133
134         for (const auto& entry : cache.requestHeaders(cacheUrl))
135             request.addHTTPHeaderField(entry.key, entry.value);
136
137         d->m_addedCacheValidationHeaders = true;
138     }
139 }
140
141 Ref<CurlRequest> ResourceHandle::createCurlRequest(ResourceRequest&& request, RequestStatus status)
142 {
143     ASSERT(isMainThread());
144
145     if (status == RequestStatus::NewRequest) {
146         addCacheValidationHeaders(request);
147
148         auto& storageSession = *d->m_context->storageSession();
149         auto& cookieJar = storageSession.cookieStorage();
150         auto includeSecureCookies = request.url().protocolIs("https") ? IncludeSecureCookies::Yes : IncludeSecureCookies::No;
151         String cookieHeaderField = cookieJar.cookieRequestHeaderFieldValue(storageSession, request.firstPartyForCookies(), SameSiteInfo::create(request), request.url(), WTF::nullopt, WTF::nullopt, includeSecureCookies).first;
152         if (!cookieHeaderField.isEmpty())
153             request.addHTTPHeaderField(HTTPHeaderName::Cookie, cookieHeaderField);
154     }
155
156     CurlRequest::ShouldSuspend shouldSuspend = d->m_defersLoading ? CurlRequest::ShouldSuspend::Yes : CurlRequest::ShouldSuspend::No;
157     auto curlRequest = CurlRequest::create(request, *delegate(), shouldSuspend, CurlRequest::EnableMultipart::Yes, CurlRequest::CaptureNetworkLoadMetrics::Basic, d->m_messageQueue);
158     
159     return curlRequest;
160 }
161
162 CurlResourceHandleDelegate* ResourceHandle::delegate()
163 {
164     if (!d->m_delegate)
165         d->m_delegate = std::make_unique<CurlResourceHandleDelegate>(*this);
166
167     return d->m_delegate.get();
168 }
169
170 #if OS(WINDOWS)
171
172 void ResourceHandle::setHostAllowsAnyHTTPSCertificate(const String& host)
173 {
174     ASSERT(isMainThread());
175
176     CurlContext::singleton().sslHandle().allowAnyHTTPSCertificatesForHost(host);
177 }
178
179 void ResourceHandle::setClientCertificateInfo(const String& host, const String& certificate, const String& key)
180 {
181     ASSERT(isMainThread());
182
183     if (FileSystem::fileExists(certificate))
184         CurlContext::singleton().sslHandle().setClientCertificateInfo(host, certificate, key);
185     else
186         LOG(Network, "Invalid client certificate file: %s!\n", certificate.latin1().data());
187 }
188
189 #endif
190
191 #if OS(WINDOWS) && USE(CF)
192
193 void ResourceHandle::setClientCertificate(const String&, CFDataRef)
194 {
195 }
196
197 #endif
198
199 void ResourceHandle::platformSetDefersLoading(bool defers)
200 {
201     ASSERT(isMainThread());
202
203     if (defers == d->m_defersLoading)
204         return;
205
206     d->m_defersLoading = defers;
207
208     if (!d->m_curlRequest)
209         return;
210
211     if (d->m_defersLoading)
212         d->m_curlRequest->suspend();
213     else
214         d->m_curlRequest->resume();
215 }
216
217 bool ResourceHandle::shouldUseCredentialStorage()
218 {
219     return (!client() || client()->shouldUseCredentialStorage(this)) && firstRequest().url().protocolIsInHTTPFamily();
220 }
221
222 void ResourceHandle::didReceiveAuthenticationChallenge(const AuthenticationChallenge& challenge)
223 {
224     ASSERT(isMainThread());
225
226     String partition = firstRequest().cachePartition();
227
228     if (!d->m_user.isNull() && !d->m_pass.isNull()) {
229         Credential credential(d->m_user, d->m_pass, CredentialPersistenceNone);
230
231         URL urlToStore;
232         if (challenge.failureResponse().httpStatusCode() == 401)
233             urlToStore = challenge.failureResponse().url();
234         d->m_context->storageSession()->credentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
235
236         restartRequestWithCredential(challenge.protectionSpace(), credential);
237
238         d->m_user = String();
239         d->m_pass = String();
240         // FIXME: Per the specification, the user shouldn't be asked for credentials if there were incorrect ones provided explicitly.
241         return;
242     }
243
244     if (shouldUseCredentialStorage()) {
245         if (!d->m_initialCredential.isEmpty() || challenge.previousFailureCount()) {
246             // The stored credential wasn't accepted, stop using it.
247             // There is a race condition here, since a different credential might have already been stored by another ResourceHandle,
248             // but the observable effect should be very minor, if any.
249             d->m_context->storageSession()->credentialStorage().remove(partition, challenge.protectionSpace());
250         }
251
252         if (!challenge.previousFailureCount()) {
253             Credential credential = d->m_context->storageSession()->credentialStorage().get(partition, challenge.protectionSpace());
254             if (!credential.isEmpty() && credential != d->m_initialCredential) {
255                 ASSERT(credential.persistence() == CredentialPersistenceNone);
256                 if (challenge.failureResponse().httpStatusCode() == 401) {
257                     // Store the credential back, possibly adding it as a default for this directory.
258                     d->m_context->storageSession()->credentialStorage().set(partition, credential, challenge.protectionSpace(), challenge.failureResponse().url());
259                 }
260
261                 restartRequestWithCredential(challenge.protectionSpace(), credential);
262                 return;
263             }
264         }
265     }
266
267     d->m_currentWebChallenge = challenge;
268
269     if (client()) {
270         auto protectedThis = makeRef(*this);
271         client()->didReceiveAuthenticationChallenge(this, d->m_currentWebChallenge);
272     }
273 }
274
275 void ResourceHandle::receivedCredential(const AuthenticationChallenge& challenge, const Credential& credential)
276 {
277     ASSERT(isMainThread());
278
279     if (challenge != d->m_currentWebChallenge)
280         return;
281
282     if (credential.isEmpty()) {
283         receivedRequestToContinueWithoutCredential(challenge);
284         return;
285     }
286
287     String partition = firstRequest().cachePartition();
288
289     if (shouldUseCredentialStorage()) {
290         if (challenge.failureResponse().httpStatusCode() == 401) {
291             URL urlToStore = challenge.failureResponse().url();
292             d->m_context->storageSession()->credentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
293         }
294     }
295
296     restartRequestWithCredential(challenge.protectionSpace(), credential);
297
298     clearAuthentication();
299 }
300
301 void ResourceHandle::receivedRequestToContinueWithoutCredential(const AuthenticationChallenge& challenge)
302 {
303     ASSERT(isMainThread());
304
305     if (challenge != d->m_currentWebChallenge)
306         return;
307
308     clearAuthentication();
309
310     didReceiveResponse(ResourceResponse(delegate()->response()), [this, protectedThis = makeRef(*this)] {
311         continueAfterDidReceiveResponse();
312     });
313 }
314
315 void ResourceHandle::receivedCancellation(const AuthenticationChallenge& challenge)
316 {
317     ASSERT(isMainThread());
318
319     if (challenge != d->m_currentWebChallenge)
320         return;
321
322     if (client()) {
323         auto protectedThis = makeRef(*this);
324         client()->receivedCancellation(this, challenge);
325     }
326 }
327
328 void ResourceHandle::receivedRequestToPerformDefaultHandling(const AuthenticationChallenge&)
329 {
330     ASSERT_NOT_REACHED();
331 }
332
333 void ResourceHandle::receivedChallengeRejection(const AuthenticationChallenge&)
334 {
335     ASSERT_NOT_REACHED();
336 }
337
338 Optional<Credential> ResourceHandle::getCredential(const ResourceRequest& request, bool redirect)
339 {
340     // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
341     Credential credential { d->m_user, d->m_pass, CredentialPersistenceNone };
342
343     if (shouldUseCredentialStorage()) {
344         String partition = request.cachePartition();
345
346         if (credential.isEmpty()) {
347             // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 
348             // try and reuse the credential preemptively, as allowed by RFC 2617.
349             d->m_initialCredential = d->m_context->storageSession()->credentialStorage().get(partition, request.url());
350         } else if (!redirect) {
351             // If there is already a protection space known for the URL, update stored credentials
352             // before sending a request. This makes it possible to implement logout by sending an
353             // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
354             // an authentication dialog doesn't pop up).
355             d->m_context->storageSession()->credentialStorage().set(partition, credential, request.url());
356         }
357     }
358
359     if (!d->m_initialCredential.isEmpty())
360         return d->m_initialCredential;
361
362     return WTF::nullopt;
363 }
364
365 void ResourceHandle::restartRequestWithCredential(const ProtectionSpace& protectionSpace, const Credential& credential)
366 {
367     ASSERT(isMainThread());
368
369     if (!d->m_curlRequest)
370         return;
371     
372     auto previousRequest = d->m_curlRequest->resourceRequest();
373     d->m_curlRequest->cancel();
374
375     d->m_curlRequest = createCurlRequest(WTFMove(previousRequest), RequestStatus::ReusedRequest);
376     d->m_curlRequest->setAuthenticationScheme(protectionSpace.authenticationScheme());
377     d->m_curlRequest->setUserPass(credential.user(), credential.password());
378     d->m_curlRequest->setStartTime(d->m_startTime);
379     d->m_curlRequest->start();
380 }
381
382 void ResourceHandle::platformLoadResourceSynchronously(NetworkingContext* context, const ResourceRequest& request, StoredCredentialsPolicy storedCredentialsPolicy, ResourceError& error, ResourceResponse& response, Vector<char>& data)
383 {
384     ASSERT(isMainThread());
385     ASSERT(!request.isEmpty());
386
387     SynchronousLoaderClient client;
388     client.setAllowStoredCredentials(storedCredentialsPolicy == StoredCredentialsPolicy::Use);
389
390     bool defersLoading = false;
391     bool shouldContentSniff = true;
392     bool shouldContentEncodingSniff = true;
393     RefPtr<ResourceHandle> handle = adoptRef(new ResourceHandle(context, request, &client, defersLoading, shouldContentSniff, shouldContentEncodingSniff));
394     handle->d->m_messageQueue = &client.messageQueue();
395     handle->d->m_startTime = MonotonicTime::now();
396
397     if (request.url().protocolIsData()) {
398         handle->handleDataURL();
399         return;
400     }
401
402     auto requestCopy = handle->firstRequest();
403     handle->d->m_curlRequest = handle->createCurlRequest(WTFMove(requestCopy));
404
405     if (auto credential = handle->getCredential(handle->d->m_firstRequest, false)) {
406         handle->d->m_curlRequest->setUserPass(credential->user(), credential->password());
407         handle->d->m_curlRequest->setAuthenticationScheme(ProtectionSpaceAuthenticationSchemeHTTPBasic);
408     }
409
410     handle->d->m_curlRequest->setStartTime(handle->d->m_startTime);
411     handle->d->m_curlRequest->start();
412
413     do {
414         if (auto task = client.messageQueue().waitForMessage())
415             (*task)();
416     } while (!client.messageQueue().killed() && !handle->cancelledOrClientless());
417
418     error = client.error();
419     data.swap(client.mutableData());
420     response = client.response();
421 }
422
423 void ResourceHandle::platformContinueSynchronousDidReceiveResponse()
424 {
425     ASSERT(isMainThread());
426
427     continueAfterDidReceiveResponse();
428 }
429
430 void ResourceHandle::continueAfterDidReceiveResponse()
431 {
432     ASSERT(isMainThread());
433
434     // continueDidReceiveResponse might cancel the load.
435     if (cancelledOrClientless() || !d->m_curlRequest)
436         return;
437
438     d->m_curlRequest->completeDidReceiveResponse();
439 }
440
441 bool ResourceHandle::shouldRedirectAsGET(const ResourceRequest& request, bool crossOrigin)
442 {
443     if (request.httpMethod() == "GET" || request.httpMethod() == "HEAD")
444         return false;
445
446     if (!request.url().protocolIsInHTTPFamily())
447         return true;
448
449     if (delegate()->response().isSeeOther())
450         return true;
451
452     if ((delegate()->response().isMovedPermanently() || delegate()->response().isFound()) && (request.httpMethod() == "POST"))
453         return true;
454
455     if (crossOrigin && (request.httpMethod() == "DELETE"))
456         return true;
457
458     return false;
459 }
460
461 void ResourceHandle::willSendRequest()
462 {
463     ASSERT(isMainThread());
464
465     static const int maxRedirects = 20;
466
467     if (d->m_redirectCount++ > maxRedirects) {
468         client()->didFail(this, ResourceError::httpError(CURLE_TOO_MANY_REDIRECTS, delegate()->response().url()));
469         return;
470     }
471
472     String location = delegate()->response().httpHeaderField(HTTPHeaderName::Location);
473     URL newURL = URL(delegate()->response().url(), location);
474     bool crossOrigin = !protocolHostAndPortAreEqual(d->m_firstRequest.url(), newURL);
475
476     ResourceRequest newRequest = d->m_firstRequest;
477     newRequest.setURL(newURL);
478
479     if (shouldRedirectAsGET(newRequest, crossOrigin)) {
480         newRequest.setHTTPMethod("GET");
481         newRequest.setHTTPBody(nullptr);
482         newRequest.clearHTTPContentType();
483     }
484
485     // Should not set Referer after a redirect from a secure resource to non-secure one.
486     if (!newURL.protocolIs("https") && protocolIs(newRequest.httpReferrer(), "https") && context()->shouldClearReferrerOnHTTPSToHTTPRedirect())
487         newRequest.clearHTTPReferrer();
488
489     d->m_user = newURL.user();
490     d->m_pass = newURL.pass();
491     newRequest.removeCredentials();
492
493     if (crossOrigin) {
494         // If the network layer carries over authentication headers from the original request
495         // in a cross-origin redirect, we want to clear those headers here. 
496         newRequest.clearHTTPAuthorization();
497         newRequest.clearHTTPOrigin();
498         d->m_startTime = WTF::MonotonicTime::now();
499     }
500
501     ResourceResponse responseCopy = delegate()->response();
502     client()->willSendRequestAsync(this, WTFMove(newRequest), WTFMove(responseCopy), [this, protectedThis = makeRef(*this)] (ResourceRequest&& request) {
503         continueAfterWillSendRequest(WTFMove(request));
504     });
505 }
506
507 void ResourceHandle::continueAfterWillSendRequest(ResourceRequest&& request)
508 {
509     ASSERT(isMainThread());
510
511     // willSendRequest might cancel the load.
512     if (cancelledOrClientless() || !d->m_curlRequest)
513         return;
514
515     if (request.isNull()) {
516         cancel();
517         return;
518     }
519
520     auto shouldForwardCredential = protocolHostAndPortAreEqual(request.url(), delegate()->response().url());
521     auto credential = getCredential(request, true);
522
523     d->m_curlRequest->cancel();
524     d->m_curlRequest = createCurlRequest(WTFMove(request));
525
526     if (shouldForwardCredential && credential)
527         d->m_curlRequest->setUserPass(credential->user(), credential->password());
528
529     d->m_curlRequest->setStartTime(d->m_startTime);
530     d->m_curlRequest->start();
531 }
532
533 void ResourceHandle::handleDataURL()
534 {
535     ASSERT(d->m_firstRequest.url().protocolIsData());
536     String url = d->m_firstRequest.url().string();
537
538     ASSERT(client());
539
540     auto index = url.find(',');
541     if (index == notFound) {
542         client()->cannotShowURL(this);
543         return;
544     }
545
546     String mediaType = url.substring(5, index - 5);
547     String data = url.substring(index + 1);
548     auto originalSize = data.length();
549
550     bool base64 = mediaType.endsWithIgnoringASCIICase(";base64");
551     if (base64)
552         mediaType = mediaType.left(mediaType.length() - 7);
553
554     if (mediaType.isEmpty())
555         mediaType = "text/plain"_s;
556
557     String mimeType = extractMIMETypeFromMediaType(mediaType);
558     String charset = extractCharsetFromMediaType(mediaType);
559
560     if (charset.isEmpty())
561         charset = "US-ASCII"_s;
562
563     ResourceResponse response;
564     response.setMimeType(mimeType);
565     response.setTextEncodingName(charset);
566     response.setURL(d->m_firstRequest.url());
567
568     if (base64) {
569         data = decodeURLEscapeSequences(data);
570         didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
571             continueAfterDidReceiveResponse();
572         });
573
574         // didReceiveResponse might cause the client to be deleted.
575         if (client()) {
576             Vector<char> out;
577             if (base64Decode(data, out, Base64IgnoreSpacesAndNewLines) && out.size() > 0)
578                 client()->didReceiveBuffer(this, SharedBuffer::create(out.data(), out.size()), originalSize);
579         }
580     } else {
581         TextEncoding encoding(charset);
582         data = decodeURLEscapeSequences(data, encoding);
583         didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
584             continueAfterDidReceiveResponse();
585         });
586
587         // didReceiveResponse might cause the client to be deleted.
588         if (client()) {
589             auto encodedData = encoding.encode(data, UnencodableHandling::URLEncodedEntities);
590             if (encodedData.size())
591                 client()->didReceiveBuffer(this, SharedBuffer::create(WTFMove(encodedData)), originalSize);
592         }
593     }
594
595     if (client())
596         client()->didFinishLoading(this);
597 }
598
599 } // namespace WebCore
600
601 #endif