[WTF] Add makeUnique<T>, which ensures T is fast-allocated, makeUnique / makeUniqueWi...
[WebKit-https.git] / Source / WebCore / platform / network / curl / ResourceHandleCurl.cpp
1 /*
2  * Copyright (C) 2004, 2006 Apple Inc.  All rights reserved.
3  * Copyright (C) 2005, 2006 Michael Emmel mike.emmel@gmail.com
4  * Copyright (C) 2017 Sony Interactive Entertainment Inc.
5  * All rights reserved.
6  * Copyright (C) 2017 NAVER Corp.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
18  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
25  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  */
29
30 #include "config.h"
31 #include "ResourceHandle.h"
32
33 #if USE(CURL)
34
35 #include "CookieJar.h"
36 #include "CookieJarCurl.h"
37 #include "CredentialStorage.h"
38 #include "CurlCacheManager.h"
39 #include "CurlContext.h"
40 #include "CurlRequest.h"
41 #include "HTTPParsers.h"
42 #include "Logging.h"
43 #include "NetworkStorageSession.h"
44 #include "ResourceHandleInternal.h"
45 #include "SameSiteInfo.h"
46 #include "SharedBuffer.h"
47 #include "SynchronousLoaderClient.h"
48 #include "TextEncoding.h"
49 #include <wtf/CompletionHandler.h>
50 #include <wtf/FileSystem.h>
51 #include <wtf/text/Base64.h>
52
53 namespace WebCore {
54
55 ResourceHandleInternal::~ResourceHandleInternal()
56 {
57     if (m_curlRequest)
58         m_curlRequest->invalidateClient();
59 }
60
61 ResourceHandle::~ResourceHandle() = default;
62
63 bool ResourceHandle::start()
64 {
65     ASSERT(isMainThread());
66
67     CurlContext::singleton();
68
69     // The frame could be null if the ResourceHandle is not associated to any
70     // Frame, e.g. if we are downloading a file.
71     // If the frame is not null but the page is null this must be an attempted
72     // load from an unload handler, so let's just block it.
73     // If both the frame and the page are not null the context is valid.
74     if (d->m_context && !d->m_context->isValid())
75         return false;
76
77     // Only allow the POST and GET methods for non-HTTP requests.
78     auto request = firstRequest();
79     if (!request.url().protocolIsInHTTPFamily() && request.httpMethod() != "GET" && request.httpMethod() != "POST") {
80         scheduleFailure(InvalidURLFailure); // Error must not be reported immediately
81         return true;
82     }
83
84     d->m_startTime = MonotonicTime::now();
85
86     d->m_curlRequest = createCurlRequest(WTFMove(request));
87
88     if (auto credential = getCredential(d->m_firstRequest, false)) {
89         d->m_curlRequest->setUserPass(credential->user(), credential->password());
90         d->m_curlRequest->setAuthenticationScheme(ProtectionSpaceAuthenticationSchemeHTTPBasic);
91     }
92
93     d->m_curlRequest->setStartTime(d->m_startTime);
94     d->m_curlRequest->start();
95
96     return true;
97 }
98
99 void ResourceHandle::cancel()
100 {
101     ASSERT(isMainThread());
102
103     d->m_cancelled = true;
104
105     if (d->m_curlRequest)
106         d->m_curlRequest->cancel();
107 }
108
109 bool ResourceHandle::cancelledOrClientless()
110 {
111     if (d->m_cancelled)
112         return true;
113
114     return !client();
115 }
116
117 void ResourceHandle::addCacheValidationHeaders(ResourceRequest& request)
118 {
119     ASSERT(isMainThread());
120
121     d->m_addedCacheValidationHeaders = false;
122
123     auto hasCacheHeaders = request.httpHeaderFields().contains(HTTPHeaderName::IfModifiedSince) || request.httpHeaderFields().contains(HTTPHeaderName::IfNoneMatch);
124     if (hasCacheHeaders)
125         return;
126
127     auto& cache = CurlCacheManager::singleton();
128     URL cacheUrl = request.url();
129     cacheUrl.removeFragmentIdentifier();
130
131     if (cache.isCached(cacheUrl)) {
132         cache.addCacheEntryClient(cacheUrl, this);
133
134         for (const auto& entry : cache.requestHeaders(cacheUrl))
135             request.addHTTPHeaderField(entry.key, entry.value);
136
137         d->m_addedCacheValidationHeaders = true;
138     }
139 }
140
141 Ref<CurlRequest> ResourceHandle::createCurlRequest(ResourceRequest&& request, RequestStatus status)
142 {
143     ASSERT(isMainThread());
144
145     if (status == RequestStatus::NewRequest) {
146         addCacheValidationHeaders(request);
147
148         auto& storageSession = *d->m_context->storageSession();
149         auto& cookieJar = storageSession.cookieStorage();
150         auto includeSecureCookies = request.url().protocolIs("https") ? IncludeSecureCookies::Yes : IncludeSecureCookies::No;
151         String cookieHeaderField = cookieJar.cookieRequestHeaderFieldValue(storageSession, request.firstPartyForCookies(), SameSiteInfo::create(request), request.url(), WTF::nullopt, WTF::nullopt, includeSecureCookies).first;
152         if (!cookieHeaderField.isEmpty())
153             request.addHTTPHeaderField(HTTPHeaderName::Cookie, cookieHeaderField);
154     }
155
156     CurlRequest::ShouldSuspend shouldSuspend = d->m_defersLoading ? CurlRequest::ShouldSuspend::Yes : CurlRequest::ShouldSuspend::No;
157     // FIXME: Use a correct sessionID.
158     auto curlRequest = CurlRequest::create(request, *delegate(), PAL::SessionID::emptySessionID(), shouldSuspend, CurlRequest::EnableMultipart::Yes, CurlRequest::CaptureNetworkLoadMetrics::Basic, d->m_messageQueue);
159     
160     return curlRequest;
161 }
162
163 CurlResourceHandleDelegate* ResourceHandle::delegate()
164 {
165     if (!d->m_delegate)
166         d->m_delegate = makeUnique<CurlResourceHandleDelegate>(*this);
167
168     return d->m_delegate.get();
169 }
170
171 #if OS(WINDOWS)
172
173 void ResourceHandle::setHostAllowsAnyHTTPSCertificate(const String& host)
174 {
175     ASSERT(isMainThread());
176
177     CurlContext::singleton().sslHandle().allowAnyHTTPSCertificatesForHost(host);
178 }
179
180 void ResourceHandle::setClientCertificateInfo(const String& host, const String& certificate, const String& key)
181 {
182     ASSERT(isMainThread());
183
184     if (FileSystem::fileExists(certificate))
185         CurlContext::singleton().sslHandle().setClientCertificateInfo(host, certificate, key);
186     else
187         LOG(Network, "Invalid client certificate file: %s!\n", certificate.latin1().data());
188 }
189
190 #endif
191
192 #if OS(WINDOWS) && USE(CF)
193
194 void ResourceHandle::setClientCertificate(const String&, CFDataRef)
195 {
196 }
197
198 #endif
199
200 void ResourceHandle::platformSetDefersLoading(bool defers)
201 {
202     ASSERT(isMainThread());
203
204     if (defers == d->m_defersLoading)
205         return;
206
207     d->m_defersLoading = defers;
208
209     if (!d->m_curlRequest)
210         return;
211
212     if (d->m_defersLoading)
213         d->m_curlRequest->suspend();
214     else
215         d->m_curlRequest->resume();
216 }
217
218 bool ResourceHandle::shouldUseCredentialStorage()
219 {
220     return (!client() || client()->shouldUseCredentialStorage(this)) && firstRequest().url().protocolIsInHTTPFamily();
221 }
222
223 void ResourceHandle::didReceiveAuthenticationChallenge(const AuthenticationChallenge& challenge)
224 {
225     ASSERT(isMainThread());
226
227     String partition = firstRequest().cachePartition();
228
229     if (!d->m_user.isNull() && !d->m_pass.isNull()) {
230         Credential credential(d->m_user, d->m_pass, CredentialPersistenceNone);
231
232         URL urlToStore;
233         if (challenge.failureResponse().httpStatusCode() == 401)
234             urlToStore = challenge.failureResponse().url();
235         d->m_context->storageSession()->credentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
236
237         restartRequestWithCredential(challenge.protectionSpace(), credential);
238
239         d->m_user = String();
240         d->m_pass = String();
241         // FIXME: Per the specification, the user shouldn't be asked for credentials if there were incorrect ones provided explicitly.
242         return;
243     }
244
245     if (shouldUseCredentialStorage()) {
246         if (!d->m_initialCredential.isEmpty() || challenge.previousFailureCount()) {
247             // The stored credential wasn't accepted, stop using it.
248             // There is a race condition here, since a different credential might have already been stored by another ResourceHandle,
249             // but the observable effect should be very minor, if any.
250             d->m_context->storageSession()->credentialStorage().remove(partition, challenge.protectionSpace());
251         }
252
253         if (!challenge.previousFailureCount()) {
254             Credential credential = d->m_context->storageSession()->credentialStorage().get(partition, challenge.protectionSpace());
255             if (!credential.isEmpty() && credential != d->m_initialCredential) {
256                 ASSERT(credential.persistence() == CredentialPersistenceNone);
257                 if (challenge.failureResponse().httpStatusCode() == 401) {
258                     // Store the credential back, possibly adding it as a default for this directory.
259                     d->m_context->storageSession()->credentialStorage().set(partition, credential, challenge.protectionSpace(), challenge.failureResponse().url());
260                 }
261
262                 restartRequestWithCredential(challenge.protectionSpace(), credential);
263                 return;
264             }
265         }
266     }
267
268     d->m_currentWebChallenge = challenge;
269
270     if (client()) {
271         auto protectedThis = makeRef(*this);
272         client()->didReceiveAuthenticationChallenge(this, d->m_currentWebChallenge);
273     }
274 }
275
276 void ResourceHandle::receivedCredential(const AuthenticationChallenge& challenge, const Credential& credential)
277 {
278     ASSERT(isMainThread());
279
280     if (challenge != d->m_currentWebChallenge)
281         return;
282
283     if (credential.isEmpty()) {
284         receivedRequestToContinueWithoutCredential(challenge);
285         return;
286     }
287
288     String partition = firstRequest().cachePartition();
289
290     if (shouldUseCredentialStorage()) {
291         if (challenge.failureResponse().httpStatusCode() == 401) {
292             URL urlToStore = challenge.failureResponse().url();
293             d->m_context->storageSession()->credentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
294         }
295     }
296
297     restartRequestWithCredential(challenge.protectionSpace(), credential);
298
299     clearAuthentication();
300 }
301
302 void ResourceHandle::receivedRequestToContinueWithoutCredential(const AuthenticationChallenge& challenge)
303 {
304     ASSERT(isMainThread());
305
306     if (challenge != d->m_currentWebChallenge)
307         return;
308
309     clearAuthentication();
310
311     didReceiveResponse(ResourceResponse(delegate()->response()), [this, protectedThis = makeRef(*this)] {
312         continueAfterDidReceiveResponse();
313     });
314 }
315
316 void ResourceHandle::receivedCancellation(const AuthenticationChallenge& challenge)
317 {
318     ASSERT(isMainThread());
319
320     if (challenge != d->m_currentWebChallenge)
321         return;
322
323     if (client()) {
324         auto protectedThis = makeRef(*this);
325         client()->receivedCancellation(this, challenge);
326     }
327 }
328
329 void ResourceHandle::receivedRequestToPerformDefaultHandling(const AuthenticationChallenge&)
330 {
331     ASSERT_NOT_REACHED();
332 }
333
334 void ResourceHandle::receivedChallengeRejection(const AuthenticationChallenge&)
335 {
336     ASSERT_NOT_REACHED();
337 }
338
339 Optional<Credential> ResourceHandle::getCredential(const ResourceRequest& request, bool redirect)
340 {
341     // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
342     Credential credential { d->m_user, d->m_pass, CredentialPersistenceNone };
343
344     if (shouldUseCredentialStorage()) {
345         String partition = request.cachePartition();
346
347         if (credential.isEmpty()) {
348             // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 
349             // try and reuse the credential preemptively, as allowed by RFC 2617.
350             d->m_initialCredential = d->m_context->storageSession()->credentialStorage().get(partition, request.url());
351         } else if (!redirect) {
352             // If there is already a protection space known for the URL, update stored credentials
353             // before sending a request. This makes it possible to implement logout by sending an
354             // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
355             // an authentication dialog doesn't pop up).
356             d->m_context->storageSession()->credentialStorage().set(partition, credential, request.url());
357         }
358     }
359
360     if (!d->m_initialCredential.isEmpty())
361         return d->m_initialCredential;
362
363     return WTF::nullopt;
364 }
365
366 void ResourceHandle::restartRequestWithCredential(const ProtectionSpace& protectionSpace, const Credential& credential)
367 {
368     ASSERT(isMainThread());
369
370     if (!d->m_curlRequest)
371         return;
372     
373     auto previousRequest = d->m_curlRequest->resourceRequest();
374     d->m_curlRequest->cancel();
375
376     d->m_curlRequest = createCurlRequest(WTFMove(previousRequest), RequestStatus::ReusedRequest);
377     d->m_curlRequest->setAuthenticationScheme(protectionSpace.authenticationScheme());
378     d->m_curlRequest->setUserPass(credential.user(), credential.password());
379     d->m_curlRequest->setStartTime(d->m_startTime);
380     d->m_curlRequest->start();
381 }
382
383 void ResourceHandle::platformLoadResourceSynchronously(NetworkingContext* context, const ResourceRequest& request, StoredCredentialsPolicy storedCredentialsPolicy, ResourceError& error, ResourceResponse& response, Vector<char>& data)
384 {
385     ASSERT(isMainThread());
386     ASSERT(!request.isEmpty());
387
388     SynchronousLoaderClient client;
389     client.setAllowStoredCredentials(storedCredentialsPolicy == StoredCredentialsPolicy::Use);
390
391     bool defersLoading = false;
392     bool shouldContentSniff = true;
393     bool shouldContentEncodingSniff = true;
394     RefPtr<ResourceHandle> handle = adoptRef(new ResourceHandle(context, request, &client, defersLoading, shouldContentSniff, shouldContentEncodingSniff));
395     handle->d->m_messageQueue = &client.messageQueue();
396     handle->d->m_startTime = MonotonicTime::now();
397
398     if (request.url().protocolIsData()) {
399         handle->handleDataURL();
400         return;
401     }
402
403     auto requestCopy = handle->firstRequest();
404     handle->d->m_curlRequest = handle->createCurlRequest(WTFMove(requestCopy));
405
406     if (auto credential = handle->getCredential(handle->d->m_firstRequest, false)) {
407         handle->d->m_curlRequest->setUserPass(credential->user(), credential->password());
408         handle->d->m_curlRequest->setAuthenticationScheme(ProtectionSpaceAuthenticationSchemeHTTPBasic);
409     }
410
411     handle->d->m_curlRequest->setStartTime(handle->d->m_startTime);
412     handle->d->m_curlRequest->start();
413
414     do {
415         if (auto task = client.messageQueue().waitForMessage())
416             (*task)();
417     } while (!client.messageQueue().killed() && !handle->cancelledOrClientless());
418
419     error = client.error();
420     data.swap(client.mutableData());
421     response = client.response();
422 }
423
424 void ResourceHandle::platformContinueSynchronousDidReceiveResponse()
425 {
426     ASSERT(isMainThread());
427
428     continueAfterDidReceiveResponse();
429 }
430
431 void ResourceHandle::continueAfterDidReceiveResponse()
432 {
433     ASSERT(isMainThread());
434
435     // continueDidReceiveResponse might cancel the load.
436     if (cancelledOrClientless() || !d->m_curlRequest)
437         return;
438
439     d->m_curlRequest->completeDidReceiveResponse();
440 }
441
442 bool ResourceHandle::shouldRedirectAsGET(const ResourceRequest& request, bool crossOrigin)
443 {
444     if (request.httpMethod() == "GET" || request.httpMethod() == "HEAD")
445         return false;
446
447     if (!request.url().protocolIsInHTTPFamily())
448         return true;
449
450     if (delegate()->response().isSeeOther())
451         return true;
452
453     if ((delegate()->response().isMovedPermanently() || delegate()->response().isFound()) && (request.httpMethod() == "POST"))
454         return true;
455
456     if (crossOrigin && (request.httpMethod() == "DELETE"))
457         return true;
458
459     return false;
460 }
461
462 void ResourceHandle::willSendRequest()
463 {
464     ASSERT(isMainThread());
465
466     static const int maxRedirects = 20;
467
468     if (d->m_redirectCount++ > maxRedirects) {
469         client()->didFail(this, ResourceError::httpError(CURLE_TOO_MANY_REDIRECTS, delegate()->response().url()));
470         return;
471     }
472
473     String location = delegate()->response().httpHeaderField(HTTPHeaderName::Location);
474     URL newURL = URL(delegate()->response().url(), location);
475     bool crossOrigin = !protocolHostAndPortAreEqual(d->m_firstRequest.url(), newURL);
476
477     ResourceRequest newRequest = d->m_firstRequest;
478     newRequest.setURL(newURL);
479
480     if (shouldRedirectAsGET(newRequest, crossOrigin)) {
481         newRequest.setHTTPMethod("GET");
482         newRequest.setHTTPBody(nullptr);
483         newRequest.clearHTTPContentType();
484     }
485
486     // Should not set Referer after a redirect from a secure resource to non-secure one.
487     if (!newURL.protocolIs("https") && protocolIs(newRequest.httpReferrer(), "https") && context()->shouldClearReferrerOnHTTPSToHTTPRedirect())
488         newRequest.clearHTTPReferrer();
489
490     d->m_user = newURL.user();
491     d->m_pass = newURL.pass();
492     newRequest.removeCredentials();
493
494     if (crossOrigin) {
495         // If the network layer carries over authentication headers from the original request
496         // in a cross-origin redirect, we want to clear those headers here. 
497         newRequest.clearHTTPAuthorization();
498         newRequest.clearHTTPOrigin();
499         d->m_startTime = WTF::MonotonicTime::now();
500     }
501
502     ResourceResponse responseCopy = delegate()->response();
503     client()->willSendRequestAsync(this, WTFMove(newRequest), WTFMove(responseCopy), [this, protectedThis = makeRef(*this)] (ResourceRequest&& request) {
504         continueAfterWillSendRequest(WTFMove(request));
505     });
506 }
507
508 void ResourceHandle::continueAfterWillSendRequest(ResourceRequest&& request)
509 {
510     ASSERT(isMainThread());
511
512     // willSendRequest might cancel the load.
513     if (cancelledOrClientless() || !d->m_curlRequest)
514         return;
515
516     if (request.isNull()) {
517         cancel();
518         return;
519     }
520
521     auto shouldForwardCredential = protocolHostAndPortAreEqual(request.url(), delegate()->response().url());
522     auto credential = getCredential(request, true);
523
524     d->m_curlRequest->cancel();
525     d->m_curlRequest = createCurlRequest(WTFMove(request));
526
527     if (shouldForwardCredential && credential)
528         d->m_curlRequest->setUserPass(credential->user(), credential->password());
529
530     d->m_curlRequest->setStartTime(d->m_startTime);
531     d->m_curlRequest->start();
532 }
533
534 void ResourceHandle::handleDataURL()
535 {
536     ASSERT(d->m_firstRequest.url().protocolIsData());
537     String url = d->m_firstRequest.url().string();
538
539     ASSERT(client());
540
541     auto index = url.find(',');
542     if (index == notFound) {
543         client()->cannotShowURL(this);
544         return;
545     }
546
547     String mediaType = url.substring(5, index - 5);
548     String data = url.substring(index + 1);
549     auto originalSize = data.length();
550
551     bool base64 = mediaType.endsWithIgnoringASCIICase(";base64");
552     if (base64)
553         mediaType = mediaType.left(mediaType.length() - 7);
554
555     if (mediaType.isEmpty())
556         mediaType = "text/plain"_s;
557
558     String mimeType = extractMIMETypeFromMediaType(mediaType);
559     String charset = extractCharsetFromMediaType(mediaType);
560
561     if (charset.isEmpty())
562         charset = "US-ASCII"_s;
563
564     ResourceResponse response;
565     response.setMimeType(mimeType);
566     response.setTextEncodingName(charset);
567     response.setURL(d->m_firstRequest.url());
568
569     if (base64) {
570         data = decodeURLEscapeSequences(data);
571         didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
572             continueAfterDidReceiveResponse();
573         });
574
575         // didReceiveResponse might cause the client to be deleted.
576         if (client()) {
577             Vector<char> out;
578             if (base64Decode(data, out, Base64IgnoreSpacesAndNewLines) && out.size() > 0)
579                 client()->didReceiveBuffer(this, SharedBuffer::create(out.data(), out.size()), originalSize);
580         }
581     } else {
582         TextEncoding encoding(charset);
583         data = decodeURLEscapeSequences(data, encoding);
584         didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
585             continueAfterDidReceiveResponse();
586         });
587
588         // didReceiveResponse might cause the client to be deleted.
589         if (client()) {
590             auto encodedData = encoding.encode(data, UnencodableHandling::URLEncodedEntities);
591             if (encodedData.size())
592                 client()->didReceiveBuffer(this, SharedBuffer::create(WTFMove(encodedData)), originalSize);
593         }
594     }
595
596     if (client())
597         client()->didFinishLoading(this);
598 }
599
600 } // namespace WebCore
601
602 #endif