[WTF] Add user-defined literal for ASCIILiteral
[WebKit-https.git] / Source / WebCore / platform / network / curl / ResourceHandleCurl.cpp
1 /*
2  * Copyright (C) 2004, 2006 Apple Inc.  All rights reserved.
3  * Copyright (C) 2005, 2006 Michael Emmel mike.emmel@gmail.com
4  * Copyright (C) 2017 Sony Interactive Entertainment Inc.
5  * All rights reserved.
6  * Copyright (C) 2017 NAVER Corp.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
18  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
25  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  */
29
30 #include "config.h"
31 #include "ResourceHandle.h"
32
33 #if USE(CURL)
34
35 #include "CookieJarCurl.h"
36 #include "CookiesStrategy.h"
37 #include "CredentialStorage.h"
38 #include "CurlCacheManager.h"
39 #include "CurlContext.h"
40 #include "CurlRequest.h"
41 #include "FileSystem.h"
42 #include "HTTPParsers.h"
43 #include "Logging.h"
44 #include "NetworkStorageSession.h"
45 #include "ResourceHandleInternal.h"
46 #include "SameSiteInfo.h"
47 #include "SharedBuffer.h"
48 #include "SynchronousLoaderClient.h"
49 #include "TextEncoding.h"
50 #include <wtf/CompletionHandler.h>
51 #include <wtf/text/Base64.h>
52
53 namespace WebCore {
54
55 ResourceHandleInternal::~ResourceHandleInternal()
56 {
57     if (m_curlRequest)
58         m_curlRequest->invalidateClient();
59 }
60
61 ResourceHandle::~ResourceHandle() = default;
62
63 bool ResourceHandle::start()
64 {
65     ASSERT(isMainThread());
66
67     CurlContext::singleton();
68
69     if (!d->m_delegate.get())
70         d->m_delegate = std::make_unique<CurlResourceHandleDelegate>(*this);
71
72     // The frame could be null if the ResourceHandle is not associated to any
73     // Frame, e.g. if we are downloading a file.
74     // If the frame is not null but the page is null this must be an attempted
75     // load from an unload handler, so let's just block it.
76     // If both the frame and the page are not null the context is valid.
77     if (d->m_context && !d->m_context->isValid())
78         return false;
79
80     // Only allow the POST and GET methods for non-HTTP requests.
81     const ResourceRequest& request = firstRequest();
82     if (!request.url().protocolIsInHTTPFamily() && request.httpMethod() != "GET" && request.httpMethod() != "POST") {
83         scheduleFailure(InvalidURLFailure); // Error must not be reported immediately
84         return true;
85     }
86
87     d->m_curlRequest = createCurlRequest(d->m_firstRequest);
88
89     if (auto credential = getCredential(d->m_firstRequest, false))
90         d->m_curlRequest->setUserPass(credential->first, credential->second);
91
92     d->m_curlRequest->start();
93
94     return true;
95 }
96
97 void ResourceHandle::cancel()
98 {
99     ASSERT(isMainThread());
100
101     d->m_cancelled = true;
102
103     if (!d->m_curlRequest)
104         d->m_curlRequest->cancel();
105 }
106
107 bool ResourceHandle::cancelledOrClientless()
108 {
109     if (d->m_cancelled)
110         return true;
111
112     return !client();
113 }
114
115 void ResourceHandle::addCacheValidationHeaders(ResourceRequest& request)
116 {
117     ASSERT(isMainThread());
118
119     d->m_addedCacheValidationHeaders = false;
120
121     auto hasCacheHeaders = request.httpHeaderFields().contains(HTTPHeaderName::IfModifiedSince) || request.httpHeaderFields().contains(HTTPHeaderName::IfNoneMatch);
122     if (hasCacheHeaders)
123         return;
124
125     auto& cache = CurlCacheManager::singleton();
126     URL cacheUrl = request.url();
127     cacheUrl.removeFragmentIdentifier();
128
129     if (cache.isCached(cacheUrl)) {
130         cache.addCacheEntryClient(cacheUrl, this);
131
132         for (const auto& entry : cache.requestHeaders(cacheUrl))
133             request.addHTTPHeaderField(entry.key, entry.value);
134
135         d->m_addedCacheValidationHeaders = true;
136     }
137 }
138
139 Ref<CurlRequest> ResourceHandle::createCurlRequest(ResourceRequest& request, RequestStatus status)
140 {
141     ASSERT(isMainThread());
142
143     if (status == RequestStatus::NewRequest) {
144         addCacheValidationHeaders(request);
145
146         auto& storageSession = NetworkStorageSession::defaultStorageSession();
147         auto& cookieJar = storageSession.cookieStorage();
148         auto includeSecureCookies = request.url().protocolIs("https") ? IncludeSecureCookies::Yes : IncludeSecureCookies::No;
149         String cookieHeaderField = cookieJar.cookieRequestHeaderFieldValue(storageSession, request.firstPartyForCookies(), SameSiteInfo::create(request), request.url(), std::nullopt, std::nullopt, includeSecureCookies).first;
150         if (!cookieHeaderField.isEmpty())
151             request.addHTTPHeaderField(HTTPHeaderName::Cookie, cookieHeaderField);
152     }
153
154     CurlRequest::ShouldSuspend shouldSuspend = d->m_defersLoading ? CurlRequest::ShouldSuspend::Yes : CurlRequest::ShouldSuspend::No;
155     auto curlRequest = CurlRequest::create(request, *delegate(), shouldSuspend, CurlRequest::EnableMultipart::Yes);
156     
157     return curlRequest;
158 }
159
160 CurlResourceHandleDelegate* ResourceHandle::delegate()
161 {
162     ASSERT(d->m_delegate);
163     return d->m_delegate.get();
164 }
165
166 #if OS(WINDOWS)
167
168 void ResourceHandle::setHostAllowsAnyHTTPSCertificate(const String& host)
169 {
170     ASSERT(isMainThread());
171
172     CurlContext::singleton().sslHandle().setHostAllowsAnyHTTPSCertificate(host);
173 }
174
175 void ResourceHandle::setClientCertificateInfo(const String& host, const String& certificate, const String& key)
176 {
177     ASSERT(isMainThread());
178
179     if (FileSystem::fileExists(certificate))
180         CurlContext::singleton().sslHandle().setClientCertificateInfo(host, certificate, key);
181     else
182         LOG(Network, "Invalid client certificate file: %s!\n", certificate.latin1().data());
183 }
184
185 #endif
186
187 #if OS(WINDOWS) && USE(CF)
188
189 void ResourceHandle::setClientCertificate(const String&, CFDataRef)
190 {
191 }
192
193 #endif
194
195 void ResourceHandle::platformSetDefersLoading(bool defers)
196 {
197     ASSERT(isMainThread());
198
199     if (defers == d->m_defersLoading)
200         return;
201
202     d->m_defersLoading = defers;
203
204     if (!d->m_curlRequest)
205         return;
206
207     if (d->m_defersLoading)
208         d->m_curlRequest->suspend();
209     else
210         d->m_curlRequest->resume();
211 }
212
213 bool ResourceHandle::shouldUseCredentialStorage()
214 {
215     return (!client() || client()->shouldUseCredentialStorage(this)) && firstRequest().url().protocolIsInHTTPFamily();
216 }
217
218 void ResourceHandle::didReceiveAuthenticationChallenge(const AuthenticationChallenge& challenge)
219 {
220     ASSERT(isMainThread());
221
222     String partition = firstRequest().cachePartition();
223
224     if (!d->m_user.isNull() && !d->m_pass.isNull()) {
225         Credential credential(d->m_user, d->m_pass, CredentialPersistenceNone);
226
227         URL urlToStore;
228         if (challenge.failureResponse().httpStatusCode() == 401)
229             urlToStore = challenge.failureResponse().url();
230         CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
231
232         restartRequestWithCredential(credential.user(), credential.password());
233
234         d->m_user = String();
235         d->m_pass = String();
236         // FIXME: Per the specification, the user shouldn't be asked for credentials if there were incorrect ones provided explicitly.
237         return;
238     }
239
240     if (shouldUseCredentialStorage()) {
241         if (!d->m_initialCredential.isEmpty() || challenge.previousFailureCount()) {
242             // The stored credential wasn't accepted, stop using it.
243             // There is a race condition here, since a different credential might have already been stored by another ResourceHandle,
244             // but the observable effect should be very minor, if any.
245             CredentialStorage::defaultCredentialStorage().remove(partition, challenge.protectionSpace());
246         }
247
248         if (!challenge.previousFailureCount()) {
249             Credential credential = CredentialStorage::defaultCredentialStorage().get(partition, challenge.protectionSpace());
250             if (!credential.isEmpty() && credential != d->m_initialCredential) {
251                 ASSERT(credential.persistence() == CredentialPersistenceNone);
252                 if (challenge.failureResponse().httpStatusCode() == 401) {
253                     // Store the credential back, possibly adding it as a default for this directory.
254                     CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), challenge.failureResponse().url());
255                 }
256
257                 restartRequestWithCredential(credential.user(), credential.password());
258                 return;
259             }
260         }
261     }
262
263     d->m_currentWebChallenge = challenge;
264
265     if (client()) {
266         auto protectedThis = makeRef(*this);
267         client()->didReceiveAuthenticationChallenge(this, d->m_currentWebChallenge);
268     }
269 }
270
271 void ResourceHandle::receivedCredential(const AuthenticationChallenge& challenge, const Credential& credential)
272 {
273     ASSERT(isMainThread());
274
275     if (challenge != d->m_currentWebChallenge)
276         return;
277
278     if (credential.isEmpty()) {
279         receivedRequestToContinueWithoutCredential(challenge);
280         return;
281     }
282
283     String partition = firstRequest().cachePartition();
284
285     if (shouldUseCredentialStorage()) {
286         if (challenge.failureResponse().httpStatusCode() == 401) {
287             URL urlToStore = challenge.failureResponse().url();
288             CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
289         }
290     }
291
292     restartRequestWithCredential(credential.user(), credential.password());
293
294     clearAuthentication();
295 }
296
297 void ResourceHandle::receivedRequestToContinueWithoutCredential(const AuthenticationChallenge& challenge)
298 {
299     ASSERT(isMainThread());
300
301     if (challenge != d->m_currentWebChallenge)
302         return;
303
304     clearAuthentication();
305
306     didReceiveResponse(ResourceResponse(delegate()->response()), [this, protectedThis = makeRef(*this)] {
307         continueAfterDidReceiveResponse();
308     });
309 }
310
311 void ResourceHandle::receivedCancellation(const AuthenticationChallenge& challenge)
312 {
313     ASSERT(isMainThread());
314
315     if (challenge != d->m_currentWebChallenge)
316         return;
317
318     if (client()) {
319         auto protectedThis = makeRef(*this);
320         client()->receivedCancellation(this, challenge);
321     }
322 }
323
324 void ResourceHandle::receivedRequestToPerformDefaultHandling(const AuthenticationChallenge&)
325 {
326     ASSERT_NOT_REACHED();
327 }
328
329 void ResourceHandle::receivedChallengeRejection(const AuthenticationChallenge&)
330 {
331     ASSERT_NOT_REACHED();
332 }
333
334 std::optional<std::pair<String, String>> ResourceHandle::getCredential(ResourceRequest& request, bool redirect)
335 {
336     // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
337     String partition = request.cachePartition();
338
339     if (shouldUseCredentialStorage()) {
340         if (d->m_user.isEmpty() && d->m_pass.isEmpty()) {
341             // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 
342             // try and reuse the credential preemptively, as allowed by RFC 2617.
343             d->m_initialCredential = CredentialStorage::defaultCredentialStorage().get(partition, request.url());
344         } else if (!redirect) {
345             // If there is already a protection space known for the URL, update stored credentials
346             // before sending a request. This makes it possible to implement logout by sending an
347             // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
348             // an authentication dialog doesn't pop up).
349             CredentialStorage::defaultCredentialStorage().set(partition, Credential(d->m_user, d->m_pass, CredentialPersistenceNone), request.url());
350         }
351     }
352
353     String user = d->m_user;
354     String password = d->m_pass;
355
356     if (!d->m_initialCredential.isEmpty()) {
357         user = d->m_initialCredential.user();
358         password = d->m_initialCredential.password();
359     }
360
361     if (user.isEmpty() && password.isEmpty())
362         return std::nullopt;
363
364     return std::pair<String, String>(user, password);
365 }
366
367 void ResourceHandle::restartRequestWithCredential(const String& user, const String& password)
368 {
369     ASSERT(isMainThread());
370
371     if (!d->m_curlRequest)
372         return;
373     
374     auto wasSyncRequest = d->m_curlRequest->isSyncRequest();
375     auto previousRequest = d->m_curlRequest->resourceRequest();
376     d->m_curlRequest->cancel();
377
378     d->m_curlRequest = createCurlRequest(previousRequest, RequestStatus::ReusedRequest);
379     d->m_curlRequest->setUserPass(user, password);
380     d->m_curlRequest->start(wasSyncRequest);
381 }
382
383 void ResourceHandle::platformLoadResourceSynchronously(NetworkingContext* context, const ResourceRequest& request, StoredCredentialsPolicy, ResourceError& error, ResourceResponse& response, Vector<char>& data)
384 {
385     ASSERT(isMainThread());
386
387     auto localRequest = request;
388     SynchronousLoaderClient client;
389     bool defersLoading = false;
390     bool shouldContentSniff = true;
391     bool shouldContentEncodingSniff = true;
392     RefPtr<ResourceHandle> handle = adoptRef(new ResourceHandle(context, request, &client, defersLoading, shouldContentSniff, shouldContentEncodingSniff));
393
394     if (localRequest.url().protocolIsData()) {
395         handle->handleDataURL();
396         return;
397     }
398
399     // If defersLoading is true and we call curl_easy_perform
400     // on a paused handle, libcURL would do the transfer anyway
401     // and we would assert so force defersLoading to be false.
402     handle->d->m_defersLoading = false;
403
404     handle->d->m_curlRequest = handle->createCurlRequest(localRequest);
405     handle->d->m_curlRequest->start(true);
406
407     error = client.error();
408     data.swap(client.mutableData());
409     response = client.response();
410 }
411
412 void ResourceHandle::platformContinueSynchronousDidReceiveResponse()
413 {
414     ASSERT(isMainThread());
415
416     continueAfterDidReceiveResponse();
417 }
418
419 void ResourceHandle::continueAfterDidReceiveResponse()
420 {
421     ASSERT(isMainThread());
422
423     // continueDidReceiveResponse might cancel the load.
424     if (cancelledOrClientless() || !d->m_curlRequest)
425         return;
426
427     d->m_curlRequest->completeDidReceiveResponse();
428 }
429
430 bool ResourceHandle::shouldRedirectAsGET(const ResourceRequest& request, bool crossOrigin)
431 {
432     if (request.httpMethod() == "GET" || request.httpMethod() == "HEAD")
433         return false;
434
435     if (!request.url().protocolIsInHTTPFamily())
436         return true;
437
438     if (delegate()->response().isSeeOther())
439         return true;
440
441     if ((delegate()->response().isMovedPermanently() || delegate()->response().isFound()) && (request.httpMethod() == "POST"))
442         return true;
443
444     if (crossOrigin && (request.httpMethod() == "DELETE"))
445         return true;
446
447     return false;
448 }
449
450 void ResourceHandle::willSendRequest()
451 {
452     ASSERT(isMainThread());
453
454     static const int maxRedirects = 20;
455
456     if (d->m_redirectCount++ > maxRedirects) {
457         client()->didFail(this, ResourceError::httpError(CURLE_TOO_MANY_REDIRECTS, delegate()->response().url()));
458         return;
459     }
460
461     String location = delegate()->response().httpHeaderField(HTTPHeaderName::Location);
462     URL newURL = URL(delegate()->response().url(), location);
463     bool crossOrigin = !protocolHostAndPortAreEqual(d->m_firstRequest.url(), newURL);
464
465     ResourceRequest newRequest = d->m_firstRequest;
466     newRequest.setURL(newURL);
467
468     if (shouldRedirectAsGET(newRequest, crossOrigin)) {
469         newRequest.setHTTPMethod("GET");
470         newRequest.setHTTPBody(nullptr);
471         newRequest.clearHTTPContentType();
472     }
473
474     // Should not set Referer after a redirect from a secure resource to non-secure one.
475     if (!newURL.protocolIs("https") && protocolIs(newRequest.httpReferrer(), "https") && context()->shouldClearReferrerOnHTTPSToHTTPRedirect())
476         newRequest.clearHTTPReferrer();
477
478     d->m_user = newURL.user();
479     d->m_pass = newURL.pass();
480     newRequest.removeCredentials();
481
482     if (crossOrigin) {
483         // If the network layer carries over authentication headers from the original request
484         // in a cross-origin redirect, we want to clear those headers here. 
485         newRequest.clearHTTPAuthorization();
486         newRequest.clearHTTPOrigin();
487     }
488
489     ResourceResponse responseCopy = delegate()->response();
490     client()->willSendRequestAsync(this, WTFMove(newRequest), WTFMove(responseCopy), [this, protectedThis = makeRef(*this)] (ResourceRequest&& request) {
491         continueAfterWillSendRequest(WTFMove(request));
492     });
493 }
494
495 void ResourceHandle::continueAfterWillSendRequest(ResourceRequest&& request)
496 {
497     ASSERT(isMainThread());
498
499     // willSendRequest might cancel the load.
500     if (cancelledOrClientless() || !d->m_curlRequest)
501         return;
502
503     auto wasSyncRequest = d->m_curlRequest->isSyncRequest();
504     d->m_curlRequest->cancel();
505
506     d->m_curlRequest = createCurlRequest(request);
507
508     if (protocolHostAndPortAreEqual(request.url(), delegate()->response().url())) {
509         if (auto credential = getCredential(request, true))
510             d->m_curlRequest->setUserPass(credential->first, credential->second);
511     }
512
513     d->m_curlRequest->start(wasSyncRequest);
514 }
515
516 void ResourceHandle::handleDataURL()
517 {
518     ASSERT(d->m_firstRequest.url().protocolIsData());
519     String url = d->m_firstRequest.url().string();
520
521     ASSERT(client());
522
523     auto index = url.find(',');
524     if (index == notFound) {
525         client()->cannotShowURL(this);
526         return;
527     }
528
529     String mediaType = url.substring(5, index - 5);
530     String data = url.substring(index + 1);
531     auto originalSize = data.length();
532
533     bool base64 = mediaType.endsWithIgnoringASCIICase(";base64");
534     if (base64)
535         mediaType = mediaType.left(mediaType.length() - 7);
536
537     if (mediaType.isEmpty())
538         mediaType = "text/plain"_s;
539
540     String mimeType = extractMIMETypeFromMediaType(mediaType);
541     String charset = extractCharsetFromMediaType(mediaType);
542
543     if (charset.isEmpty())
544         charset = "US-ASCII"_s;
545
546     ResourceResponse response;
547     response.setMimeType(mimeType);
548     response.setTextEncodingName(charset);
549     response.setURL(d->m_firstRequest.url());
550
551     if (base64) {
552         data = decodeURLEscapeSequences(data);
553         didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
554             continueAfterDidReceiveResponse();
555         });
556
557         // didReceiveResponse might cause the client to be deleted.
558         if (client()) {
559             Vector<char> out;
560             if (base64Decode(data, out, Base64IgnoreSpacesAndNewLines) && out.size() > 0)
561                 client()->didReceiveBuffer(this, SharedBuffer::create(out.data(), out.size()), originalSize);
562         }
563     } else {
564         TextEncoding encoding(charset);
565         data = decodeURLEscapeSequences(data, encoding);
566         didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
567             continueAfterDidReceiveResponse();
568         });
569
570         // didReceiveResponse might cause the client to be deleted.
571         if (client()) {
572             auto encodedData = encoding.encode(data, UnencodableHandling::URLEncodedEntities);
573             if (encodedData.size())
574                 client()->didReceiveBuffer(this, SharedBuffer::create(WTFMove(encodedData)), originalSize);
575         }
576     }
577
578     if (client())
579         client()->didFinishLoading(this);
580 }
581
582 } // namespace WebCore
583
584 #endif