[Curl] Bug fix after r219606
[WebKit-https.git] / Source / WebCore / platform / network / curl / ResourceHandleCurl.cpp
1 /*
2  * Copyright (C) 2004, 2006 Apple Inc.  All rights reserved.
3  * Copyright (C) 2005, 2006 Michael Emmel mike.emmel@gmail.com
4  * Copyright (C) 2017 Sony Interactive Entertainment Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28
29 #include "config.h"
30 #include "ResourceHandle.h"
31
32 #if USE(CURL)
33
34 #include "CachedResourceLoader.h"
35 #include "CredentialStorage.h"
36 #include "CurlCacheManager.h"
37 #include "CurlContext.h"
38 #include "CurlJobManager.h"
39 #include "FileSystem.h"
40 #include "Logging.h"
41 #include "MIMETypeRegistry.h"
42 #include "NetworkingContext.h"
43 #include "ResourceHandleInternal.h"
44 #include "SSLHandle.h"
45 #include "SynchronousLoaderClient.h"
46 #include <wtf/text/Base64.h>
47
48 namespace WebCore {
49
50 ResourceHandleInternal::~ResourceHandleInternal()
51 {
52 }
53
54 ResourceHandle::~ResourceHandle()
55 {
56 }
57
58 bool ResourceHandle::start()
59 {
60     ASSERT(isMainThread());
61
62     // The frame could be null if the ResourceHandle is not associated to any
63     // Frame, e.g. if we are downloading a file.
64     // If the frame is not null but the page is null this must be an attempted
65     // load from an unload handler, so let's just block it.
66     // If both the frame and the page are not null the context is valid.
67     if (d->m_context && !d->m_context->isValid())
68         return false;
69
70     d->initialize();
71
72     d->m_job = CurlJobManager::singleton().add(d->m_curlHandle, [this, protectedThis = makeRef(*this)](CurlJobResult result) {
73         ASSERT(isMainThread());
74
75         switch (result) {
76         case CurlJobResult::Done:
77             d->didFinish();
78             break;
79
80         case CurlJobResult::Error:
81             d->didFail();
82             break;
83
84         case CurlJobResult::Cancelled:
85             break;
86         }
87     });
88     ASSERT(d->m_job);
89
90     return true;
91 }
92
93 void ResourceHandle::cancel()
94 {
95     d->m_cancelled = true;
96     CurlJobManager::singleton().cancel(d->m_job);
97 }
98
99 void ResourceHandleInternal::initialize()
100 {
101     CurlContext& context = CurlContext::singleton();
102
103     URL url = m_firstRequest.url();
104
105     // Remove any fragment part, otherwise curl will send it as part of the request.
106     url.removeFragmentIdentifier();
107
108     String urlString = url.string();
109
110     if (url.isLocalFile()) {
111         // Remove any query part sent to a local file.
112         if (!url.query().isEmpty()) {
113             // By setting the query to a null string it'll be removed.
114             url.setQuery(String());
115             urlString = url.string();
116         }
117         // Determine the MIME type based on the path.
118         m_response.setMimeType(MIMETypeRegistry::getMIMETypeForPath(url));
119     }
120
121     if (m_defersLoading) {
122         CURLcode error = m_curlHandle.pause(CURLPAUSE_ALL);
123         // If we did not pause the handle, we would ASSERT in the
124         // header callback. So just assert here.
125         ASSERT_UNUSED(error, error == CURLE_OK);
126     }
127
128 #ifndef NDEBUG
129     m_curlHandle.enableVerboseIfUsed();
130     m_curlHandle.enableStdErrIfUsed();
131 #endif
132
133     m_curlHandle.setSslVerifyPeer(CurlHandle::VerifyPeerEnable);
134     m_curlHandle.setSslVerifyHost(CurlHandle::VerifyHostStrictNameCheck);
135     m_curlHandle.setPrivateData(this);
136     m_curlHandle.setWriteCallbackFunction(writeCallback, this);
137     m_curlHandle.setHeaderCallbackFunction(headerCallback, this);
138     m_curlHandle.enableAutoReferer();
139     m_curlHandle.enableFollowLocation();
140     m_curlHandle.enableHttpAuthentication(CURLAUTH_ANY);
141     m_curlHandle.enableShareHandle();
142     m_curlHandle.enableTimeout();
143     m_curlHandle.enableAllowedProtocols();
144     setSSLClientCertificate(m_handle);
145
146     if (CurlContext::singleton().shouldIgnoreSSLErrors())
147         m_curlHandle.setSslVerifyPeer(CurlHandle::VerifyPeerDisable);
148     else
149         setSSLVerifyOptions(m_handle);
150
151     m_curlHandle.enableCAInfoIfExists();
152
153     m_curlHandle.enableAcceptEncoding();
154     m_curlHandle.setUrl(urlString);
155     m_curlHandle.enableCookieJarIfExists();
156
157     if (m_firstRequest.httpHeaderFields().size()) {
158         auto customHeaders = m_firstRequest.httpHeaderFields();
159         auto& cache = CurlCacheManager::getInstance();
160
161         bool hasCacheHeaders = customHeaders.contains(HTTPHeaderName::IfModifiedSince) || customHeaders.contains(HTTPHeaderName::IfNoneMatch);
162         if (!hasCacheHeaders && cache.isCached(url)) {
163             cache.addCacheEntryClient(url, m_handle);
164
165             // append additional cache information
166             for (auto entry : cache.requestHeaders(url))
167                 customHeaders.set(entry.key, entry.value);
168
169             m_addedCacheValidationHeaders = true;
170         }
171
172         m_curlHandle.appendRequestHeaders(customHeaders);
173     }
174
175     String method = m_firstRequest.httpMethod();
176     if ("GET" == method)
177         m_curlHandle.enableHttpGetRequest();
178     else if ("POST" == method)
179         setupPOST();
180     else if ("PUT" == method)
181         setupPUT();
182     else if ("HEAD" == method)
183         m_curlHandle.enableHttpHeadRequest();
184     else {
185         m_curlHandle.setHttpCustomRequest(method);
186         setupPUT();
187     }
188
189     m_curlHandle.enableRequestHeaders();
190
191     applyAuthentication();
192
193     m_curlHandle.enableProxyIfExists();
194 }
195
196 void ResourceHandleInternal::applyAuthentication()
197 {
198     // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
199     String partition = m_firstRequest.cachePartition();
200
201     if (m_handle->shouldUseCredentialStorage()) {
202         if (m_user.isEmpty() && m_pass.isEmpty()) {
203             // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 
204             // try and reuse the credential preemptively, as allowed by RFC 2617.
205             m_initialCredential = CredentialStorage::defaultCredentialStorage().get(partition, m_firstRequest.url());
206         } else {
207             // If there is already a protection space known for the URL, update stored credentials
208             // before sending a request. This makes it possible to implement logout by sending an
209             // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
210             // an authentication dialog doesn't pop up).
211             CredentialStorage::defaultCredentialStorage().set(partition, Credential(m_user, m_pass, CredentialPersistenceNone), m_firstRequest.url());
212         }
213     }
214
215     String user = m_user;
216     String password = m_pass;
217
218     if (!m_initialCredential.isEmpty()) {
219         user = m_initialCredential.user();
220         password = m_initialCredential.password();
221         m_curlHandle.enableHttpAuthentication(CURLAUTH_BASIC);
222     }
223
224     // It seems we need to set CURLOPT_USERPWD even if username and password is empty.
225     // Otherwise cURL will not automatically continue with a new request after a 401 response.
226
227     // curl CURLOPT_USERPWD expects username:password
228     m_curlHandle.setHttpAuthUserPass(user, password);
229 }
230
231 static inline size_t getFormElementsCount(ResourceHandle* job)
232 {
233     RefPtr<FormData> formData = job->firstRequest().httpBody();
234
235     if (!formData)
236         return 0;
237
238     // Resolve the blob elements so the formData can correctly report it's size.
239     formData = formData->resolveBlobReferences();
240     size_t size = formData->elements().size();
241     job->firstRequest().setHTTPBody(WTFMove(formData));
242
243     return size;
244 }
245
246 void ResourceHandleInternal::setupPUT()
247 {
248     m_curlHandle.enableHttpPutRequest();
249
250     // Disable the Expect: 100 continue header
251     m_curlHandle.appendRequestHeader("Expect:");
252
253     size_t numElements = getFormElementsCount(m_handle);
254     if (!numElements)
255         return;
256
257     setupFormData(false);
258 }
259
260 void ResourceHandleInternal::setupPOST()
261 {
262     m_curlHandle.enableHttpPostRequest();
263
264     size_t numElements = getFormElementsCount(m_handle);
265     if (!numElements)
266         return;
267
268     // Do not stream for simple POST data
269     if (numElements == 1) {
270         m_firstRequest.httpBody()->flatten(m_postBytes);
271         if (m_postBytes.size())
272             m_curlHandle.setPostFields(m_postBytes.data(), m_postBytes.size());
273         return;
274     }
275
276     setupFormData(true);
277 }
278
279 void ResourceHandleInternal::setupFormData(bool isPostRequest)
280 {
281     Vector<FormDataElement> elements = m_firstRequest.httpBody()->elements();
282     size_t numElements = elements.size();
283
284     static const long long maxCurlOffT = m_curlHandle.maxCurlOffT();
285
286     // Obtain the total size of the form data
287     curl_off_t size = 0;
288     bool chunkedTransfer = false;
289     for (size_t i = 0; i < numElements; i++) {
290         FormDataElement element = elements[i];
291         if (element.m_type == FormDataElement::Type::EncodedFile) {
292             long long fileSizeResult;
293             if (getFileSize(element.m_filename, fileSizeResult)) {
294                 if (fileSizeResult > maxCurlOffT) {
295                     // File size is too big for specifying it to cURL
296                     chunkedTransfer = true;
297                     break;
298                 }
299                 size += fileSizeResult;
300             } else {
301                 chunkedTransfer = true;
302                 break;
303             }
304         } else
305             size += elements[i].m_data.size();
306     }
307
308     // cURL guesses that we want chunked encoding as long as we specify the header
309     if (chunkedTransfer)
310         m_curlHandle.appendRequestHeader("Transfer-Encoding: chunked");
311     else {
312         if (isPostRequest)
313             m_curlHandle.setPostFieldLarge(size);
314         else
315             m_curlHandle.setInFileSizeLarge(size);
316     }
317
318     m_curlHandle.setReadCallbackFunction(readCallback, this);
319 }
320
321 #if OS(WINDOWS)
322
323 void ResourceHandle::setHostAllowsAnyHTTPSCertificate(const String& host)
324 {
325     ASSERT(isMainThread());
326
327     allowsAnyHTTPSCertificateHosts(host);
328 }
329
330 void ResourceHandle::setClientCertificateInfo(const String& host, const String& certificate, const String& key)
331 {
332     ASSERT(isMainThread());
333
334     if (fileExists(certificate))
335         addAllowedClientCertificate(host, certificate, key);
336     else
337         LOG(Network, "Invalid client certificate file: %s!\n", certificate.latin1().data());
338 }
339
340 #endif
341
342 #if OS(WINDOWS) && USE(CF)
343
344 void ResourceHandle::setClientCertificate(const String&, CFDataRef)
345 {
346 }
347
348 #endif
349
350 void ResourceHandle::platformSetDefersLoading(bool defers)
351 {
352     ASSERT(isMainThread());
353
354     auto action = [defers, this, protectedThis = makeRef(*this)]() {
355         if (defers) {
356             CURLcode error = d->m_curlHandle.pause(CURLPAUSE_ALL);
357             // If we could not defer the handle, so don't do it.
358             if (error != CURLE_OK)
359                 return;
360         } else {
361             CURLcode error = d->m_curlHandle.pause(CURLPAUSE_CONT);
362             if (error != CURLE_OK) {
363                 // Restarting the handle has failed so just cancel it.
364                 cancel();
365             }
366         }
367     };
368
369     if (d->m_job) {
370         CurlJobManager::singleton().callOnJobThread(WTFMove(action));
371     } else {
372         action();
373     }
374 }
375
376 void ResourceHandleInternal::didFinish()
377 {
378 #if ENABLE(WEB_TIMING)
379     calculateWebTimingInformations();
380 #endif
381     if (m_cancelled)
382         return;
383
384     if (!m_response.responseFired()) {
385         handleLocalReceiveResponse();
386         if (m_cancelled)
387             return;
388     }
389
390     if (m_multipartHandle)
391         m_multipartHandle->contentEnded();
392
393     if (client()) {
394         client()->didFinishLoading(m_handle);
395         CurlCacheManager::getInstance().didFinishLoading(*m_handle);
396     }
397 }
398
399 void ResourceHandleInternal::didFail()
400 {
401     if (m_cancelled)
402         return;
403     URL url = m_curlHandle.getEffectiveURL();
404     if (client()) {
405         client()->didFail(m_handle, ResourceError(m_curlHandle, m_sslErrors));
406         CurlCacheManager::getInstance().didFail(*m_handle);
407     }
408 }
409
410 bool ResourceHandle::shouldUseCredentialStorage()
411 {
412     return (!client() || client()->shouldUseCredentialStorage(this)) && firstRequest().url().protocolIsInHTTPFamily();
413 }
414
415 void ResourceHandle::didReceiveAuthenticationChallenge(const AuthenticationChallenge& challenge)
416 {
417     ASSERT(isMainThread());
418
419     String partition = firstRequest().cachePartition();
420
421     if (!d->m_user.isNull() && !d->m_pass.isNull()) {
422         Credential credential(d->m_user, d->m_pass, CredentialPersistenceNone);
423
424         URL urlToStore;
425         if (challenge.failureResponse().httpStatusCode() == 401)
426             urlToStore = challenge.failureResponse().url();
427         CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
428         
429         d->m_curlHandle.setHttpAuthUserPass(credential.user(), credential.password());
430
431         d->m_user = String();
432         d->m_pass = String();
433         // FIXME: Per the specification, the user shouldn't be asked for credentials if there were incorrect ones provided explicitly.
434         return;
435     }
436
437     if (shouldUseCredentialStorage()) {
438         if (!d->m_initialCredential.isEmpty() || challenge.previousFailureCount()) {
439             // The stored credential wasn't accepted, stop using it.
440             // There is a race condition here, since a different credential might have already been stored by another ResourceHandle,
441             // but the observable effect should be very minor, if any.
442             CredentialStorage::defaultCredentialStorage().remove(partition, challenge.protectionSpace());
443         }
444
445         if (!challenge.previousFailureCount()) {
446             Credential credential = CredentialStorage::defaultCredentialStorage().get(partition, challenge.protectionSpace());
447             if (!credential.isEmpty() && credential != d->m_initialCredential) {
448                 ASSERT(credential.persistence() == CredentialPersistenceNone);
449                 if (challenge.failureResponse().httpStatusCode() == 401) {
450                     // Store the credential back, possibly adding it as a default for this directory.
451                     CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), challenge.failureResponse().url());
452                 }
453
454                 d->m_curlHandle.setHttpAuthUserPass(credential.user(), credential.password());
455                 return;
456             }
457         }
458     }
459
460     d->m_currentWebChallenge = challenge;
461     
462     if (client())
463         client()->didReceiveAuthenticationChallenge(this, d->m_currentWebChallenge);
464 }
465
466 void ResourceHandle::receivedCredential(const AuthenticationChallenge& challenge, const Credential& credential)
467 {
468     ASSERT(isMainThread());
469
470     if (challenge != d->m_currentWebChallenge)
471         return;
472
473     if (credential.isEmpty()) {
474         receivedRequestToContinueWithoutCredential(challenge);
475         return;
476     }
477
478     String partition = firstRequest().cachePartition();
479
480     if (shouldUseCredentialStorage()) {
481         if (challenge.failureResponse().httpStatusCode() == 401) {
482             URL urlToStore = challenge.failureResponse().url();
483             CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
484         }
485     }
486
487     d->m_curlHandle.setHttpAuthUserPass(credential.user(), credential.password());
488     clearAuthentication();
489 }
490
491 void ResourceHandle::receivedRequestToContinueWithoutCredential(const AuthenticationChallenge& challenge)
492 {
493     ASSERT(isMainThread());
494
495     if (challenge != d->m_currentWebChallenge)
496         return;
497
498     d->m_curlHandle.setHttpAuthUserPass("", "");
499     clearAuthentication();
500 }
501
502 void ResourceHandle::receivedCancellation(const AuthenticationChallenge& challenge)
503 {
504     ASSERT(isMainThread());
505
506     if (challenge != d->m_currentWebChallenge)
507         return;
508
509     if (client())
510         client()->receivedCancellation(this, challenge);
511 }
512
513 void ResourceHandle::receivedRequestToPerformDefaultHandling(const AuthenticationChallenge&)
514 {
515     ASSERT_NOT_REACHED();
516 }
517
518 void ResourceHandle::receivedChallengeRejection(const AuthenticationChallenge&)
519 {
520     ASSERT_NOT_REACHED();
521 }
522
523 #if ENABLE(WEB_TIMING)
524 void ResourceHandleInternal::calculateWebTimingInformations()
525 {
526     double preTransferTime = 0;
527     double dnslookupTime = 0;
528     double connectTime = 0;
529     double appConnectTime = 0;
530
531     m_curlHandle.getTimes(preTransferTime, dnslookupTime, connectTime, appConnectTime);
532
533     m_response.deprecatedNetworkLoadMetrics().domainLookupStart = Seconds(0);
534     m_response.deprecatedNetworkLoadMetrics().domainLookupEnd = Seconds(dnslookupTime);
535
536     m_response.deprecatedNetworkLoadMetrics().connectStart = Seconds(dnslookupTime);
537     m_response.deprecatedNetworkLoadMetrics().connectEnd = Seconds(connectTime);
538
539     m_response.deprecatedNetworkLoadMetrics().requestStart = Seconds(connectTime);
540     m_response.deprecatedNetworkLoadMetrics().responseStart = Seconds(preTransferTime);
541
542     if (appConnectTime)
543         m_response.deprecatedNetworkLoadMetrics().secureConnectionStart = Seconds(connectTime);
544 }
545 #endif
546
547 void ResourceHandleInternal::handleLocalReceiveResponse()
548 {
549     ASSERT(isMainThread());
550
551     // since the code in headerCallback will not have run for local files
552     // the code to set the URL and fire didReceiveResponse is never run,
553     // which means the ResourceLoader's response does not contain the URL.
554     // Run the code here for local files to resolve the issue.
555     // TODO: See if there is a better approach for handling this.
556     URL url = m_curlHandle.getEffectiveURL();
557     ASSERT(url.isValid());
558     m_response.setURL(url);
559     if (client())
560         client()->didReceiveResponse(m_handle, ResourceResponse(m_response));
561     m_response.setResponseFired(true);
562 }
563
564 inline static bool isHttpInfo(int statusCode)
565 {
566     return 100 <= statusCode && statusCode < 200;
567 }
568
569 inline static bool isHttpRedirect(int statusCode)
570 {
571     return 300 <= statusCode && statusCode < 400 && statusCode != 304;
572 }
573
574 inline static bool isHttpAuthentication(int statusCode)
575 {
576     return statusCode == 401;
577 }
578
579 inline static bool isHttpNotModified(int statusCode)
580 {
581     return statusCode == 304;
582 }
583
584 static bool isAppendableHeader(const String &key)
585 {
586     static const char* appendableHeaders[] = {
587         "access-control-allow-headers",
588         "access-control-allow-methods",
589         "access-control-allow-origin",
590         "access-control-expose-headers",
591         "allow",
592         "cache-control",
593         "connection",
594         "content-encoding",
595         "content-language",
596         "if-match",
597         "if-none-match",
598         "keep-alive",
599         "pragma",
600         "proxy-authenticate",
601         "public",
602         "server",
603         "set-cookie",
604         "te",
605         "trailer",
606         "transfer-encoding",
607         "upgrade",
608         "user-agent",
609         "vary",
610         "via",
611         "warning",
612         "www-authenticate"
613     };
614
615     // Custom headers start with 'X-', and need no further checking.
616     if (key.startsWith("x-", /* caseSensitive */ false))
617         return true;
618
619     for (auto& header : appendableHeaders) {
620         if (equalIgnoringASCIICase(key, header))
621             return true;
622     }
623
624     return false;
625 }
626
627 static void removeLeadingAndTrailingQuotes(String& value)
628 {
629     unsigned length = value.length();
630     if (value.startsWith('"') && value.endsWith('"') && length > 1)
631         value = value.substring(1, length - 2);
632 }
633
634 static bool getProtectionSpace(ResourceHandle* job, const ResourceResponse& response, ProtectionSpace& protectionSpace)
635 {
636     ResourceHandleInternal* d = job->getInternal();
637
638     CURLcode err;
639
640     long port = 0;
641     err = d->m_curlHandle.getPrimaryPort(port);
642     if (err != CURLE_OK)
643         return false;
644
645     long availableAuth = CURLAUTH_NONE;
646     err = d->m_curlHandle.getHttpAuthAvail(availableAuth);
647     if (err != CURLE_OK)
648         return false;
649
650     URL url = d->m_curlHandle.getEffectiveURL();
651     if (!url.isValid())
652         return false;
653
654     String host = url.host();
655     StringView protocol = url.protocol();
656
657     String realm;
658
659     const String authHeader = response.httpHeaderField(HTTPHeaderName::Authorization);
660     const String realmString = "realm=";
661     int realmPos = authHeader.find(realmString);
662     if (realmPos > 0) {
663         realm = authHeader.substring(realmPos + realmString.length());
664         realm = realm.left(realm.find(','));
665         removeLeadingAndTrailingQuotes(realm);
666     }
667
668     ProtectionSpaceServerType serverType = ProtectionSpaceServerHTTP;
669     if (protocol == "https")
670         serverType = ProtectionSpaceServerHTTPS;
671
672     ProtectionSpaceAuthenticationScheme authScheme = ProtectionSpaceAuthenticationSchemeUnknown;
673
674     if (availableAuth & CURLAUTH_BASIC)
675         authScheme = ProtectionSpaceAuthenticationSchemeHTTPBasic;
676     if (availableAuth & CURLAUTH_DIGEST)
677         authScheme = ProtectionSpaceAuthenticationSchemeHTTPDigest;
678     if (availableAuth & CURLAUTH_GSSNEGOTIATE)
679         authScheme = ProtectionSpaceAuthenticationSchemeNegotiate;
680     if (availableAuth & CURLAUTH_NTLM)
681         authScheme = ProtectionSpaceAuthenticationSchemeNTLM;
682
683     protectionSpace = ProtectionSpace(host, port, serverType, realm, authScheme);
684
685     return true;
686 }
687
688 size_t ResourceHandleInternal::willPrepareSendData(char* ptr, size_t blockSize, size_t numberOfBlocks)
689 {
690     if (!m_formDataStream.hasMoreElements())
691         return 0;
692
693     size_t size = m_formDataStream.read(ptr, blockSize, numberOfBlocks);
694
695     // Something went wrong so cancel the job.
696     if (!size) {
697         m_handle->cancel();
698         return 0;
699     }
700
701     return size;
702
703 }
704
705 void ResourceHandleInternal::didReceiveHeaderLine(const String& header)
706 {
707     ASSERT(isMainThread());
708
709     auto splitPosition = header.find(":");
710     if (splitPosition != notFound) {
711         String key = header.left(splitPosition).stripWhiteSpace();
712         String value = header.substring(splitPosition + 1).stripWhiteSpace();
713
714         if (isAppendableHeader(key))
715             m_response.addHTTPHeaderField(key, value);
716         else
717             m_response.setHTTPHeaderField(key, value);
718     } else if (header.startsWith("HTTP", false)) {
719         // This is the first line of the response.
720         // Extract the http status text from this.
721         //
722         // If the FOLLOWLOCATION option is enabled for the curl handle then
723         // curl will follow the redirections internally. Thus this header callback
724         // will be called more than one time with the line starting "HTTP" for one job.
725         long httpCode = 0;
726         m_curlHandle.getResponseCode(httpCode);
727
728         String httpCodeString = String::number(httpCode);
729         int statusCodePos = header.find(httpCodeString);
730
731         if (statusCodePos != notFound) {
732             // The status text is after the status code.
733             String status = header.substring(statusCodePos + httpCodeString.length());
734             m_response.setHTTPStatusText(status.stripWhiteSpace());
735         }
736     }
737 }
738
739 void ResourceHandleInternal::didReceiveAllHeaders(long httpCode, long long contentLength)
740 {
741     ASSERT(isMainThread());
742
743     m_response.setExpectedContentLength(contentLength);
744
745     m_response.setURL(m_curlHandle.getEffectiveURL());
746
747     m_response.setHTTPStatusCode(httpCode);
748     m_response.setMimeType(extractMIMETypeFromMediaType(m_response.httpHeaderField(HTTPHeaderName::ContentType)).convertToASCIILowercase());
749     m_response.setTextEncodingName(extractCharsetFromMediaType(m_response.httpHeaderField(HTTPHeaderName::ContentType)));
750
751     if (m_response.isMultipart()) {
752         String boundary;
753         bool parsed = MultipartHandle::extractBoundary(m_response.httpHeaderField(HTTPHeaderName::ContentType), boundary);
754         if (parsed)
755             m_multipartHandle = std::make_unique<MultipartHandle>(m_handle, boundary);
756     }
757
758     // HTTP redirection
759     if (isHttpRedirect(httpCode)) {
760         String location = m_response.httpHeaderField(HTTPHeaderName::Location);
761         if (!location.isEmpty()) {
762             URL newURL = URL(m_firstRequest.url(), location);
763
764             ResourceRequest redirectedRequest = m_firstRequest;
765             redirectedRequest.setURL(newURL);
766             ResourceResponse response = m_response;
767             if (client())
768                 client()->willSendRequest(m_handle, WTFMove(redirectedRequest), WTFMove(response));
769
770             m_firstRequest.setURL(newURL);
771
772             return;
773         }
774     } else if (isHttpAuthentication(httpCode)) {
775         ProtectionSpace protectionSpace;
776         if (getProtectionSpace(m_handle, m_response, protectionSpace)) {
777             Credential credential;
778             AuthenticationChallenge challenge(protectionSpace, credential, m_authFailureCount, m_response, ResourceError());
779             challenge.setAuthenticationClient(m_handle);
780             m_handle->didReceiveAuthenticationChallenge(challenge);
781             m_authFailureCount++;
782             return;
783         }
784     }
785
786     if (client()) {
787         if (isHttpNotModified(httpCode)) {
788             const String& url = m_firstRequest.url().string();
789             if (CurlCacheManager::getInstance().getCachedResponse(url, m_response)) {
790                 if (m_addedCacheValidationHeaders) {
791                     m_response.setHTTPStatusCode(200);
792                     m_response.setHTTPStatusText("OK");
793                 }
794             }
795         }
796         client()->didReceiveResponse(m_handle, ResourceResponse(m_response));
797         CurlCacheManager::getInstance().didReceiveResponse(*m_handle, m_response);
798     }
799
800     m_response.setResponseFired(true);
801 }
802
803 void ResourceHandleInternal::didReceiveContentData()
804 {
805     ASSERT(isMainThread());
806
807     if (!m_response.responseFired())
808         handleLocalReceiveResponse();
809
810     Vector<char> buffer;
811     {
812         LockHolder locker { m_receivedBufferMutex };
813         buffer = WTFMove(m_receivedBuffer);
814     }
815
816     char* ptr = buffer.begin();
817     size_t size = buffer.size();
818
819     if (m_multipartHandle)
820         m_multipartHandle->contentReceived(static_cast<const char*>(ptr), size);
821     else if (client()) {
822         client()->didReceiveData(m_handle, ptr, size, 0);
823         CurlCacheManager::getInstance().didReceiveData(*m_handle, ptr, size);
824     }
825 }
826
827 /* This is called to obtain HTTP POST or PUT data.
828 Iterate through FormData elements and upload files.
829 Carefully respect the given buffer size and fill the rest of the data at the next calls.
830 */
831 size_t ResourceHandleInternal::readCallback(char* ptr, size_t size, size_t nmemb, void* data)
832 {
833     ASSERT(!isMainThread());
834
835     ResourceHandleInternal* d = static_cast<ResourceHandleInternal*>(data);
836
837     if (d->m_cancelled)
838         return 0;
839
840     // We should never be called when deferred loading is activated.
841     ASSERT(!d->m_defersLoading);
842
843     if (!size || !nmemb)
844         return 0;
845
846     return d->willPrepareSendData(ptr, size, nmemb);
847 }
848
849 /*
850 * This is being called for each HTTP header in the response. This includes '\r\n'
851 * for the last line of the header.
852 *
853 * We will add each HTTP Header to the ResourceResponse and on the termination
854 * of the header (\r\n) we will parse Content-Type and Content-Disposition and
855 * update the ResourceResponse and then send it away.
856 *
857 */
858 size_t ResourceHandleInternal::headerCallback(char* ptr, size_t size, size_t nmemb, void* data)
859 {
860     ASSERT(!isMainThread());
861
862     ResourceHandleInternal* d = static_cast<ResourceHandleInternal*>(data);
863     ResourceHandle* job = d->m_handle;
864
865     if (d->m_cancelled)
866         return 0;
867
868     // We should never be called when deferred loading is activated.
869     ASSERT(!d->m_defersLoading);
870
871     size_t totalSize = size * nmemb;
872
873     String header(static_cast<const char*>(ptr), totalSize);
874
875     /*
876     * a) We can finish and send the ResourceResponse
877     * b) We will add the current header to the HTTPHeaderMap of the ResourceResponse
878     *
879     * The HTTP standard requires to use \r\n but for compatibility it recommends to
880     * accept also \n.
881     */
882     if (header == AtomicString("\r\n") || header == AtomicString("\n")) {
883         long httpCode = 0;
884         d->m_curlHandle.getResponseCode(httpCode);
885
886         if (!httpCode) {
887             // Comes here when receiving 200 Connection Established. Just return.
888             return totalSize;
889         }
890         if (isHttpInfo(httpCode)) {
891             // Just return when receiving http info, e.g. HTTP/1.1 100 Continue.
892             // If not, the request might be cancelled, because the MIME type will be empty for this response.
893             return totalSize;
894         }
895
896         long long contentLength = 0;
897         d->m_curlHandle.getContentLenghtDownload(contentLength);
898
899         callOnMainThread([job = RefPtr<ResourceHandle>(job), d, httpCode, contentLength] {
900             if (!d->m_cancelled)
901                 d->didReceiveAllHeaders(httpCode, contentLength);
902         });
903     } else {
904         callOnMainThread([job = RefPtr<ResourceHandle>(job), d, header] {
905             if (!d->m_cancelled)
906                 d->didReceiveHeaderLine(header);
907         });
908     }
909
910     return totalSize;
911 }
912
913 // called with data after all headers have been processed via headerCallback
914 size_t ResourceHandleInternal::writeCallback(char* ptr, size_t size, size_t nmemb, void* data)
915 {
916     ASSERT(!isMainThread());
917
918     ResourceHandleInternal* d = static_cast<ResourceHandleInternal*>(data);
919     ResourceHandle* job = d->m_handle;
920
921     if (d->m_cancelled)
922         return 0;
923
924     // We should never be called when deferred loading is activated.
925     ASSERT(!d->m_defersLoading);
926
927     size_t totalSize = size * nmemb;
928
929     // this shouldn't be necessary but apparently is. CURL writes the data
930     // of html page even if it is a redirect that was handled internally
931     // can be observed e.g. on gmail.com
932     long httpCode = 0;
933     CURLcode errCd = d->m_curlHandle.getResponseCode(httpCode);
934     if (CURLE_OK == errCd && httpCode >= 300 && httpCode < 400)
935         return totalSize;
936
937     bool shouldCall { false };
938     {
939         LockHolder locker(d->m_receivedBufferMutex);
940         
941         if (d->m_receivedBuffer.isEmpty())
942             shouldCall = true;
943         
944         d->m_receivedBuffer.append(ptr, totalSize);
945     }
946
947     if (shouldCall) {
948         callOnMainThread([job = RefPtr<ResourceHandle>(job), d] {
949             if (!d->m_cancelled)
950                 d->didReceiveContentData();
951         });
952     }
953
954     return totalSize;
955 }
956
957 // sync loader
958
959 void ResourceHandle::platformLoadResourceSynchronously(NetworkingContext* context, const ResourceRequest& request, StoredCredentials, ResourceError& error, ResourceResponse& response, Vector<char>& data)
960 {
961     ASSERT(isMainThread());
962
963     SynchronousLoaderClient client;
964     RefPtr<ResourceHandle> handle = adoptRef(new ResourceHandle(context, request, &client, false, false));
965
966     handle->d->dispatchSynchronousJob();
967
968     error = client.error();
969     data.swap(client.mutableData());
970     response = client.response();
971 }
972
973 void ResourceHandleInternal::dispatchSynchronousJob()
974 {
975     URL kurl = m_firstRequest.url();
976
977     if (kurl.protocolIsData()) {
978         handleDataURL();
979         return;
980     }
981
982     // If defersLoading is true and we call curl_easy_perform
983     // on a paused handle, libcURL would do the transfert anyway
984     // and we would assert so force defersLoading to be false.
985     m_defersLoading = false;
986
987     initialize();
988
989     // curl_easy_perform blocks until the transfert is finished.
990     CURLcode ret = m_curlHandle.perform();
991
992 #if ENABLE(WEB_TIMING)
993     calculateWebTimingInformations();
994 #endif
995
996     if (client()) {
997         if (ret != CURLE_OK)
998             client()->didFail(m_handle, ResourceError(m_curlHandle, m_sslErrors));
999         else
1000             client()->didReceiveResponse(m_handle, ResourceResponse(m_response));
1001     }
1002 }
1003
1004 void ResourceHandleInternal::handleDataURL()
1005 {
1006     ASSERT(m_firstRequest.url().protocolIsData());
1007     String url = m_firstRequest.url().string();
1008
1009     ASSERT(client());
1010
1011     int index = url.find(',');
1012     if (index == -1) {
1013         client()->cannotShowURL(m_handle);
1014         return;
1015     }
1016
1017     String mediaType = url.substring(5, index - 5);
1018     String data = url.substring(index + 1);
1019
1020     bool base64 = mediaType.endsWith(";base64", false);
1021     if (base64)
1022         mediaType = mediaType.left(mediaType.length() - 7);
1023
1024     if (mediaType.isEmpty())
1025         mediaType = "text/plain";
1026
1027     String mimeType = extractMIMETypeFromMediaType(mediaType);
1028     String charset = extractCharsetFromMediaType(mediaType);
1029
1030     if (charset.isEmpty())
1031         charset = "US-ASCII";
1032
1033     ResourceResponse response;
1034     response.setMimeType(mimeType);
1035     response.setTextEncodingName(charset);
1036     response.setURL(m_firstRequest.url());
1037
1038     if (base64) {
1039         data = decodeURLEscapeSequences(data);
1040         client()->didReceiveResponse(m_handle, WTFMove(response));
1041
1042         // didReceiveResponse might cause the client to be deleted.
1043         if (client()) {
1044             Vector<char> out;
1045             if (base64Decode(data, out, Base64IgnoreSpacesAndNewLines) && out.size() > 0)
1046                 client()->didReceiveData(m_handle, out.data(), out.size(), 0);
1047         }
1048     } else {
1049         TextEncoding encoding(charset);
1050         data = decodeURLEscapeSequences(data, encoding);
1051         client()->didReceiveResponse(m_handle, WTFMove(response));
1052
1053         // didReceiveResponse might cause the client to be deleted.
1054         if (client()) {
1055             CString encodedData = encoding.encode(data, URLEncodedEntitiesForUnencodables);
1056             if (encodedData.length())
1057                 client()->didReceiveData(m_handle, encodedData.data(), encodedData.length(), 0);
1058         }
1059     }
1060
1061     if (client())
1062         client()->didFinishLoading(m_handle);
1063 }
1064
1065 } // namespace WebCore
1066
1067 #endif